diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 5b1af29590..43f5234d31 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -60,7 +60,6 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.utils.DateTimeUtils; /** @@ -135,7 +134,6 @@ public void register(BuiltinFunctionRepository repository) { * (DATE, LONG) -> DATE * (STRING/DATETIME/TIMESTAMP, LONG) -> DATETIME */ - private DefaultFunctionResolver add_date(FunctionName functionName) { return define(functionName, impl(nullMissingHandling(DateTimeFunction::exprAddDateInterval), @@ -171,38 +169,38 @@ private DefaultFunctionResolver convert_tz() { ); } - private FunctionResolver curdate(FunctionName functionName) { + private DefaultFunctionResolver curdate(FunctionName functionName) { return define(functionName, impl(() -> new ExprDateValue(formatNow(null).toLocalDate()), DATE) ); } - private FunctionResolver curdate() { + private DefaultFunctionResolver curdate() { return curdate(BuiltinFunctionName.CURDATE.getName()); } /** * Synonym for @see `now`. */ - private FunctionResolver curtime(FunctionName functionName) { + private DefaultFunctionResolver curtime(FunctionName functionName) { return define(functionName, impl(() -> new ExprTimeValue(formatNow(null).toLocalTime()), TIME) ); } - private FunctionResolver curtime() { + private DefaultFunctionResolver curtime() { return curtime(BuiltinFunctionName.CURTIME.getName()); } - private FunctionResolver current_date() { + private DefaultFunctionResolver current_date() { return curdate(BuiltinFunctionName.CURRENT_DATE.getName()); } - private FunctionResolver current_time() { + private DefaultFunctionResolver current_time() { return curtime(BuiltinFunctionName.CURRENT_TIME.getName()); } - private FunctionResolver current_timestamp() { + private DefaultFunctionResolver current_timestamp() { return now(BuiltinFunctionName.CURRENT_TIMESTAMP.getName()); } @@ -225,7 +223,7 @@ private DefaultFunctionResolver date() { * (STRING, STRING) -> DATETIME * (STRING) -> DATETIME */ - private FunctionResolver datetime() { + private DefaultFunctionResolver datetime() { return define(BuiltinFunctionName.DATETIME.getName(), impl(nullMissingHandling(DateTimeFunction::exprDateTime), DATETIME, STRING, STRING), @@ -337,7 +335,7 @@ private DefaultFunctionResolver from_days() { impl(nullMissingHandling(DateTimeFunction::exprFromDays), DATE, LONG)); } - private FunctionResolver from_unixtime() { + private DefaultFunctionResolver from_unixtime() { return define(BuiltinFunctionName.FROM_UNIXTIME.getName(), impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, DOUBLE), impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), @@ -356,11 +354,11 @@ private DefaultFunctionResolver hour() { ); } - private FunctionResolver localtime() { + private DefaultFunctionResolver localtime() { return now(BuiltinFunctionName.LOCALTIME.getName()); } - private FunctionResolver localtimestamp() { + private DefaultFunctionResolver localtimestamp() { return now(BuiltinFunctionName.LOCALTIMESTAMP.getName()); } @@ -369,22 +367,22 @@ private FunctionResolver localtimestamp() { * `fsp` argument support is removed until refactoring to avoid bug where `now()`, `now(x)` and * `now(y) return different values. */ - private FunctionResolver now(FunctionName functionName) { + private DefaultFunctionResolver now(FunctionName functionName) { return define(functionName, impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME) ); } - private FunctionResolver now() { + private DefaultFunctionResolver now() { return now(BuiltinFunctionName.NOW.getName()); } - private FunctionResolver makedate() { + private DefaultFunctionResolver makedate() { return define(BuiltinFunctionName.MAKEDATE.getName(), impl(nullMissingHandling(DateTimeFunction::exprMakeDate), DATE, DOUBLE, DOUBLE)); } - private FunctionResolver maketime() { + private DefaultFunctionResolver maketime() { return define(BuiltinFunctionName.MAKETIME.getName(), impl(nullMissingHandling(DateTimeFunction::exprMakeTime), TIME, DOUBLE, DOUBLE, DOUBLE)); } @@ -536,7 +534,7 @@ private DefaultFunctionResolver to_days() { impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, DATETIME)); } - private FunctionResolver unix_timestamp() { + private DefaultFunctionResolver unix_timestamp() { return define(BuiltinFunctionName.UNIX_TIMESTAMP.getName(), impl(DateTimeFunction::unixTimeStamp, LONG), impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATE), @@ -1008,7 +1006,7 @@ private ExprValue exprSubDateInterval(ExprValue date, ExprValue expr) { /** * SYSDATE() returns the time at which it executes. */ - private FunctionResolver sysdate() { + private DefaultFunctionResolver sysdate() { return define(BuiltinFunctionName.SYSDATE.getName(), impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME), impl((v) -> new ExprDatetimeValue(formatNow(v.integerValue())), DATETIME, INTEGER) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/utils/QueryDataAnonymizer.java b/legacy/src/main/java/org/opensearch/sql/legacy/utils/QueryDataAnonymizer.java index 91406333ae..b58691c022 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/utils/QueryDataAnonymizer.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/utils/QueryDataAnonymizer.java @@ -26,7 +26,8 @@ public class QueryDataAnonymizer { * Sensitive data includes index names, column names etc., * which in druid parser are parsed to SQLIdentifierExpr instances * @param query entire sql query string - * @return sql query string with all identifiers replaced with "***" + * @return sql query string with all identifiers replaced with "***" on success + * and failure string otherwise to ensure no non-anonymized data is logged in production. */ public static String anonymizeData(String query) { String resultQuery; @@ -38,8 +39,9 @@ public static String anonymizeData(String query) { .replaceAll("false", "boolean_literal") .replaceAll("[\\n][\\t]+", " "); } catch (Exception e) { - LOG.warn("Caught an exception when anonymizing sensitive data"); - resultQuery = query; + LOG.warn("Caught an exception when anonymizing sensitive data."); + LOG.debug("String {} failed anonymization.", query); + resultQuery = "Failed to anonymize data."; } return resultQuery; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/FunctionParameterRepository.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/FunctionParameterRepository.java new file mode 100644 index 0000000000..373df4e5fc --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/FunctionParameterRepository.java @@ -0,0 +1,355 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; + +import com.google.common.collect.ImmutableMap; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.experimental.UtilityClass; +import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.index.query.MatchBoolPrefixQueryBuilder; +import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; +import org.opensearch.index.query.MatchPhraseQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.Operator; +import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.index.query.SimpleQueryStringBuilder; +import org.opensearch.index.query.SimpleQueryStringFlag; +import org.opensearch.index.query.support.QueryParsers; +import org.opensearch.index.search.MatchQuery; +import org.opensearch.sql.data.model.ExprValue; + +@UtilityClass +public class FunctionParameterRepository { + + public static final Map> + MatchBoolPrefixQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("fuzziness", (b, v) -> b.fuzziness(convertFuzziness(v))) + .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(checkRewrite(v, "fuzzy_rewrite"))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("max_expansions", (b, v) -> b.maxExpansions(convertIntValue(v, "max_expansions"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("operator", (b, v) -> b.operator(convertOperator(v, "operator"))) + .put("prefix_length", (b, v) -> b.prefixLength(convertIntValue(v, "prefix_length"))) + .build(); + + public static final Map> + MatchPhrasePrefixQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("max_expansions", (b, v) -> b.maxExpansions(convertIntValue(v, "max_expansions"))) + .put("slop", (b, v) -> b.slop(convertIntValue(v, "slop"))) + .put("zero_terms_query", (b, v) -> b.zeroTermsQuery(convertZeroTermsQuery(v))) + .build(); + + public static final Map> + MatchPhraseQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("slop", (b, v) -> b.slop(convertIntValue(v, "slop"))) + .put("zero_terms_query", (b, v) -> b.zeroTermsQuery(convertZeroTermsQuery(v))) + .build(); + + public static final Map> + MatchQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("auto_generate_synonyms_phrase_query", (b, v) -> b.autoGenerateSynonymsPhraseQuery( + convertBoolValue(v, "auto_generate_synonyms_phrase_query"))) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("fuzziness", (b, v) -> b.fuzziness(convertFuzziness(v))) + .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(checkRewrite(v, "fuzzy_rewrite"))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("lenient", (b, v) -> b.lenient(convertBoolValue(v, "lenient"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("max_expansions", (b, v) -> b.maxExpansions(convertIntValue(v, "max_expansions"))) + .put("operator", (b, v) -> b.operator(convertOperator(v, "operator"))) + .put("prefix_length", (b, v) -> b.prefixLength(convertIntValue(v, "prefix_length"))) + .put("zero_terms_query", (b, v) -> b.zeroTermsQuery(convertZeroTermsQuery(v))) + .build(); + + @SuppressWarnings("deprecation") // cutoffFrequency is deprecated + public static final Map> + MultiMatchQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("auto_generate_synonyms_phrase_query", (b, v) -> b.autoGenerateSynonymsPhraseQuery( + convertBoolValue(v, "auto_generate_synonyms_phrase_query"))) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("cutoff_frequency", (b, v) -> b.cutoffFrequency( + convertFloatValue(v, "cutoff_frequency"))) + .put("fuzziness", (b, v) -> b.fuzziness(convertFuzziness(v))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("lenient", (b, v) -> b.lenient(convertBoolValue(v, "lenient"))) + .put("max_expansions", (b, v) -> b.maxExpansions(convertIntValue(v, "max_expansions"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("operator", (b, v) -> b.operator(convertOperator(v, "operator"))) + .put("prefix_length", (b, v) -> b.prefixLength(convertIntValue(v, "prefix_length"))) + .put("slop", (b, v) -> b.slop(convertIntValue(v, "slop"))) + .put("tie_breaker", (b, v) -> b.tieBreaker(convertFloatValue(v, "tie_breaker"))) + .put("type", (b, v) -> b.type(convertType(v))) + .put("zero_terms_query", (b, v) -> b.zeroTermsQuery(convertZeroTermsQuery(v))) + .build(); + + public static final Map> + QueryStringQueryBuildActions = ImmutableMap.>builder() + .put("allow_leading_wildcard", (b, v) -> b.allowLeadingWildcard( + convertBoolValue(v, "allow_leading_wildcard"))) + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("analyze_wildcard", (b, v) -> b.analyzeWildcard( + convertBoolValue(v, "analyze_wildcard"))) + .put("auto_generate_synonyms_phrase_query", (b, v) -> b.autoGenerateSynonymsPhraseQuery( + convertBoolValue(v, "auto_generate_synonyms_phrase_query"))) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("default_operator", (b, v) -> b.defaultOperator( + convertOperator(v, "default_operator"))) + .put("enable_position_increments", (b, v) -> b.enablePositionIncrements( + convertBoolValue(v, "enable_position_increments"))) + .put("escape", (b, v) -> b.escape(convertBoolValue(v, "escape"))) + .put("fuzziness", (b, v) -> b.fuzziness(convertFuzziness(v))) + .put("fuzzy_max_expansions", (b, v) -> b.fuzzyMaxExpansions( + convertIntValue(v, "fuzzy_max_expansions"))) + .put("fuzzy_prefix_length", (b, v) -> b.fuzzyPrefixLength( + convertIntValue(v, "fuzzy_prefix_length"))) + .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(checkRewrite(v, "fuzzy_rewrite"))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("lenient", (b, v) -> b.lenient(convertBoolValue(v, "lenient"))) + .put("max_determinized_states", (b, v) -> b.maxDeterminizedStates( + convertIntValue(v, "max_determinized_states"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("phrase_slop", (b, v) -> b.phraseSlop(convertIntValue(v, "phrase_slop"))) + .put("quote_analyzer", (b, v) -> b.quoteAnalyzer(v.stringValue())) + .put("quote_field_suffix", (b, v) -> b.quoteFieldSuffix(v.stringValue())) + .put("rewrite", (b, v) -> b.rewrite(checkRewrite(v, "rewrite"))) + .put("tie_breaker", (b, v) -> b.tieBreaker(convertFloatValue(v, "tie_breaker"))) + .put("time_zone", (b, v) -> b.timeZone(checkTimeZone(v))) + .put("type", (b, v) -> b.type(convertType(v))) + .build(); + + public static final Map> + SimpleQueryStringQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("analyze_wildcard", (b, v) -> b.analyzeWildcard( + convertBoolValue(v, "analyze_wildcard"))) + .put("auto_generate_synonyms_phrase_query", (b, v) -> b.autoGenerateSynonymsPhraseQuery( + convertBoolValue(v, "auto_generate_synonyms_phrase_query"))) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("default_operator", (b, v) -> b.defaultOperator( + convertOperator(v, "default_operator"))) + .put("flags", (b, v) -> b.flags(convertFlags(v))) + .put("fuzzy_max_expansions", (b, v) -> b.fuzzyMaxExpansions( + convertIntValue(v, "fuzzy_max_expansions"))) + .put("fuzzy_prefix_length", (b, v) -> b.fuzzyPrefixLength( + convertIntValue(v, "fuzzy_prefix_length"))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("lenient", (b, v) -> b.lenient(convertBoolValue(v, "lenient"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("quote_field_suffix", (b, v) -> b.quoteFieldSuffix(v.stringValue())) + .build(); + + public static final Map ArgumentLimitations = + ImmutableMap.builder() + .put("boost", "Accepts only floating point values greater than 0.") + .put("tie_breaker", "Accepts only floating point values in range 0 to 1.") + .put("rewrite", "Available values are: constant_score, " + + "scoring_boolean, constant_score_boolean, top_terms_X, top_terms_boost_X, " + + "top_terms_blended_freqs_X, where X is an integer value.") + .put("flags", String.format( + "Available values are: %s and any combinations of these separated by '|'.", + Arrays.stream(SimpleQueryStringFlag.class.getEnumConstants()) + .map(Enum::toString).collect(Collectors.joining(", ")))) + .put("time_zone", "For more information, follow this link: " + + "https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html#of-java.lang.String-") + .put("fuzziness", "Available values are: " + + "'AUTO', 'AUTO:x,y' or z, where x, y, z - integer values.") + .put("operator", String.format("Available values are: %s.", + Arrays.stream(Operator.class.getEnumConstants()) + .map(Enum::toString).collect(Collectors.joining(", ")))) + .put("type", String.format("Available values are: %s.", + Arrays.stream(MultiMatchQueryBuilder.Type.class.getEnumConstants()) + .map(Enum::toString).collect(Collectors.joining(", ")))) + .put("zero_terms_query", String.format("Available values are: %s.", + Arrays.stream(MatchQuery.ZeroTermsQuery.class.getEnumConstants()) + .map(Enum::toString).collect(Collectors.joining(", ")))) + .put("int", "Accepts only integer values.") + .put("float", "Accepts only floating point values.") + .put("bool", "Accepts only boolean values: 'true' or 'false'.") + .build(); + + + private static String formatErrorMessage(String name, String value) { + return formatErrorMessage(name, value, name); + } + + private static String formatErrorMessage(String name, String value, String limitationName) { + return String.format("Invalid %s value: '%s'. %s", + name, value, ArgumentLimitations.containsKey(name) ? ArgumentLimitations.get(name) + : ArgumentLimitations.getOrDefault(limitationName, "")); + } + + /** + * Check whether value is valid for 'rewrite' or 'fuzzy_rewrite'. + * @param value Value + * @param name Value name + * @return Converted + */ + public static String checkRewrite(ExprValue value, String name) { + try { + QueryParsers.parseRewriteMethod( + value.stringValue().toLowerCase(), null, LoggingDeprecationHandler.INSTANCE); + return value.stringValue(); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "rewrite")); + } + } + + /** + * Convert ExprValue to Flags. + * @param value Value + * @return Array of flags + */ + public static SimpleQueryStringFlag[] convertFlags(ExprValue value) { + try { + return Arrays.stream(value.stringValue().toUpperCase().split("\\|")) + .map(SimpleQueryStringFlag::valueOf) + .toArray(SimpleQueryStringFlag[]::new); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("flags", value.stringValue()), e); + } + } + + /** + * Check whether ExprValue could be converted to timezone object. + * @param value Value + * @return Converted to string + */ + public static String checkTimeZone(ExprValue value) { + try { + ZoneId.of(value.stringValue()); + return value.stringValue(); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("time_zone", value.stringValue()), e); + } + } + + /** + * Convert ExprValue to Fuzziness object. + * @param value Value + * @return Fuzziness + */ + public static Fuzziness convertFuzziness(ExprValue value) { + try { + return Fuzziness.build(value.stringValue().toUpperCase()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("fuzziness", value.stringValue()), e); + } + } + + /** + * Convert ExprValue to Operator object, could be used for 'operator' and 'default_operator'. + * @param value Value + * @param name Value name + * @return Operator + */ + public static Operator convertOperator(ExprValue value, String name) { + try { + return Operator.fromString(value.stringValue().toUpperCase()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "operator")); + } + } + + /** + * Convert ExprValue to Type object. + * @param value Value + * @return Type + */ + public static MultiMatchQueryBuilder.Type convertType(ExprValue value) { + try { + return MultiMatchQueryBuilder.Type.parse(value.stringValue().toLowerCase(), + LoggingDeprecationHandler.INSTANCE); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("type", value.stringValue()), e); + } + } + + /** + * Convert ExprValue to ZeroTermsQuery object. + * @param value Value + * @return ZeroTermsQuery + */ + public static MatchQuery.ZeroTermsQuery convertZeroTermsQuery(ExprValue value) { + try { + return MatchQuery.ZeroTermsQuery.valueOf(value.stringValue().toUpperCase()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("zero_terms_query", value.stringValue()), e); + } + } + + /** + * Convert ExprValue to int. + * @param value Value + * @param name Value name + * @return int + */ + public static int convertIntValue(ExprValue value, String name) { + try { + return Integer.parseInt(value.stringValue()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "int"), e); + } + } + + /** + * Convert ExprValue to float. + * @param value Value + * @param name Value name + * @return float + */ + public static float convertFloatValue(ExprValue value, String name) { + try { + return Float.parseFloat(value.stringValue()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "float"), e); + } + } + + /** + * Convert ExprValue to bool. + * @param value Value + * @param name Value name + * @return bool + */ + public static boolean convertBoolValue(ExprValue value, String name) { + try { + // Boolean.parseBoolean interprets integers or any other stuff as a valid value + Boolean res = Boolean.parseBoolean(value.stringValue()); + if (value.stringValue().equalsIgnoreCase(res.toString())) { + return res; + } else { + throw new Exception("Invalid boolean value"); + } + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "bool"), e); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchBoolPrefixQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchBoolPrefixQuery.java index 33e357afe3..7044a56035 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchBoolPrefixQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchBoolPrefixQuery.java @@ -5,9 +5,7 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import com.google.common.collect.ImmutableMap; import org.opensearch.index.query.MatchBoolPrefixQueryBuilder; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilders; /** @@ -20,18 +18,7 @@ public class MatchBoolPrefixQuery * with support of optional parameters. */ public MatchBoolPrefixQuery() { - super(ImmutableMap.>builder() - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("fuzziness", (b, v) -> b.fuzziness(v.stringValue())) - .put("prefix_length", (b, v) -> b.prefixLength(Integer.parseInt(v.stringValue()))) - .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) - .put("fuzzy_transpositions", - (b, v) -> b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(v.stringValue())) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("operator", (b,v) -> b.operator(Operator.fromString(v.stringValue()))) - .build()); + super(FunctionParameterRepository.MatchBoolPrefixQueryBuildActions); } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java index 6d181daa4c..8ee9ae299e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java @@ -5,27 +5,19 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import com.google.common.collect.ImmutableMap; import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; import org.opensearch.index.query.QueryBuilders; /** * Lucene query that builds a match_phrase_prefix query. */ -public class MatchPhrasePrefixQuery extends SingleFieldQuery { +public class MatchPhrasePrefixQuery extends SingleFieldQuery { /** * Default constructor for MatchPhrasePrefixQuery configures how RelevanceQuery.build() handles * named arguments. */ public MatchPhrasePrefixQuery() { - super(ImmutableMap.>builder() - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("slop", (b, v) -> b.slop(Integer.parseInt(v.stringValue()))) - .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) - .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .build()); + super(FunctionParameterRepository.MatchPhrasePrefixQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java index 6a7694f629..2afaca1a7a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java @@ -5,20 +5,8 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import com.google.common.collect.ImmutableMap; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.function.BiFunction; import org.opensearch.index.query.MatchPhraseQueryBuilder; -import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.NamedArgumentExpression; -import org.opensearch.sql.opensearch.storage.script.filter.lucene.LuceneQuery; /** * Lucene query that builds a match_phrase query. @@ -29,13 +17,7 @@ public class MatchPhraseQuery extends SingleFieldQuery * named arguments. */ public MatchPhraseQuery() { - super(ImmutableMap.>builder() - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("slop", (b, v) -> b.slop(Integer.parseInt(v.stringValue()))) - .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) - .build()); + super(FunctionParameterRepository.MatchPhraseQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java index f6d88013e4..a4de1c0831 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java @@ -5,9 +5,7 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import com.google.common.collect.ImmutableMap; import org.opensearch.index.query.MatchQueryBuilder; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilders; /** @@ -19,23 +17,7 @@ public class MatchQuery extends SingleFieldQuery { * named arguments. */ public MatchQuery() { - super(ImmutableMap.>builder() - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("auto_generate_synonyms_phrase_query", - (b, v) -> b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) - .put("fuzziness", (b, v) -> b.fuzziness(valueOfToUpper(v))) - .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) - .put("prefix_length", (b, v) -> b.prefixLength(Integer.parseInt(v.stringValue()))) - .put("fuzzy_transpositions", - (b, v) -> b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(v.stringValue())) - .put("lenient", (b, v) -> b.lenient(Boolean.parseBoolean(v.stringValue()))) - .put("operator", (b, v) -> b.operator(Operator.fromString(v.stringValue()))) - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .build()); + super(FunctionParameterRepository.MatchQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java index b447f2ffe2..8390b5ef44 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java @@ -6,8 +6,10 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.NamedArgumentExpression; /** @@ -21,7 +23,18 @@ public MultiFieldQuery(Map> queryBuildActions) { } @Override - public T createQueryBuilder(NamedArgumentExpression fields, NamedArgumentExpression queryExpr) { + public T createQueryBuilder(List arguments) { + // Extract 'fields' and 'query' + var fields = arguments.stream() + .filter(a -> a.getArgName().equalsIgnoreCase("fields")) + .findFirst() + .orElseThrow(() -> new SemanticCheckException("'fields' parameter is missing.")); + + var query = arguments.stream() + .filter(a -> a.getArgName().equalsIgnoreCase("query")) + .findFirst() + .orElseThrow(() -> new SemanticCheckException("'query' parameter is missing")); + var fieldsAndWeights = fields .getValue() .valueOf(null) @@ -29,8 +42,8 @@ public T createQueryBuilder(NamedArgumentExpression fields, NamedArgumentExpress .entrySet() .stream() .collect(ImmutableMap.toImmutableMap(e -> e.getKey(), e -> e.getValue().floatValue())); - var query = queryExpr.getValue().valueOf(null).stringValue(); - return createBuilder(fieldsAndWeights, query); + + return createBuilder(fieldsAndWeights, query.getValue().valueOf(null).stringValue()); } protected abstract T createBuilder(ImmutableMap fields, String query); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java index 549f58cb19..a791bf756b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java @@ -7,7 +7,6 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.index.query.MultiMatchQueryBuilder; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilders; public class MultiMatchQuery extends MultiFieldQuery { @@ -16,26 +15,7 @@ public class MultiMatchQuery extends MultiFieldQuery { * named arguments. */ public MultiMatchQuery() { - super(ImmutableMap.>builder() - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("auto_generate_synonyms_phrase_query", (b, v) -> - b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("cutoff_frequency", (b, v) -> b.cutoffFrequency(Float.parseFloat(v.stringValue()))) - .put("fuzziness", (b, v) -> b.fuzziness(v.stringValue())) - .put("fuzzy_transpositions", (b, v) -> - b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("lenient", (b, v) -> b.lenient(Boolean.parseBoolean(v.stringValue()))) - .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("operator", (b, v) -> b.operator(Operator.fromString(v.stringValue()))) - .put("prefix_length", (b, v) -> b.prefixLength(Integer.parseInt(v.stringValue()))) - .put("tie_breaker", (b, v) -> b.tieBreaker(Float.parseFloat(v.stringValue()))) - .put("type", (b, v) -> b.type(v.stringValue())) - .put("slop", (b, v) -> b.slop(Integer.parseInt(v.stringValue()))) - .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) - .build()); + super(FunctionParameterRepository.MultiMatchQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java index 21eb3f8837..43131baa3e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java @@ -6,19 +6,8 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; import com.google.common.collect.ImmutableMap; -import java.util.Iterator; -import java.util.Objects; -import org.opensearch.common.unit.Fuzziness; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.index.query.MultiMatchQueryBuilder; -import org.opensearch.index.query.Operator; -import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.QueryStringQueryBuilder; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.NamedArgumentExpression; /** * Class for Lucene query that builds the query_string query. @@ -29,44 +18,9 @@ public class QueryStringQuery extends MultiFieldQuery { * named arguments. */ public QueryStringQuery() { - super(ImmutableMap.>builder() - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("allow_leading_wildcard", (b, v) -> - b.allowLeadingWildcard(Boolean.parseBoolean(v.stringValue()))) - .put("analyze_wildcard", (b, v) -> - b.analyzeWildcard(Boolean.parseBoolean(v.stringValue()))) - .put("auto_generate_synonyms_phrase_query", (b, v) -> - b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("default_operator", (b, v) -> - b.defaultOperator(Operator.fromString(v.stringValue()))) - .put("enable_position_increments", (b, v) -> - b.enablePositionIncrements(Boolean.parseBoolean(v.stringValue()))) - .put("fuzziness", (b, v) -> b.fuzziness(Fuzziness.build(v.stringValue()))) - .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(v.stringValue())) - .put("escape", (b, v) -> b.escape(Boolean.parseBoolean(v.stringValue()))) - .put("fuzzy_max_expansions", (b, v) -> - b.fuzzyMaxExpansions(Integer.parseInt(v.stringValue()))) - .put("fuzzy_prefix_length", (b, v) -> - b.fuzzyPrefixLength(Integer.parseInt(v.stringValue()))) - .put("fuzzy_transpositions", (b, v) -> - b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("lenient", (b, v) -> b.lenient(Boolean.parseBoolean(v.stringValue()))) - .put("max_determinized_states", (b, v) -> - b.maxDeterminizedStates(Integer.parseInt(v.stringValue()))) - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("quote_analyzer", (b, v) -> b.quoteAnalyzer(v.stringValue())) - .put("phrase_slop", (b, v) -> b.phraseSlop(Integer.parseInt(v.stringValue()))) - .put("quote_field_suffix", (b, v) -> b.quoteFieldSuffix(v.stringValue())) - .put("rewrite", (b, v) -> b.rewrite(v.stringValue())) - .put("type", (b, v) -> b.type(MultiMatchQueryBuilder.Type.parse(valueOfToLower(v), - LoggingDeprecationHandler.INSTANCE))) - .put("tie_breaker", (b, v) -> b.tieBreaker(Float.parseFloat(v.stringValue()))) - .put("time_zone", (b, v) -> b.timeZone(v.stringValue())) - .build()); + super(FunctionParameterRepository.QueryStringQueryBuildActions); } - /** * Builds QueryBuilder with query value and other default parameter values set. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java index 282c5478b4..579f77d2cd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java @@ -5,19 +5,16 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.BiFunction; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.opensearch.index.query.QueryBuilder; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.opensearch.storage.script.filter.lucene.LuceneQuery; @@ -31,26 +28,32 @@ public abstract class RelevanceQuery extends LuceneQuery @Override public QueryBuilder build(FunctionExpression func) { - List arguments = func.getArguments(); + var arguments = func.getArguments().stream() + .map(a -> (NamedArgumentExpression)a).collect(Collectors.toList()); if (arguments.size() < 2) { throw new SyntaxCheckException( String.format("%s requires at least two parameters", getQueryName())); } - NamedArgumentExpression field = (NamedArgumentExpression) arguments.get(0); - NamedArgumentExpression query = (NamedArgumentExpression) arguments.get(1); - T queryBuilder = createQueryBuilder(field, query); - Iterator iterator = arguments.listIterator(2); - Set visitedParms = new HashSet(); + // Aggregate parameters by name, so getting a Map + arguments.stream().collect(Collectors.groupingBy(a -> a.getArgName().toLowerCase())) + .forEach((k, v) -> { + if (v.size() > 1) { + throw new SemanticCheckException( + String.format("Parameter '%s' can only be specified once.", k)); + } + }); + + T queryBuilder = createQueryBuilder(arguments); + + arguments.removeIf(a -> a.getArgName().equalsIgnoreCase("field") + || a.getArgName().equalsIgnoreCase("fields") + || a.getArgName().equalsIgnoreCase("query")); + + var iterator = arguments.listIterator(); while (iterator.hasNext()) { - NamedArgumentExpression arg = (NamedArgumentExpression) iterator.next(); + NamedArgumentExpression arg = iterator.next(); String argNormalized = arg.getArgName().toLowerCase(); - if (visitedParms.contains(argNormalized)) { - throw new SemanticCheckException(String.format("Parameter '%s' can only be specified once.", - argNormalized)); - } else { - visitedParms.add(argNormalized); - } if (!queryBuildActions.containsKey(argNormalized)) { throw new SemanticCheckException( @@ -65,8 +68,7 @@ public QueryBuilder build(FunctionExpression func) { return queryBuilder; } - protected abstract T createQueryBuilder(NamedArgumentExpression field, - NamedArgumentExpression query); + protected abstract T createQueryBuilder(List arguments); protected abstract String getQueryName(); @@ -79,12 +81,4 @@ protected abstract T createQueryBuilder(NamedArgumentExpression field, protected interface QueryBuilderStep extends BiFunction { } - - public static String valueOfToUpper(ExprValue v) { - return v.stringValue().toUpperCase(); - } - - public static String valueOfToLower(ExprValue v) { - return v.stringValue().toLowerCase(); - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java index 1b7c18cb2c..157921572a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java @@ -6,13 +6,8 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; import com.google.common.collect.ImmutableMap; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Objects; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.SimpleQueryStringBuilder; -import org.opensearch.index.query.SimpleQueryStringFlag; public class SimpleQueryStringQuery extends MultiFieldQuery { /** @@ -20,26 +15,7 @@ public class SimpleQueryStringQuery extends MultiFieldQuery>builder() - .put("analyze_wildcard", (b, v) -> b.analyzeWildcard(Boolean.parseBoolean(v.stringValue()))) - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("auto_generate_synonyms_phrase_query", (b, v) -> - b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("default_operator", (b, v) -> b.defaultOperator(Operator.fromString(v.stringValue()))) - .put("flags", (b, v) -> b.flags(Arrays.stream(valueOfToUpper(v).split("\\|")) - .map(SimpleQueryStringFlag::valueOf) - .toArray(SimpleQueryStringFlag[]::new))) - .put("fuzzy_max_expansions", (b, v) -> - b.fuzzyMaxExpansions(Integer.parseInt(v.stringValue()))) - .put("fuzzy_prefix_length", (b, v) -> - b.fuzzyPrefixLength(Integer.parseInt(v.stringValue()))) - .put("fuzzy_transpositions", (b, v) -> - b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("lenient", (b, v) -> b.lenient(Boolean.parseBoolean(v.stringValue()))) - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("quote_field_suffix", (b, v) -> b.quoteFieldSuffix(v.stringValue())) - .build()); + super(FunctionParameterRepository.SimpleQueryStringQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java index 9876c62cce..15eda7f483 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java @@ -5,8 +5,10 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; +import java.util.List; import java.util.Map; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.NamedArgumentExpression; /** @@ -21,9 +23,20 @@ public SingleFieldQuery(Map> queryBuildActions) { } @Override - protected T createQueryBuilder(NamedArgumentExpression fields, NamedArgumentExpression query) { + protected T createQueryBuilder(List arguments) { + // Extract 'field' and 'query' + var field = arguments.stream() + .filter(a -> a.getArgName().equalsIgnoreCase("field")) + .findFirst() + .orElseThrow(() -> new SemanticCheckException("'field' parameter is missing.")); + + var query = arguments.stream() + .filter(a -> a.getArgName().equalsIgnoreCase("query")) + .findFirst() + .orElseThrow(() -> new SemanticCheckException("'query' parameter is missing")); + return createBuilder( - fields.getValue().valueOf(null).stringValue(), + field.getValue().valueOf(null).stringValue(), query.getValue().valueOf(null).stringValue()); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 75ddd1dd93..ff80f3bcc0 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -332,7 +332,7 @@ void should_build_match_query_with_custom_parameters() { + " \"prefix_length\" : 0,\n" + " \"max_expansions\" : 50,\n" + " \"minimum_should_match\" : \"3\"," - + " \"fuzzy_rewrite\" : \"top_terms_N\"," + + " \"fuzzy_rewrite\" : \"top_terms_1\"," + " \"fuzzy_transpositions\" : false,\n" + " \"lenient\" : false,\n" + " \"zero_terms_query\" : \"ALL\",\n" @@ -352,7 +352,7 @@ void should_build_match_query_with_custom_parameters() { dsl.namedArgument("max_expansions", literal("50")), dsl.namedArgument("prefix_length", literal("0")), dsl.namedArgument("fuzzy_transpositions", literal("false")), - dsl.namedArgument("fuzzy_rewrite", literal("top_terms_N")), + dsl.namedArgument("fuzzy_rewrite", literal("top_terms_1")), dsl.namedArgument("lenient", literal("false")), dsl.namedArgument("minimum_should_match", literal("3")), dsl.namedArgument("zero_terms_query", literal("ALL")), @@ -366,7 +366,49 @@ void match_invalid_parameter() { dsl.namedArgument("query", literal("search query")), dsl.namedArgument("invalid_parameter", literal("invalid_value"))); var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); - assertEquals("Parameter invalid_parameter is invalid for match function.", msg); + assertTrue(msg.startsWith("Parameter invalid_parameter is invalid for match function.")); + } + + @Test + void match_disallow_duplicate_parameter() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("field", literal("message")), + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword")), + dsl.namedArgument("AnalYzer", literal("english"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("Parameter 'analyzer' can only be specified once.", msg); + } + + @Test + void match_disallow_duplicate_query() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("field", literal("message")), + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword")), + dsl.namedArgument("QUERY", literal("something"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("Parameter 'query' can only be specified once.", msg); + } + + @Test + void match_disallow_duplicate_field() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("field", literal("message")), + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword")), + dsl.namedArgument("Field", literal("something"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("Parameter 'field' can only be specified once.", msg); + } + + @Test + void match_missing_field() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("'field' parameter is missing.", msg); } @Test @@ -570,12 +612,13 @@ void should_build_match_phrase_query_with_custom_parameters() { + " \"analyzer\" : \"keyword\"," + " \"slop\" : 2,\n" + " \"zero_terms_query\" : \"ALL\",\n" - + " \"boost\" : 1.0\n" + + " \"boost\" : 1.2\n" + " }\n" + " }\n" + "}", buildQuery( dsl.match_phrase( + dsl.namedArgument("boost", literal("1.2")), dsl.namedArgument("field", literal("message")), dsl.namedArgument("query", literal("search query")), dsl.namedArgument("analyzer", literal("keyword")), @@ -831,32 +874,72 @@ void match_phrase_invalid_parameter() { dsl.namedArgument("query", literal("search query")), dsl.namedArgument("invalid_parameter", literal("invalid_value"))); var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); - assertEquals("Parameter invalid_parameter is invalid for match_phrase function.", msg); + assertTrue(msg.startsWith("Parameter invalid_parameter is invalid for match_phrase function.")); } @Test - void match_phrase_invalid_value_slop() { - FunctionExpression expr = dsl.match_phrase( - dsl.namedArgument("field", literal("message")), - dsl.namedArgument("query", literal("search query")), + void relevancy_func_invalid_arg_values() { + final var field = dsl.namedArgument("field", literal("message")); + final var fields = dsl.namedArgument("fields", DSL.literal( + new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "field1", ExprValueUtils.floatValue(1.F), + "field2", ExprValueUtils.floatValue(.3F)))))); + final var query = dsl.namedArgument("query", literal("search query")); + + var slopTest = dsl.match_phrase(field, query, dsl.namedArgument("slop", literal("1.5"))); - var msg = assertThrows(NumberFormatException.class, () -> buildQuery(expr)).getMessage(); - assertEquals("For input string: \"1.5\"", msg); - } + var msg = assertThrows(RuntimeException.class, () -> buildQuery(slopTest)).getMessage(); + assertEquals("Invalid slop value: '1.5'. Accepts only integer values.", msg); - @Test - void match_phrase_invalid_value_ztq() { - FunctionExpression expr = dsl.match_phrase( - dsl.namedArgument("field", literal("message")), - dsl.namedArgument("query", literal("search query")), + var ztqTest = dsl.match_phrase(field, query, dsl.namedArgument("zero_terms_query", literal("meow"))); - var msg = assertThrows(IllegalArgumentException.class, () -> buildQuery(expr)).getMessage(); - assertEquals("No enum constant org.opensearch.index.search.MatchQuery.ZeroTermsQuery.MEOW", - msg); + msg = assertThrows(RuntimeException.class, () -> buildQuery(ztqTest)).getMessage(); + assertEquals( + "Invalid zero_terms_query value: 'meow'. Available values are: NONE, ALL, NULL.", msg); + + var boostTest = dsl.match(field, query, + dsl.namedArgument("boost", literal("pewpew"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(boostTest)).getMessage(); + assertEquals( + "Invalid boost value: 'pewpew'. Accepts only floating point values greater than 0.", msg); + + var boolTest = dsl.query_string(fields, query, + dsl.namedArgument("escape", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(boolTest)).getMessage(); + assertEquals( + "Invalid escape value: '42'. Accepts only boolean values: 'true' or 'false'.", msg); + + var typeTest = dsl.multi_match(fields, query, + dsl.namedArgument("type", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(typeTest)).getMessage(); + assertTrue(msg.startsWith("Invalid type value: '42'. Available values are:")); + + var operatorTest = dsl.simple_query_string(fields, query, + dsl.namedArgument("default_operator", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(operatorTest)).getMessage(); + assertTrue(msg.startsWith("Invalid default_operator value: '42'. Available values are:")); + + var flagsTest = dsl.simple_query_string(fields, query, + dsl.namedArgument("flags", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(flagsTest)).getMessage(); + assertTrue(msg.startsWith("Invalid flags value: '42'. Available values are:")); + + var fuzzinessTest = dsl.match_bool_prefix(field, query, + dsl.namedArgument("fuzziness", literal("AUTO:"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(fuzzinessTest)).getMessage(); + assertTrue(msg.startsWith("Invalid fuzziness value: 'AUTO:'. Available values are:")); + + var rewriteTest = dsl.match_bool_prefix(field, query, + dsl.namedArgument("fuzzy_rewrite", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(rewriteTest)).getMessage(); + assertTrue(msg.startsWith("Invalid fuzzy_rewrite value: '42'. Available values are:")); + + var timezoneTest = dsl.query_string(fields, query, + dsl.namedArgument("time_zone", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(timezoneTest)).getMessage(); + assertTrue(msg.startsWith("Invalid time_zone value: '42'.")); } - - @Test void should_build_match_bool_prefix_query_with_default_parameters() { assertJsonEquals( @@ -878,6 +961,26 @@ void should_build_match_bool_prefix_query_with_default_parameters() { dsl.namedArgument("query", literal("search query"))))); } + @Test + void multi_match_missing_fields() { + var msg = assertThrows(SemanticCheckException.class, () -> + dsl.multi_match( + dsl.namedArgument("query", literal("search query")))).getMessage(); + assertEquals("Expected type STRUCT instead of STRING for parameter #1", msg); + } + + @Test + void multi_match_missing_fields_even_with_struct() { + FunctionExpression expr = dsl.multi_match( + dsl.namedArgument("something-but-not-fields", DSL.literal( + new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "pewpew", ExprValueUtils.integerValue(42)))))), + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("'fields' parameter is missing.", msg); + } + @Test void should_build_match_phrase_prefix_query_with_default_parameters() { assertJsonEquals( @@ -899,7 +1002,7 @@ void should_build_match_phrase_prefix_query_with_default_parameters() { } @Test - void should_build_match_phrase_prefix_query_with_analyzer() { + void should_build_match_phrase_prefix_query_with_non_default_parameters() { assertJsonEquals( "{\n" + " \"match_phrase_prefix\" : {\n" @@ -907,8 +1010,8 @@ void should_build_match_phrase_prefix_query_with_analyzer() { + " \"query\" : \"search query\",\n" + " \"slop\" : 0,\n" + " \"zero_terms_query\" : \"NONE\",\n" - + " \"max_expansions\" : 50,\n" - + " \"boost\" : 1.0,\n" + + " \"max_expansions\" : 42,\n" + + " \"boost\" : 1.2,\n" + " \"analyzer\": english\n" + " }\n" + " }\n" @@ -917,6 +1020,8 @@ void should_build_match_phrase_prefix_query_with_analyzer() { dsl.match_phrase_prefix( dsl.namedArgument("field", literal("message")), dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("boost", literal("1.2")), + dsl.namedArgument("max_expansions", literal("42")), dsl.namedArgument("analyzer", literal("english"))))); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java index 261870ca17..748384f4c8 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java @@ -80,7 +80,7 @@ static Stream> generateValidData() { List.of( dsl.namedArgument("fields", fields_value), dsl.namedArgument("query", query_value), - dsl.namedArgument("fuzzy_transpositions", DSL.literal("42")) + dsl.namedArgument("fuzzy_transpositions", DSL.literal("true")) ), List.of( dsl.namedArgument("fields", fields_value), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java index 21b03abab0..4692f046db 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java @@ -46,7 +46,7 @@ class QueryStringTest { private static final LiteralExpression query_value = DSL.literal("query_value"); static Stream> generateValidData() { - Expression field = dsl.namedArgument("field", fields_value); + Expression field = dsl.namedArgument("fields", fields_value); Expression query = dsl.namedArgument("query", query_value); return List.of( dsl.namedArgument("analyzer", DSL.literal("standard")), @@ -62,7 +62,7 @@ static Stream> generateValidData() { dsl.namedArgument("fuzzy_rewrite", DSL.literal("constant_score")), dsl.namedArgument("fuzzy_max_expansions", DSL.literal("42")), dsl.namedArgument("fuzzy_prefix_length", DSL.literal("42")), - dsl.namedArgument("fuzzy_transpositions", DSL.literal("42")), + dsl.namedArgument("fuzzy_transpositions", DSL.literal("true")), dsl.namedArgument("lenient", DSL.literal("true")), dsl.namedArgument("max_determinized_states", DSL.literal("10000")), dsl.namedArgument("minimum_should_match", DSL.literal("4")), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java index 8f06f48727..de8576e9d4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java @@ -105,7 +105,7 @@ static Stream> generateValidData() { List.of( dsl.namedArgument("fields", fields_value), dsl.namedArgument("query", query_value), - dsl.namedArgument("fuzzy_transpositions", DSL.literal("42")) + dsl.namedArgument("fuzzy_transpositions", DSL.literal("true")) ), List.of( dsl.namedArgument("fields", fields_value), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQueryTest.java index 7e4c6ea011..c50f2efb0d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQueryTest.java @@ -12,6 +12,7 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,10 +49,10 @@ void createQueryBuilderTest() { var fieldSpec = ImmutableMap.builder().put(sampleField, ExprValueUtils.floatValue(sampleValue)).build(); - query.createQueryBuilder(dsl.namedArgument("fields", + query.createQueryBuilder(List.of(dsl.namedArgument("fields", new LiteralExpression(ExprTupleValue.fromExprValueMap(fieldSpec))), dsl.namedArgument("query", - new LiteralExpression(ExprValueUtils.stringValue(sampleQuery)))); + new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); verify(query).createBuilder(argThat( (ArgumentMatcher>) map -> map.size() == 1 diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java index fa6a43474a..5406f4cb58 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableMap; import java.util.List; +import java.util.Map; import java.util.stream.Stream; import org.apache.commons.lang3.NotImplementedException; import org.junit.jupiter.api.BeforeEach; @@ -45,15 +46,16 @@ class RelevanceQueryBuildTest { public static final NamedArgumentExpression QUERY_ARG = namedArgument("query", "find me"); private RelevanceQuery query; private QueryBuilder queryBuilder; + private final Map> queryBuildActions = + ImmutableMap.>builder() + .put("boost", (k, v) -> k.boost(Float.parseFloat(v.stringValue()))).build(); @BeforeEach public void setUp() { - query = mock(RelevanceQuery.class, withSettings().useConstructor( - ImmutableMap.>builder() - .put("boost", (k, v) -> k.boost(Float.parseFloat(v.stringValue()))).build()) + query = mock(RelevanceQuery.class, withSettings().useConstructor(queryBuildActions) .defaultAnswer(Mockito.CALLS_REAL_METHODS)); queryBuilder = mock(QueryBuilder.class); - when(query.createQueryBuilder(any(), any())).thenReturn(queryBuilder); + when(query.createQueryBuilder(any())).thenReturn(queryBuilder); String queryName = "mock_query"; when(queryBuilder.queryName()).thenReturn(queryName); when(queryBuilder.getWriteableName()).thenReturn(queryName); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java index 5d35327116..d6f178b1d6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,10 +41,10 @@ void createQueryBuilderTest() { String sampleQuery = "sample query"; String sampleField = "fieldA"; - query.createQueryBuilder(dsl.namedArgument("field", + query.createQueryBuilder(List.of(dsl.namedArgument("field", new LiteralExpression(ExprValueUtils.stringValue(sampleField))), dsl.namedArgument("query", - new LiteralExpression(ExprValueUtils.stringValue(sampleQuery)))); + new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); verify(query).createBuilder(eq(sampleField), eq(sampleQuery));