Skip to content

Commit

Permalink
Added IEQUAL operator to support case insensitive searches
Browse files Browse the repository at this point in the history
  • Loading branch information
@Nbagga14
Nbagga14 committed Sep 30, 2024
1 parent 0187fc6 commit 4e434e8
Show file tree
Hide file tree
Showing 8 changed files with 42 additions and 9 deletions.
5 changes: 5 additions & 0 deletions datahub-graphql-core/src/main/resources/search.graphql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -575,6 +575,11 @@ enum FilterOperator {
Represent the relation: URN field matches any nested child or parent in addition to the given URN
"""
RELATED_INCL

"""
Represent the relation: field = value (case-insensitive), e.g. platform = HDFS
"""
IEQUAL
}

"""
Expand Down
1 change: 1 addition & 0 deletions docs/api/restli/restli-overview.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1203,6 +1203,7 @@ where valid conditions include
- CONTAIN
- END_WITH
- EQUAL
- IEQUAL (Supports case insensitive equals)
- GREATER_THAN
- GREATER_THAN_OR_EQUAL_TO
- LESS_THAN
Expand Down
30 changes: 24 additions & 6 deletions metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,9 @@ private static QueryBuilder getQueryBuilderFromCriterionForFieldToExpand(
}
return orQueryBuilder;
}
private static boolean isCaseInsensitiveSearchEnabled(Condition condition) {
return condition == Condition.IEQUAL;
}

@Nonnull
private static QueryBuilder getQueryBuilderFromCriterionForSingleField(
Expand All @@ -573,9 +576,10 @@ private static QueryBuilder getQueryBuilderFromCriterionForSingleField(
.must(QueryBuilders.existsQuery(fieldName))
.queryName(queryName != null ? queryName : fieldName);
} else if (criterion.hasValues()) {
if (condition == Condition.EQUAL) {
if (condition == Condition.EQUAL || condition ==Condition.IEQUAL) {
boolean enableCaseInsensitiveSearches = isCaseInsensitiveSearchEnabled(condition);
return buildEqualsConditionFromCriterion(
fieldName, criterion, isTimeseries, searchableFieldTypes, aspectRetriever)
fieldName, criterion, isTimeseries, searchableFieldTypes, aspectRetriever, enableCaseInsensitiveSearches)
.queryName(queryName != null ? queryName : fieldName);
} else if (RANGE_QUERY_CONDITIONS.contains(condition)) {
return buildRangeQueryFromCriterion(
Expand Down Expand Up @@ -605,7 +609,7 @@ private static QueryBuilder getQueryBuilderFromCriterionForSingleField(
.rewrite(
opContext,
buildEqualsConditionFromCriterion(
fieldName, criterion, isTimeseries, searchableFieldTypes, aspectRetriever))
fieldName, criterion, isTimeseries, searchableFieldTypes, aspectRetriever,false))
.queryName(queryName != null ? queryName : fieldName);
}
}
Expand Down Expand Up @@ -670,9 +674,9 @@ private static QueryBuilder buildEqualsConditionFromCriterion(
@Nonnull final Criterion criterion,
final boolean isTimeseries,
final Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypes,
@Nonnull AspectRetriever aspectRetriever) {
@Nonnull AspectRetriever aspectRetriever,boolean enableCaseInsensitiveSearches ) {
return buildEqualsConditionFromCriterionWithValues(
fieldName, criterion, isTimeseries, searchableFieldTypes, aspectRetriever);
fieldName, criterion, isTimeseries, searchableFieldTypes, aspectRetriever, enableCaseInsensitiveSearches);
}

/**
Expand All @@ -684,7 +688,7 @@ private static QueryBuilder buildEqualsConditionFromCriterionWithValues(
@Nonnull final Criterion criterion,
final boolean isTimeseries,
final Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypes,
@Nonnull AspectRetriever aspectRetriever) {
@Nonnull AspectRetriever aspectRetriever,boolean enableCaseInsensitiveSearches) {
Set<String> fieldTypes = getFieldTypes(searchableFieldTypes, fieldName, aspectRetriever);
if (fieldTypes.size() > 1) {
log.warn(
Expand All @@ -704,6 +708,20 @@ private static QueryBuilder buildEqualsConditionFromCriterionWithValues(
criterion.getValues().stream().map(Double::parseDouble).collect(Collectors.toList());
return QueryBuilders.termsQuery(fieldName, doubleValues).queryName(fieldName);
}

if (enableCaseInsensitiveSearches) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
criterion.getValues().forEach(value ->
boolQuery.should(
QueryBuilders.termQuery(
toKeywordField(criterion.getField(), isTimeseries, aspectRetriever),
value.trim()
).caseInsensitive(true)
)
);
return boolQuery;
}

return QueryBuilders.termsQuery(
toKeywordField(criterion.getField(), isTimeseries, aspectRetriever),
criterion.getValues())
Expand Down
5 changes: 5 additions & 0 deletions metadata-models/src/main/pegasus/com/linkedin/metadata/query/filter/Condition.pdl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ enum Condition {
*/
EQUAL

/**
* Represent the relation: field = value, e.g. platform = hdfs
*/
IEQUAL

/**
* Represent the relation: field is null, e.g. platform is null
*/
Expand Down
1 change: 1 addition & 0 deletions metadata-service/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1291,6 +1291,7 @@ where valid conditions include
- CONTAIN
- END_WITH
- EQUAL
- IEQUAL (Supports case insensitive equals)
- GREATER_THAN
- GREATER_THAN_OR_EQUAL_TO
- LESS_THAN
Expand Down
3 changes: 2 additions & 1 deletion metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,14 @@
"type" : "enum",
"name" : "Condition",
"doc" : "The matching condition in a filter criterion",
"symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "IS_NULL", "EXISTS", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH", "DESCENDANTS_INCL", "ANCESTORS_INCL", "RELATED_INCL" ],
"symbols" : [ "CONTAIN", "END_WITH", "EQUAL","IEQUAL", "IS_NULL", "EXISTS", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH", "DESCENDANTS_INCL", "ANCESTORS_INCL", "RELATED_INCL" ],
"symbolDocs" : {
"ANCESTORS_INCL" : "Represent the relation: URN field matches any nested parent in addition to the given URN",
"CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile",
"DESCENDANTS_INCL" : "Represent the relation: URN field any nested children in addition to the given URN",
"END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event",
"EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
"IEQUAL" : "Represent the relation: case-insensitive field = value, e.g. platform = hdfs",
"EXISTS" : "Represents the relation: field exists and is non-empty, e.g. owners is not null and != [] (empty)",
"GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5",
"GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
Expand Down
3 changes: 2 additions & 1 deletion metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,13 +162,14 @@
"type" : "enum",
"name" : "Condition",
"doc" : "The matching condition in a filter criterion",
"symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "IS_NULL", "EXISTS", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH", "DESCENDANTS_INCL", "ANCESTORS_INCL", "RELATED_INCL" ],
"symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "IEQUAL","IS_NULL", "EXISTS", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH", "DESCENDANTS_INCL", "ANCESTORS_INCL", "RELATED_INCL" ],
"symbolDocs" : {
"ANCESTORS_INCL" : "Represent the relation: URN field matches any nested parent in addition to the given URN",
"CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile",
"DESCENDANTS_INCL" : "Represent the relation: URN field any nested children in addition to the given URN",
"END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event",
"EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
"IEQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
"EXISTS" : "Represents the relation: field exists and is non-empty, e.g. owners is not null and != [] (empty)",
"GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5",
"GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
Expand Down
3 changes: 2 additions & 1 deletion metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6057,13 +6057,14 @@
"name" : "Condition",
"namespace" : "com.linkedin.metadata.query.filter",
"doc" : "The matching condition in a filter criterion",
"symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "IS_NULL", "EXISTS", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH", "DESCENDANTS_INCL", "ANCESTORS_INCL", "RELATED_INCL" ],
"symbols" : [ "CONTAIN", "END_WITH", "EQUAL","IEQUAL", "IS_NULL", "EXISTS", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH", "DESCENDANTS_INCL", "ANCESTORS_INCL", "RELATED_INCL" ],
"symbolDocs" : {
"ANCESTORS_INCL" : "Represent the relation: URN field matches any nested parent in addition to the given URN",
"CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile",
"DESCENDANTS_INCL" : "Represent the relation: URN field any nested children in addition to the given URN",
"END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event",
"EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
"IEQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
"EXISTS" : "Represents the relation: field exists and is non-empty, e.g. owners is not null and != [] (empty)",
"GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5",
"GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
Expand Down

0 comments on commit 4e434e8

Please sign in to comment.