From b6a3d3d1cf77a4f22370996001594c92bccd6816 Mon Sep 17 00:00:00 2001 From: mwan-ep <54115904+mwan-ep@users.noreply.github.com> Date: Fri, 16 Jun 2023 12:34:00 -0700 Subject: [PATCH] Unary operator is_null (#16) * Unary operator is_null * Added is_null to mongo query builder * Fix function that broke after rebasing * Update external/epsearchast/v3/mongo/mongo_query_builder.go Co-authored-by: Steve Ramage <49958178+steve-r-west@users.noreply.github.com> * Update external/epsearchast/v3/validate_test.go Co-authored-by: Steve Ramage <49958178+steve-r-west@users.noreply.github.com> * Fixed linting issues --------- Co-authored-by: Steve Ramage <49958178+steve-r-west@users.noreply.github.com> --- external/epsearchast/v3/aliases_test.go | 12 +- external/epsearchast/v3/ast.go | 12 ++ external/epsearchast/v3/ast_visitor_test.go | 5 + .../epsearchast/v3/gorm/gorm_query_builder.go | 6 + .../v3/gorm/gorm_query_builder_test.go | 33 +++++ .../v3/mongo/mongo_query_builder.go | 7 ++ .../v3/mongo/mongo_query_builder_test.go | 36 ++++++ external/epsearchast/v3/semantic_reduce.go | 3 + external/epsearchast/v3/validate_test.go | 114 ++++++++++++++++++ external/epsearchast/v3/validating_visitor.go | 10 ++ 10 files changed, 236 insertions(+), 2 deletions(-) diff --git a/external/epsearchast/v3/aliases_test.go b/external/epsearchast/v3/aliases_test.go index 4255e37..3daad4a 100644 --- a/external/epsearchast/v3/aliases_test.go +++ b/external/epsearchast/v3/aliases_test.go @@ -119,6 +119,10 @@ func TestApplyAliasesReturnsCorrectAstWhenAliasTwoFieldsAreAliasedInAnAnd(t *tes { "type": "EQ", "args": [ "customer.email", "ron@swanson.com"] + }, + { + "type": "IS_NULL", + "args": [ "billing-email"] } ] } @@ -140,7 +144,11 @@ func TestApplyAliasesReturnsCorrectAstWhenAliasTwoFieldsAreAliasedInAnAnd(t *tes { "type": "EQ", "args": [ "customer.email", "ron@swanson.com"] - } + }, + { + "type": "IS_NULL", + "args": [ "billing.email"] + } ] }` @@ -151,7 +159,7 @@ func TestApplyAliasesReturnsCorrectAstWhenAliasTwoFieldsAreAliasedInAnAnd(t *tes require.NoError(t, err) // Execute SUT - aliasedAst, err := ApplyAliases(inputAstNode, map[string]string{"payment_status": "status", "customer_name": "customer.name"}) + aliasedAst, err := ApplyAliases(inputAstNode, map[string]string{"payment_status": "status", "customer_name": "customer.name", "billing-email": "billing.email"}) // Verify require.NoError(t, err) diff --git a/external/epsearchast/v3/ast.go b/external/epsearchast/v3/ast.go index f09d736..26ccae3 100644 --- a/external/epsearchast/v3/ast.go +++ b/external/epsearchast/v3/ast.go @@ -47,6 +47,7 @@ type AstVisitor interface { VisitGe(astNode *AstNode) (bool, error) VisitGt(astNode *AstNode) (bool, error) VisitLike(astNode *AstNode) (bool, error) + VisitIsNull(astNode *AstNode) (bool, error) } // Accept triggers a visit of the AST. @@ -88,6 +89,8 @@ func (a *AstNode) accept(v AstVisitor) error { descend, err = v.VisitGe(a) case "LIKE": descend, err = v.VisitLike(a) + case "IS_NULL": + descend, err = v.VisitIsNull(a) default: return fmt.Errorf("unknown operator %s", a.NodeType) } @@ -145,6 +148,15 @@ func (a *AstNode) checkValid() error { if len(a.Args) != 2 { return fmt.Errorf("operator %v should have exactly 2 arguments", strings.ToLower(a.NodeType)) + } + case "IS_NULL": + if len(a.Children) > 0 { + return fmt.Errorf("operator %v should not have any children", strings.ToLower(a.NodeType)) + } + + if len(a.Args) != 1 { + return fmt.Errorf("operator %v should have exactly 1 argument", strings.ToLower(a.NodeType)) + } default: return fmt.Errorf("unknown operator %s", a.NodeType) diff --git a/external/epsearchast/v3/ast_visitor_test.go b/external/epsearchast/v3/ast_visitor_test.go index 2dd677e..2fd4c08 100644 --- a/external/epsearchast/v3/ast_visitor_test.go +++ b/external/epsearchast/v3/ast_visitor_test.go @@ -606,4 +606,9 @@ func (m *MyMockedVisitor) VisitLike(astNode *AstNode) (bool, error) { return args.Bool(0), args.Error(1) } +func (m *MyMockedVisitor) VisitIsNull(astNode *AstNode) (bool, error) { + args := m.Called(astNode) + return args.Bool(0), args.Error(1) +} + var _ AstVisitor = (*MyMockedVisitor)(nil) diff --git a/external/epsearchast/v3/gorm/gorm_query_builder.go b/external/epsearchast/v3/gorm/gorm_query_builder.go index db191a8..a158a50 100644 --- a/external/epsearchast/v3/gorm/gorm_query_builder.go +++ b/external/epsearchast/v3/gorm/gorm_query_builder.go @@ -85,6 +85,12 @@ func (g DefaultGormQueryBuilder) VisitLike(first, second string) (*SubQuery, err }, nil } +func (g DefaultGormQueryBuilder) VisitIsNull(first string) (*SubQuery, error) { + return &SubQuery{ + Clause: fmt.Sprintf("%s IS NULL", first), + }, nil +} + func (g DefaultGormQueryBuilder) ProcessLikeWildcards(valString string) string { if valString == "*" { return "%" diff --git a/external/epsearchast/v3/gorm/gorm_query_builder_test.go b/external/epsearchast/v3/gorm/gorm_query_builder_test.go index ede6b45..5479424 100644 --- a/external/epsearchast/v3/gorm/gorm_query_builder_test.go +++ b/external/epsearchast/v3/gorm/gorm_query_builder_test.go @@ -17,6 +17,10 @@ var binOps = []testOp{ {"LIKE", "LIKE"}, } +var unaryOps = []testOp{ + {"IS_NULL", "IS NULL"}, +} + var varOps = []testOp{ {"IN", "IN"}, } @@ -56,6 +60,35 @@ func TestSimpleBinaryOperatorFiltersGeneratesCorrectWhereClause(t *testing.T) { } +func TestSimpleUnaryOperatorFiltersGeneratesCorrectWhereClause(t *testing.T) { + for _, unaryOp := range unaryOps { + t.Run(fmt.Sprintf("%s", unaryOp.AstOp), func(t *testing.T) { + //Fixture Setup + //language=JSON + jsonTxt := fmt.Sprintf(` + { + "type": "%s", + "args": [ "amount"] + }`, unaryOp.AstOp) + + astNode, err := epsearchast_v3.GetAst(jsonTxt) + require.NoError(t, err) + + var sr epsearchast_v3.SemanticReducer[SubQuery] = DefaultGormQueryBuilder{} + + // Execute SUT + query, err := epsearchast_v3.SemanticReduceAst(astNode, sr) + + // Verification + + require.NoError(t, err) + + require.Equal(t, fmt.Sprintf("amount %s", unaryOp.SqlOp), query.Clause) + }) + } + +} + func TestSimpleVariableOperatorFiltersGeneratesCorrectWhereClause(t *testing.T) { for _, varOp := range varOps { t.Run(fmt.Sprintf("%s", varOp.AstOp), func(t *testing.T) { diff --git a/external/epsearchast/v3/mongo/mongo_query_builder.go b/external/epsearchast/v3/mongo/mongo_query_builder.go index 9263ef3..d949566 100644 --- a/external/epsearchast/v3/mongo/mongo_query_builder.go +++ b/external/epsearchast/v3/mongo/mongo_query_builder.go @@ -55,6 +55,13 @@ func (d DefaultMongoQueryBuilder) VisitLike(first, second string) (*bson.D, erro return &bson.D{{first, bson.D{{"$regex", d.ProcessLikeWildcards(second)}}}}, nil } +func (d DefaultMongoQueryBuilder) VisitIsNull(first string) (*bson.D, error) { + // https://www.mongodb.com/docs/manual/tutorial/query-for-null-fields/#equality-filter + // This will match fields that either contain the item field whose value is nil or those that do not contain the field + // Customize this method if you need different nil handling (i.e., explicit nil) + return &bson.D{{first, bson.D{{"$eq", nil}}}}, nil +} + func (d DefaultMongoQueryBuilder) ProcessLikeWildcards(valString string) string { if valString == "*" { return "^.*$" diff --git a/external/epsearchast/v3/mongo/mongo_query_builder_test.go b/external/epsearchast/v3/mongo/mongo_query_builder_test.go index fd6d9da..bd07fa6 100644 --- a/external/epsearchast/v3/mongo/mongo_query_builder_test.go +++ b/external/epsearchast/v3/mongo/mongo_query_builder_test.go @@ -21,6 +21,10 @@ var binOps = []testOp{ //{"LIKE", "$regex"}, } +var unaryOps = []testOp{ + {"IS_NULL", `"$eq":null`}, +} + var varOps = []testOp{ {"IN", "$in"}, } @@ -95,6 +99,38 @@ func TestLikeOperatorFiltersGeneratesCorrectFilter(t *testing.T) { } +func TestSimpleUnaryOperatorFiltersGeneratesCorrectFilter(t *testing.T) { + for _, unaryOp := range unaryOps { + t.Run(fmt.Sprintf("%s", unaryOp.AstOp), func(t *testing.T) { + //Fixture Setup + //language=JSON + astJson := fmt.Sprintf(` + { + "type": "%s", + "args": [ "amount"] + }`, unaryOp.AstOp) + + astNode, err := epsearchast_v3.GetAst(astJson) + + var qb epsearchast_v3.SemanticReducer[bson.D] = DefaultMongoQueryBuilder{} + + expectedSearchJson := fmt.Sprintf(`{"amount":{%s}}`, unaryOp.MongoOp) + + // Execute SUT + queryObj, err := epsearchast_v3.SemanticReduceAst(astNode, qb) + + // Verification + + require.NoError(t, err) + + doc, err := bson.MarshalExtJSON(queryObj, true, false) + require.NoError(t, err) + + require.Equal(t, expectedSearchJson, string(doc)) + }) + } +} + func TestSimpleVariableOperatorFiltersGeneratesCorrectFilter(t *testing.T) { for _, varOp := range varOps { t.Run(fmt.Sprintf("%s", varOp.AstOp), func(t *testing.T) { diff --git a/external/epsearchast/v3/semantic_reduce.go b/external/epsearchast/v3/semantic_reduce.go index 9c1f561..ea90139 100644 --- a/external/epsearchast/v3/semantic_reduce.go +++ b/external/epsearchast/v3/semantic_reduce.go @@ -15,6 +15,7 @@ type SemanticReducer[R any] interface { VisitGe(first, second string) (*R, error) VisitGt(first, second string) (*R, error) VisitLike(first, second string) (*R, error) + VisitIsNull(first string) (*R, error) } // SemanticReduceAst adapts an epsearchast_v3.SemanticReducer for use with the epsearchast_v3.ReduceAst function. @@ -37,6 +38,8 @@ func SemanticReduceAst[T any](a *AstNode, v SemanticReducer[T]) (*T, error) { return v.VisitIn(a.Args...) case "AND": return v.PostVisitAnd(t) + case "IS_NULL": + return v.VisitIsNull(a.Args[0]) default: return nil, fmt.Errorf("unsupported node type: %s", a.NodeType) } diff --git a/external/epsearchast/v3/validate_test.go b/external/epsearchast/v3/validate_test.go index 00ca891..5b6bab7 100644 --- a/external/epsearchast/v3/validate_test.go +++ b/external/epsearchast/v3/validate_test.go @@ -9,6 +9,8 @@ import ( var binOps = []string{"le", "lt", "eq", "ge", "gt", "like"} +var unaryOps = []string{"is_null"} + var varOps = []string{"in"} func TestValidationReturnsErrorForBinaryOperatorsWhenAstUsesInvalidOperatorForKnownField(t *testing.T) { @@ -165,6 +167,118 @@ func TestValidationReturnsNoErrorForBinaryOperatorsWhenAstUseAliasesAndValueVali } } +func TestValidationReturnsNoErrorForUnaryOperatorWhenAstSatisfiesConstraints(t *testing.T) { + + for _, unaryOp := range unaryOps { + t.Run(fmt.Sprintf("%s", unaryOp), func(t *testing.T) { + // Fixture Setup + // language=JSON + jsonTxt := fmt.Sprintf(` + { + "type": "%s", + "args": ["amount"] + } + `, strings.ToUpper(unaryOp)) + + ast, err := GetAst(jsonTxt) + require.NoError(t, err) + + // Execute SUT + err = ValidateAstFieldAndOperators(ast, map[string][]string{"amount": {unaryOp}}) + + // Verification + require.NoError(t, err) + }) + } +} + +func TestValidationReturnsNoErrorForUnaryOperatorWhenAstUsesAliasesAndSatisfiesConstraints(t *testing.T) { + + for _, unaryOp := range unaryOps { + t.Run(fmt.Sprintf("%s", unaryOp), func(t *testing.T) { + // Fixture Setup + // language=JSON + jsonTxt := fmt.Sprintf(` + { + "type": "%s", + "args": ["amount"] + } + `, strings.ToUpper(unaryOp)) + + ast, err := GetAst(jsonTxt) + require.NoError(t, err) + + // Execute SUT + err = ValidateAstFieldAndOperatorsWithAliases(ast, map[string][]string{"total": {unaryOp}}, map[string]string{"amount": "total"}) + + // Verification + require.NoError(t, err) + }) + } +} + +func TestValidationReturnsNoErrorForUnaryOperatorsWhenAstUseAliasesAndValueValidationAndSatisfiesConstraints(t *testing.T) { + + for _, unaryOp := range unaryOps { + t.Run(fmt.Sprintf("%s", unaryOp), func(t *testing.T) { + // Fixture Setup + // language=JSON + jsonTxt := fmt.Sprintf(` + { + "type": "%s", + "args": [ "order_status"] + } + `, strings.ToUpper(unaryOp)) + + ast, err := GetAst(jsonTxt) + require.NoError(t, err) + + // Execute SUT + // Note: value validation doesn't do anything with is_null but importantly it doesn't crash which is what we test + err = ValidateAstFieldAndOperatorsWithAliasesAndValueValidation(ast, map[string][]string{"status": {unaryOp}}, map[string]string{"order_status": "status"}, map[string]string{"status": "oneof=incomplete complete processing cancelled"}) + + // Verification + require.NoError(t, err) + }) + } +} + +func TestSmokeTestAndWithUnaryAndVariableReturnsErrorWhenBothAreInvalid(t *testing.T) { + for _, varOp := range varOps { + for _, unaryOp := range unaryOps { + + t.Run(fmt.Sprintf("%s/%s", varOp, unaryOp), func(t *testing.T) { + // Fixture Setup + // language=JSON + jsonTxt := fmt.Sprintf(` + { + "type": "AND", + "children": [ + { + "type": "%s", + "args": [ "status", "complete", "cancelled"] + }, + { + "type": "%s", + "args": [ "some_field"] + } + ] +}`, strings.ToUpper(varOp), strings.ToUpper(unaryOp)) + + ast, err := GetAst(jsonTxt) + require.NoError(t, err) + + // Execute SUT + err = ValidateAstFieldAndOperatorsWithValueValidation(ast, map[string][]string{"status": {varOp}, "other_field": {unaryOp}}, map[string]string{"status": "oneof=incomplete complete processing cancelled"}) + + // Verification + require.ErrorContains(t, err, fmt.Sprint("unknown field [some_field] specified in search filter")) + }) + + } + } +} + func TestValidationReturnsErrorForVariableOperatorsWhenAstUsesInvalidOperatorForKnownField(t *testing.T) { for idx, varOp := range varOps { t.Run(fmt.Sprintf("%s", varOp), func(t *testing.T) { diff --git a/external/epsearchast/v3/validating_visitor.go b/external/epsearchast/v3/validating_visitor.go index e064c30..1b8724b 100644 --- a/external/epsearchast/v3/validating_visitor.go +++ b/external/epsearchast/v3/validating_visitor.go @@ -136,6 +136,16 @@ func (v *validatingVisitor) VisitLike(astNode *AstNode) (bool, error) { return false, nil } +func (v *validatingVisitor) VisitIsNull(astNode *AstNode) (bool, error) { + fieldName := astNode.Args[0] + + if err := v.validateFieldAndValue("is_null", fieldName); err != nil { + return false, err + } + + return false, nil +} + func (v *validatingVisitor) isOperatorValidForField(operator, requestField string) (bool, error) { canonicalField := requestField