BED-5464 - Additional CySQL Fixes (#1189)

* fix: BP-1396, BED-5464: support additional query components; rework translate with to support better group by logic

* chore: PFC

* chore: cleanup

Author: zinic

packages/go/cypher/models/pgsql/test/translation_cases/multipart.sql

@@ -169,13 +169,13 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from
 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not exists (select 1 from edge e0 where e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id);
 
 -- case: match (s) where not (s)-[]->()-[]->() return s
-with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id), s2 as (select s1.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.id = e1.end_id) select count(*) > 0 from s2);
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id), s2 as (select s1.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.id = e1.end_id) select count(*) > 0 from s2);
 
 -- case: match (s) where not (s)-[{prop: 'a'}]-({name: 'n3'}) return s
-with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id or n1.id = e0.start_id where e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on (s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id or n1.id = e0.start_id where e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);
 
 -- case: match (s) where not (s)<-[{prop: 'a'}]-({name: 'n3'}) return s
-with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id where n1.properties ->> 'name' = 'n3' and e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id where n1.properties ->> 'name' = 'n3' and e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);
 
 -- case: match (n:NodeKind1) where n.distinguishedname = toUpper('admin') return n
 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.properties ->> 'distinguishedname' = upper('admin')::text and n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]) select s0.n0 as n from s0;
@@ -190,7 +190,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from
 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.properties ->> 'distinguishedname' like '%' || upper('admin')::text and n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]) select s0.n0 as n from s0;
 
 -- case: match (s) where not (s)-[{prop: 'a'}]->({name: 'n3'}) return s
-with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id where n1.properties ->> 'name' = 'n3' and e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where not (with s1 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.start_id join node n1 on n1.id = e0.end_id where n1.properties ->> 'name' = 'n3' and e0.properties ->> 'prop' = 'a') select count(*) > 0 from s1);
 
 -- case: match (s) where not (s)-[]-() return id(s)
 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (s0.n0).id from s0 where not exists (select 1 from edge e0 where e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id);

packages/go/cypher/models/pgsql/test/translation_cases/nodes.sql

@@ -74,7 +74,7 @@ with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::e
 with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where n1.kind_ids operator (pg_catalog.&&) array [2]::int2[] and e0.kind_id = any (array [3, 4]::int2[]) and n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]) select (s0.n0).properties -> 'name', (s0.n1).properties -> 'name' from s0;
 
 -- case: match (s)-[r:EdgeKind1]->() where (s)-[r {prop: 'a'}]->() return s
-with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.properties ->> 'prop' = 'a' and e0.kind_id = any (array [3]::int2[])) select s0.n0 as s from s0 where (with s1 as (select s0.e0 as e0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n0 on (s0.n0).id = (s0.e0).start_id join node n2 on n2.id = (s0.e0).end_id) select count(*) > 0 from s1);
+with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.properties ->> 'prop' = 'a' and e0.kind_id = any (array [3]::int2[])) select s0.n0 as s from s0 where (with s1 as (select s0.e0 as e0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e0 on (s0.n0).id = (s0.e0).start_id join node n2 on n2.id = (s0.e0).end_id) select count(*) > 0 from s1);
 
 -- case: match (s)-[r:EdgeKind1]->(e) where not (s.system_tags contains 'admin_tier_0') and id(e) = 1 return id(s), labels(s), id(r), type(r)
 with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where n1.id = 1 and e0.kind_id = any (array [3]::int2[]) and not (coalesce(n0.properties ->> 'system_tags', '')::text like '%admin_tier_0%')) select (s0.n0).id, (s0.n0).kind_ids, (s0.e0).id, (s0.e0).kind_id from s0;

packages/go/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql

@@ -55,12 +55,11 @@ func TestTranslate(t *testing.T) {
 	)
 
 	if updateCases, varSet := os.LookupEnv("CYSQL_UPDATE_CASES"); varSet && strings.ToLower(strings.TrimSpace(updateCases)) == "true" {
-		if err := UpdateTranslationTestCases(kindMapper); err  != nil {
+		if err := UpdateTranslationTestCases(kindMapper); err != nil {
 			fmt.Printf("Error updating cases: %v\n", err)
 		}
 	}
 
-
 	if testCases, err := ReadTranslationTestCases(); err != nil {
 		t.Fatal(err)
 	} else {

packages/go/cypher/models/pgsql/test/translation_test.go

@@ -18,22 +18,32 @@ package translate
 
 import (
 	"errors"
+
 	"github.com/specterops/bloodhound/cypher/models/pgsql"
 )
 
 func (s *Translator) buildInlineProjection(part *QueryPart) (pgsql.Select, error) {
-	var sqlSelect pgsql.Select
-
-	if part.projections.Frame != nil {
-		sqlSelect.From = []pgsql.FromClause{{
-			Source: part.projections.Frame.Binding.Identifier,
-		}}
+	sqlSelect := pgsql.Select{
+		Where: part.projections.Constraints,
 	}
 
-	if projectionConstraint, err := s.treeTranslator.ConsumeAll(); err != nil {
-		return sqlSelect, err
-	} else {
-		sqlSelect.Where = projectionConstraint.Expression
+	// If there's a projection frame set, some additional negotiation is required to identify which frame the
+	// from-statement should be written to. Some of this would be better figured out during the translation
+	// of the projection where query scope and other components are not yet fully translated.
+	if part.projections.Frame != nil {
+		// Look up to see if there are CTE expressions registered. If there are then it is likely
+		// there was a projection between this CTE and the previous multipart query part
+		hasCTEs := part.Model.CommonTableExpressions != nil && len(part.Model.CommonTableExpressions.Expressions) > 0
+
+		if part.Frame.Previous == nil || hasCTEs {
+			sqlSelect.From = []pgsql.FromClause{{
+				Source: part.projections.Frame.Binding.Identifier,
+			}}
+		} else {
+			sqlSelect.From = []pgsql.FromClause{{
+				Source: part.Frame.Previous.Binding.Identifier,
+			}}
+		}
 	}
 
 	for _, projection := range part.projections.Items {
@@ -161,7 +171,7 @@ func (s *Translator) translateTraversalPatternPart(part *PatternPart, isolatedPr
 }
 
 func (s *Translator) translateTraversalPatternPartWithoutExpansion(isFirstTraversalStep bool, traversalStep *PatternSegment) error {
-	if constraints, err := s.patternConstraints(isFirstTraversalStep, nonRecursivePattern, traversalStep); err != nil {
+	if constraints, err := consumePatternConstraints(isFirstTraversalStep, nonRecursivePattern, traversalStep, s.treeTranslator.IdentifierConstraints); err != nil {
 		return err
 	} else {
 		if isFirstTraversalStep {
@@ -237,7 +247,7 @@ func (s *Translator) translateTraversalPatternPartWithoutExpansion(isFirstTraver
 	} else {
 		// Zip through all projected identifiers and update their last projected frame
 		for _, binding := range boundProjections.Bindings {
-			binding.LastProjection = traversalStep.Frame
+			binding.MaterializedBy(traversalStep.Frame)
 		}
 
 		traversalStep.Projection = boundProjections.Items
@@ -246,94 +256,8 @@ func (s *Translator) translateTraversalPatternPartWithoutExpansion(isFirstTraver
 	return nil
 }
 
-type PatternConstraints struct {
-	LeftNode  *Constraint
-	Edge      *Constraint
-	RightNode *Constraint
-}
-
-// OptimizePatternConstraintBalance considers the constraints that apply to a pattern segment's bound identifiers.
-//
-// If only the right side of the pattern segment is constrained, this could result in an imbalanced expansion where one side
-// of the traversal has an extreme disparity in search space.
-//
-// In cases that match this heuristic, it's beneficial to begin the traversal with the most tightly constrained set
-// of nodes. To accomplish this we flip the order of the traversal step.
-func (s *PatternConstraints) OptimizePatternConstraintBalance(traversalStep *PatternSegment) {
-	var (
-		// If the left node is previously bound (query knows a set of IDs) the left node is considered to sill be constrained
-		leftNodeHasConstraints  = traversalStep.LeftNodeBound || s.LeftNode.Expression != nil
-		rightNodeHasConstraints = s.RightNode.Expression != nil
-	)
-
-	// (a)-[*..]->(b:Constraint)
-	// (a)<-[*..]-(b:Constraint)
-	if !leftNodeHasConstraints && rightNodeHasConstraints {
-		traversalStep.FlipNodes()
-		s.FlipNodes()
-	}
-}
-
-func (s *PatternConstraints) FlipNodes() {
-	oldLeftNode := s.LeftNode
-	s.LeftNode = s.RightNode
-	s.RightNode = oldLeftNode
-}
-
-const (
-	recursivePattern    = true
-	nonRecursivePattern = false
-)
-
-func (s *Translator) patternConstraints(isFirstTraversalStep, isRecursivePattern bool, traversalStep *PatternSegment) (PatternConstraints, error) {
-	var (
-		constraints PatternConstraints
-		err         error
-	)
-
-	// Even if this isn't the first traversal and the node may be already bound, this should result in an empty
-	// constraint instead of a nil value for `leftNode`
-	if constraints.LeftNode, err = consumeConstraintsFrom(pgsql.AsIdentifierSet(traversalStep.LeftNode.Identifier), s.treeTranslator.IdentifierConstraints); err != nil {
-		return constraints, err
-	}
-
-	if isFirstTraversalStep {
-		// If this is the first traversal step then the left node is just coming into scope
-		traversalStep.Frame.Export(traversalStep.LeftNode.Identifier)
-	}
-
-	// Track the identifiers visible at this frame to correctly assign the remaining constraints
-	knownBindings := traversalStep.Frame.Known()
-
-	if isRecursivePattern {
-		// The exclusion below is done at this step in the process since the recursive descent portion of an expansion
-		// will no longer have a reference to the root node; any dependent interaction between the root and terminal
-		// nodes would require an additional join. By not consuming the remaining constraints for the root and terminal
-		// nodes, they become visible up in the outer select of the recursive CTE.
-		knownBindings.Remove(traversalStep.LeftNode.Identifier)
-	}
-
-	// Export the edge identifier first
-	traversalStep.Frame.Export(traversalStep.Edge.Identifier)
-	knownBindings.Add(traversalStep.Edge.Identifier)
-
-	if constraints.Edge, err = consumeConstraintsFrom(knownBindings, s.treeTranslator.IdentifierConstraints); err != nil {
-		return constraints, err
-	}
-
-	// Export the right node identifier last
-	traversalStep.Frame.Export(traversalStep.RightNode.Identifier)
-	knownBindings.Add(traversalStep.RightNode.Identifier)
-
-	if constraints.RightNode, err = consumeConstraintsFrom(knownBindings, s.treeTranslator.IdentifierConstraints); err != nil {
-		return constraints, err
-	}
-
-	return constraints, nil
-}
-
 func (s *Translator) translateTraversalPatternPartWithExpansion(isFirstTraversalStep bool, traversalStep *PatternSegment) error {
-	if constraints, err := s.patternConstraints(isFirstTraversalStep, recursivePattern, traversalStep); err != nil {
+	if constraints, err := consumePatternConstraints(isFirstTraversalStep, recursivePattern, traversalStep, s.treeTranslator.IdentifierConstraints); err != nil {
 		return err
 	} else {
 		// If one side of the expansion has constraints but the other does not this may be an opportunity to reorder the traversal
@@ -392,15 +316,15 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(isFirstTraversal
 		traversalStep.Expansion.Value.RecursiveConstraints = pgsql.OptionalAnd(traversalStep.Expansion.Value.ExpansionEdgeConstraints, expansionConstraints(expansionFrame.Binding.Identifier, traversalStep.Expansion.Value.MinDepth, traversalStep.Expansion.Value.MaxDepth))
 
 		// Remove the previous projections of the root and terminal node to reproject them after expansion
-		traversalStep.LeftNode.LastProjection = nil
-		traversalStep.RightNode.LastProjection = nil
+		traversalStep.LeftNode.Dematerialize()
+		traversalStep.RightNode.Dematerialize()
 
 		if boundProjections, err := buildVisibleProjections(s.query.Scope); err != nil {
 			return err
 		} else {
 			// Zip through all projected identifiers and update their last projected frame
 			for _, binding := range boundProjections.Bindings {
-				binding.LastProjection = expansionFrame
+				binding.MaterializedBy(expansionFrame)
 			}
 
 			traversalStep.Expansion.Value.Projection = boundProjections.Items
@@ -416,7 +340,7 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(isFirstTraversal
 	} else {
 		// Zip through all projected identifiers and update their last projected frame
 		for _, binding := range boundProjections.Bindings {
-			binding.LastProjection = traversalStep.Frame
+			binding.MaterializedBy(traversalStep.Frame)
 		}
 
 		traversalStep.Projection = boundProjections.Items
@@ -433,7 +357,7 @@ func (s *Translator) translateNonTraversalPatternPart(part *PatternPart) error {
 
 		nextFrame.Export(part.NodeSelect.Binding.Identifier)
 
-		if constraint, err := consumeConstraintsFrom(nextFrame.Known(), s.treeTranslator.IdentifierConstraints); err != nil {
+		if constraint, err := s.treeTranslator.IdentifierConstraints.ConsumeSet(nextFrame.Known()); err != nil {
 			return err
 		} else if err := RewriteFrameBindings(s.query.Scope, constraint.Expression); err != nil {
 			return err
@@ -446,7 +370,7 @@ func (s *Translator) translateNonTraversalPatternPart(part *PatternPart) error {
 		} else {
 			// Zip through all projected identifiers and update their last projected frame
 			for _, binding := range boundProjections.Bindings {
-				binding.LastProjection = nextFrame
+				binding.MaterializedBy(nextFrame)
 			}
 
 			part.NodeSelect.Select.Projection = boundProjections.Items