first working implem end to end (except index creation) - some fixes

fix test

api/handle_scenario_publications.go

@@ -76,3 +76,18 @@ func (api *API) GetScenarioPublication(c *gin.Context) {
 	}
 	c.JSON(http.StatusOK, NewAPIScenarioPublication(scenarioPublication))
 }
+
+func (api *API) ValidateIndexesForPublication(c *gin.Context) {
+	var data dto.CreateScenarioPublicationBody
+	if err := c.ShouldBindJSON(&data); err != nil {
+		c.Status(http.StatusBadRequest)
+		return
+	}
+
+	usecase := api.UsecasesWithCreds(c.Request).NewScenarioPublicationUsecase()
+	_, err := usecase.CreateDatamodelIndexesForScenarioPublication(c.Request.Context(), data.ScenarioIterationId)
+	if presentError(c, err) {
+		return
+	}
+	c.Status(http.StatusOK)
+}

api/routes.go

@@ -60,6 +60,7 @@ func (api *API) routes(auth *Authentication) {
 
 	router.GET("/scenario-publications", api.ListScenarioPublications)
 	router.POST("/scenario-publications", api.CreateScenarioPublication)
+	router.POST("/scenario-publications/validate-indexes", api.ValidateIndexesForPublication)
 	router.GET("/scenario-publications/:publication_id", api.GetScenarioPublication)
 
 	router.GET("/scheduled-executions", api.handleListScheduledExecution)

models/aggregate_query.go

@@ -11,14 +11,14 @@ import (
 )
 
 type AggregateQueryFamily struct {
-	Table                   TableName
+	TableName               TableName
 	EqConditions            *set.Set[FieldName]
 	IneqConditions          *set.Set[FieldName]
 	SelectOrOtherConditions *set.Set[FieldName]
 }
 
 func (family AggregateQueryFamily) Equal(other AggregateQueryFamily) bool {
-	return family.Table == other.Table &&
+	return family.TableName == other.TableName &&
 		family.EqConditions.Equal(other.EqConditions) &&
 		family.IneqConditions.Equal(other.IneqConditions) &&
 		family.SelectOrOtherConditions.Equal(other.SelectOrOtherConditions)
@@ -48,7 +48,7 @@ func (family AggregateQueryFamily) Hash() string {
 		slices.Sort(s)
 		other = fmt.Sprintf("%v", s)
 	}
-	return fmt.Sprintf("%s %s %s %s", family.Table, eq, ineq, other)
+	return fmt.Sprintf("%s - %s - %s - %s", family.TableName, eq, ineq, other)
 }
 
 func AggregationNodeToQueryFamily(node ast.Node) (AggregateQueryFamily, error) {
@@ -68,7 +68,7 @@ func AggregationNodeToQueryFamily(node ast.Node) (AggregateQueryFamily, error) {
 	aggregatedFieldName := FieldName(aggregatedFieldNameStr)
 
 	family := AggregateQueryFamily{
-		Table:                   TableName(queryTableName),
+		TableName:               TableName(queryTableName),
 		EqConditions:            set.New[FieldName](0),
 		IneqConditions:          set.New[FieldName](0),
 		SelectOrOtherConditions: set.New[FieldName](0),
@@ -153,7 +153,7 @@ func ExtractQueryFamiliesFromAst(node ast.Node) (*set.HashSet[AggregateQueryFami
 }
 
 // simple utility function using ExtractQueryFamiliesFromAst above
-func ExtractQueryFamiliesFromAstSlice(nodes []ast.Node) (*set.HashSet[AggregateQueryFamily, string], error) {
+func extractQueryFamiliesFromAstSlice(nodes []ast.Node) (*set.HashSet[AggregateQueryFamily, string], error) {
 	families := set.NewHashSet[AggregateQueryFamily, string](0)
 
 	for _, node := range nodes {
@@ -179,6 +179,7 @@ func aggregateQueryToIndexFamily(qFamily AggregateQueryFamily) *set.HashSet[Inde
 
 	// first iterate on equality conditions and colunms to include anyway
 	base := NewIndexFamily()
+	base.TableName = qFamily.TableName
 	if qFamily.EqConditions != nil {
 		qFamily.EqConditions.ForEach(func(f FieldName) bool {
 			base.Flex.Insert(f)
@@ -216,10 +217,28 @@ func aggregateQueryToIndexFamily(qFamily AggregateQueryFamily) *set.HashSet[Inde
 	return output
 }
 
-func IndexesToCreateFromQueryFamilies(queryFamilies set.Collection[AggregateQueryFamily], existingIndexes []ConcreteIndex) []ConcreteIndex {
-	toCreateFamilies := set.NewHashSet[IndexFamily, string](0)
+func indexesToCreateFromQueryFamilies(queryFamilies set.Collection[AggregateQueryFamily], existingIndexes []ConcreteIndex) []ConcreteIndex {
+	familiesToCreate := set.NewHashSet[IndexFamily, string](0)
 	for _, q := range queryFamilies.Slice() {
-		toCreateFamilies = toCreateFamilies.Union(selectIdxFamiliesToCreate(q.ToIndexFamilies(), existingIndexes)).(*set.HashSet[IndexFamily, string])
+		familiesToCreate = familiesToCreate.Union(selectIdxFamiliesToCreate(q.ToIndexFamilies(), existingIndexes)).(*set.HashSet[IndexFamily, string])
 	}
-	return SelectConcreteIndexesToCreate(toCreateFamilies, existingIndexes)
+	reducedFamiliesToCreate := extractMinimalSetOfIdxFamilies(familiesToCreate)
+	return selectConcreteIndexesToCreate(reducedFamiliesToCreate, existingIndexes)
+}
+
+func IndexesToCreateFromScenarioIterations(scenarioIterations []ScenarioIteration, existingIndexes []ConcreteIndex) ([]ConcreteIndex, error) {
+	var asts []ast.Node
+	for _, i := range scenarioIterations {
+		asts = append(asts, *i.TriggerConditionAstExpression)
+		for _, r := range i.Rules {
+			asts = append(asts, *r.FormulaAstExpression)
+		}
+	}
+
+	queryFamilies, err := extractQueryFamiliesFromAstSlice(asts)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error extracting query families from scenario iterations")
+	}
+
+	return indexesToCreateFromQueryFamilies(queryFamilies, existingIndexes), nil
 }

models/aggregate_query_test.go

@@ -25,7 +25,7 @@ func TestAggregationNodeToQueryFamily(t *testing.T) {
 		}
 		aggregateFamily, err := AggregationNodeToQueryFamily(node)
 		asserts.NoError(err)
-		asserts.Equal(TableName("table"), aggregateFamily.Table, "table name should be input table name")
+		asserts.Equal(TableName("table"), aggregateFamily.TableName, "table name should be input table name")
 	})
 
 	t.Run("missing filters child", func(t *testing.T) {
@@ -67,7 +67,7 @@ func TestAggregationNodeToQueryFamily(t *testing.T) {
 		}
 		aggregateFamily, err := AggregationNodeToQueryFamily(node)
 		asserts.NoError(err)
-		asserts.Equal(TableName("table"), aggregateFamily.Table)
+		asserts.Equal(TableName("table"), aggregateFamily.TableName)
 		asserts.Equal(1, aggregateFamily.EqConditions.Size())
 		asserts.True(aggregateFamily.EqConditions.Contains(FieldName("field")))
 		asserts.Equal(0, aggregateFamily.IneqConditions.Size())
@@ -105,7 +105,7 @@ func TestAggregationNodeToQueryFamily(t *testing.T) {
 		}
 		aggregateFamily, err := AggregationNodeToQueryFamily(node)
 		asserts.NoError(err)
-		asserts.Equal(TableName("table"), aggregateFamily.Table)
+		asserts.Equal(TableName("table"), aggregateFamily.TableName)
 		asserts.Equal(1, aggregateFamily.EqConditions.Size())
 		asserts.True(aggregateFamily.EqConditions.Contains(FieldName("field")))
 		asserts.Equal(0, aggregateFamily.IneqConditions.Size())
@@ -143,7 +143,7 @@ func TestAggregationNodeToQueryFamily(t *testing.T) {
 		}
 		aggregateFamily, err := AggregationNodeToQueryFamily(node)
 		asserts.NoError(err)
-		asserts.Equal(TableName("table"), aggregateFamily.Table)
+		asserts.Equal(TableName("table"), aggregateFamily.TableName)
 		asserts.Equal(1, aggregateFamily.EqConditions.Size())
 		asserts.True(aggregateFamily.EqConditions.Contains(FieldName("field 1")))
 		asserts.Equal(1, aggregateFamily.IneqConditions.Size())
@@ -223,7 +223,7 @@ func TestAggregationNodeToQueryFamily(t *testing.T) {
 		}
 		aggregateFamily, err := AggregationNodeToQueryFamily(node)
 		asserts.NoError(err)
-		asserts.Equal(TableName("table"), aggregateFamily.Table)
+		asserts.Equal(TableName("table"), aggregateFamily.TableName)
 		asserts.Equal(2, aggregateFamily.EqConditions.Size(), "EqConditions should contain field 1 and field 2")
 		asserts.True(aggregateFamily.EqConditions.Contains(FieldName("field 1")), "EqConditions should contain field 1")
 		asserts.True(aggregateFamily.EqConditions.Contains(FieldName("field 2")), "EqConditions should contain field 2")
@@ -286,7 +286,7 @@ func TestAstNodeToQueryFamilies(t *testing.T) {
 		asserts.Equal(1, output.Size(), "There should be only 1 query family in the output set")
 		expected := set.NewHashSet[AggregateQueryFamily](0)
 		expected.Insert(AggregateQueryFamily{
-			Table:                   TableName("table"),
+			TableName:               TableName("table"),
 			EqConditions:            set.From[FieldName]([]FieldName{"field"}),
 			IneqConditions:          set.New[FieldName](0),
 			SelectOrOtherConditions: set.From[FieldName]([]FieldName{"field 0"}),
@@ -373,19 +373,19 @@ func TestAstNodeToQueryFamilies(t *testing.T) {
 		asserts.Equal(3, output.Size(), "There should be 2 query families in the output set")
 		expected := set.NewHashSet[AggregateQueryFamily](0)
 		expected.Insert(AggregateQueryFamily{
-			Table:                   TableName("table"),
+			TableName:               TableName("table"),
 			EqConditions:            set.From[FieldName]([]FieldName{"field"}),
 			IneqConditions:          set.New[FieldName](0),
 			SelectOrOtherConditions: set.From[FieldName]([]FieldName{"field 0"}),
 		})
 		expected.Insert(AggregateQueryFamily{
-			Table:                   TableName("table"),
+			TableName:               TableName("table"),
 			EqConditions:            set.New[FieldName](0),
 			IneqConditions:          set.From[FieldName]([]FieldName{"field"}),
 			SelectOrOtherConditions: set.From[FieldName]([]FieldName{"field 0"}),
 		})
 		expected.Insert(AggregateQueryFamily{
-			Table:                   TableName("table"),
+			TableName:               TableName("table"),
 			EqConditions:            set.From[FieldName]([]FieldName{"field 0"}),
 			IneqConditions:          set.New[FieldName](0),
 			SelectOrOtherConditions: set.From[FieldName]([]FieldName{"field 2", "field 3"}),

models/aggregate_query_to_idx_test.go

@@ -14,7 +14,7 @@ func TestAggregateQueryToIndexFamily(t *testing.T) {
 		asserts := assert.New(t)
 
 		qFamily := AggregateQueryFamily{
-			Table:                   "table",
+			TableName:               "",
 			EqConditions:            set.From[FieldName]([]FieldName{FieldName("a"), FieldName("b")}),
 			IneqConditions:          set.From[FieldName]([]FieldName{FieldName("c"), FieldName("d")}),
 			SelectOrOtherConditions: set.From[FieldName]([]FieldName{FieldName("e"), FieldName("f")}),
@@ -43,7 +43,7 @@ func TestAggregateQueryToIndexFamily(t *testing.T) {
 		asserts := assert.New(t)
 
 		qFamily := AggregateQueryFamily{
-			Table:                   "table",
+			TableName:               "",
 			EqConditions:            set.From[FieldName]([]FieldName{FieldName("a"), FieldName("b")}),
 			IneqConditions:          set.From[FieldName]([]FieldName{FieldName("c")}),
 			SelectOrOtherConditions: set.From[FieldName]([]FieldName{FieldName("d"), FieldName("e")}),
@@ -71,7 +71,7 @@ func TestAggregateQueryToIndexFamily(t *testing.T) {
 		asserts := assert.New(t)
 
 		qFamily := AggregateQueryFamily{
-			Table:                   "table",
+			TableName:               "",
 			EqConditions:            set.From[FieldName]([]FieldName{FieldName("a"), FieldName("b")}),
 			IneqConditions:          set.New[FieldName](0),
 			SelectOrOtherConditions: set.From[FieldName]([]FieldName{FieldName("d"), FieldName("e")}),

models/concrete_index.go

@@ -7,8 +7,9 @@ import (
 )
 
 type ConcreteIndex struct {
-	Indexed  []FieldName
-	Included []FieldName
+	TableName TableName
+	Indexed   []FieldName
+	Included  []FieldName
 }
 
 func (i ConcreteIndex) Equal(other ConcreteIndex) bool {
@@ -17,6 +18,10 @@ func (i ConcreteIndex) Equal(other ConcreteIndex) bool {
 }
 
 func (i ConcreteIndex) isInstanceof(f IndexFamily) bool {
+	if i.TableName != f.TableName {
+		return false
+	}
+
 	if len(i.Indexed) < len(f.Fixed) {
 		return false
 	}
@@ -57,7 +62,7 @@ func selectIdxFamiliesToCreate(idxFamilies set.Collection[IndexFamily], existing
 	return toCreate
 }
 
-func SelectConcreteIndexesToCreate(idxFamilies set.Collection[IndexFamily], existing []ConcreteIndex) []ConcreteIndex {
+func selectConcreteIndexesToCreate(idxFamilies set.Collection[IndexFamily], existing []ConcreteIndex) []ConcreteIndex {
 	toCreateFamilies := selectIdxFamiliesToCreate(idxFamilies, existing)
 
 	toCreateIdx := make([]ConcreteIndex, 0, toCreateFamilies.Size())
@@ -68,8 +73,9 @@ func SelectConcreteIndexesToCreate(idxFamilies set.Collection[IndexFamily], exis
 			indexed = append(indexed, family.Last)
 		}
 		toCreateIdx = append(toCreateIdx, ConcreteIndex{
-			Indexed:  indexed,
-			Included: family.Others.Slice(),
+			TableName: family.TableName,
+			Indexed:   indexed,
+			Included:  family.Others.Slice(),
 		})
 		return true
 	})

models/index_family.go

@@ -8,10 +8,11 @@ import (
 )
 
 type IndexFamily struct {
-	Fixed  []FieldName
-	Flex   *set.Set[FieldName]
-	Last   FieldName
-	Others *set.Set[FieldName]
+	TableName TableName
+	Fixed     []FieldName
+	Flex      *set.Set[FieldName]
+	Last      FieldName
+	Others    *set.Set[FieldName]
 }
 
 func NewIndexFamily() IndexFamily {
@@ -43,15 +44,16 @@ func (f IndexFamily) Hash() string {
 		slices.Sort(s)
 		ot = fmt.Sprintf("%v", s)
 	}
-	return fmt.Sprintf("%v %s %s %s", f.Fixed, fl, f.Last, ot)
+	return fmt.Sprintf("%s - %v - %s - %s - %s", f.TableName, f.Fixed, fl, f.Last, ot)
 }
 
 func (f IndexFamily) copy() IndexFamily {
 	return IndexFamily{
-		Fixed:  slices.Clone(f.Fixed),
-		Flex:   f.Flex.Copy(),
-		Last:   f.Last,
-		Others: f.Others.Copy(),
+		TableName: f.TableName,
+		Fixed:     slices.Clone(f.Fixed),
+		Flex:      f.Flex.Copy(),
+		Last:      f.Last,
+		Others:    f.Others.Copy(),
 	}
 }
 
@@ -109,7 +111,18 @@ func (f *IndexFamily) setLast(last FieldName) {
 	}
 }
 
-func ExtractMinimalSetOfIdxFamilies(idxFamiliesIn *set.HashSet[IndexFamily, string]) *set.HashSet[IndexFamily, string] {
+func extractMinimalSetOfIdxFamilies(idxFamiliesIn *set.HashSet[IndexFamily, string]) *set.HashSet[IndexFamily, string] {
+	// We do the procedure once for every table, on every index required on that table
+	output := []IndexFamily{}
+	familiesByTable := groupIdxFamiliesByTable(idxFamiliesIn)
+	for _, families := range familiesByTable {
+		output = append(output, extractMinimalSetOfIdxFamiliesOneTable(families).Slice()...)
+	}
+
+	return set.HashSetFrom(output)
+}
+
+func extractMinimalSetOfIdxFamiliesOneTable(idxFamiliesIn *set.HashSet[IndexFamily, string]) *set.HashSet[IndexFamily, string] {
 	// We iterate over the input set of families, and try to reduce the number in the ouput step by step by combining families
 	// or indexes where possible
 	output := []IndexFamily{}
@@ -118,22 +131,40 @@ func ExtractMinimalSetOfIdxFamilies(idxFamiliesIn *set.HashSet[IndexFamily, stri
 
 	for _, idxIn := range input {
 		foundReplacement := false
-		for _, idxOut := range output {
-			combined, ok := refineIdxFamilies(idxOut, idxIn)
+		var combined IndexFamily
+		var matchIdx int
+		for i, idxOut := range output {
+			var ok bool
+			combined, ok = refineIdxFamilies(idxOut, idxIn)
 			if ok {
 				output = append(output, combined)
 				foundReplacement = true
+				matchIdx = i
 				break
 			}
 		}
-		if !foundReplacement {
+		if foundReplacement {
+			output = slices.Delete(output, matchIdx, matchIdx+1)
+			output = append(output, combined)
+		} else {
 			output = append(output, idxIn)
 		}
 	}
 
 	return set.HashSetFrom(output)
 }
 
+func groupIdxFamiliesByTable(idxFamilies *set.HashSet[IndexFamily, string]) map[TableName]*set.HashSet[IndexFamily, string] {
+	out := make(map[TableName]*set.HashSet[IndexFamily, string])
+	for _, idxFamily := range idxFamilies.Slice() {
+		if _, ok := out[idxFamily.TableName]; !ok {
+			out[idxFamily.TableName] = set.NewHashSet[IndexFamily, string](0)
+		}
+		out[idxFamily.TableName].Insert(idxFamily)
+	}
+	return out
+}
+
 func compareIdxFamily(a, b IndexFamily) int {
 	if a.Hash() < b.Hash() {
 		return -1
@@ -174,7 +205,7 @@ func refineIdxFamilies(left, right IndexFamily) (IndexFamily, bool) {
 		long = right
 	}
 
-	return refineIdxFamiliesShortHasNoFixed(short, long)
+	return refineIdxFamiliesFirstHasNoFixed(short, long)
 }
 
 func (f IndexFamily) mergeOthers(B IndexFamily) IndexFamily {
@@ -184,17 +215,20 @@ func (f IndexFamily) mergeOthers(B IndexFamily) IndexFamily {
 	return out
 }
 
-func refineIdxFamiliesShortHasNoFixed(A, B IndexFamily) (IndexFamily, bool) {
+func refineIdxFamiliesFirstHasNoFixed(A, B IndexFamily) (IndexFamily, bool) {
 	// we know by hypothesis that A.Fixed = []
 	if A.size() > B.size() {
 		// If some values in B are not in A, it's easy to see that there is no solution (A.Last comes after so can't be used)
-		if !A.allIndexedValues().Subset(B.Flex) {
+		if !A.allIndexedValues().Subset(B.allIndexedValues()) {
 			return IndexFamily{}, false
 		}
 
 		// We know that B's values are included in A.Flex
 		out := B.copy()
 		if B.Last != "" {
+			if B.Last == A.Last {
+				return IndexFamily{}, false
+			}
 			// Now we know that B.Last != "" so we'll need to make some choices
 			fixAppend := B.Flex.Slice()
 			slices.Sort(fixAppend)