Make groups&items propagations twice faster, do not join groups table to determine group memberships, do not mark new items for propagation + testing-related fixes

.circleci/config.yml

@@ -89,7 +89,7 @@ jobs:
       - image: circleci/mysql:8.0.28
         command: --default-authentication-plugin=mysql_native_password --max-allowed-packet=10485760
         environment:
-          MYSQL_USER: franceioi
+          MYSQL_USER: algorea
           MYSQL_PASSWORD: dummy_password
           MYSQL_DATABASE: algorea_example
           MYSQL_ROOT_PASSWORD: root

app/api/answers/create_answer.robustness.feature

@@ -4,8 +4,9 @@ Feature: Create a 'saved' answer - robustness
       | login | group_id |
       | john  | 101      |
     And the database table "groups" also has the following row:
-      | id | type |
-      | 13 | Team |
+      | id | type  |
+      | 13 | Team  |
+      | 22 | Class |
     And the database has the following table "groups_groups":
       | parent_group_id | child_group_id |
       | 22              | 13             |

app/api/answers/get_answer.go

@@ -63,8 +63,8 @@ func (srv *Service) getAnswer(rw http.ResponseWriter, httpReq *http.Request) ser
 	userAndHisTeamsQuery := store.Raw("SELECT id FROM ? `teams` UNION ALL SELECT ?",
 		store.ActiveGroupGroups().
 			WhereUserIsMember(user).
-			Joins("JOIN `groups` ON groups.id = groups_groups_active.parent_group_id AND groups.type='Team'").
-			Select("groups.id").SubQuery(),
+			Where("groups_groups_active.is_team_membership = 1").
+			Select("groups_groups_active.parent_group_id AS id").SubQuery(),
 		user.GroupID)
 
 	// a participant should have at least 'content' access to the answers.item_id
@@ -77,7 +77,7 @@ func (srv *Service) getAnswer(rw http.ResponseWriter, httpReq *http.Request) ser
 		Joins("JOIN `groups_ancestors_active` ON groups_ancestors_active.ancestor_group_id = permissions.group_id").
 		Joins("JOIN `groups_groups_active` ON groups_groups_active.parent_group_id = groups_ancestors_active.child_group_id").
 		Where("groups_groups_active.child_group_id = ?", user.GroupID).
-		Joins("JOIN `groups` ON groups.id = groups_groups_active.parent_group_id AND groups.type='Team'").
+		Where("groups_groups_active.is_team_membership = 1").
 		WherePermissionIsAtLeast("view", "content").
 		Where("permissions.item_id = answers.item_id").
 		Select("1").Limit(1).SubQuery()

app/api/answers/submit.feature

@@ -6,10 +6,6 @@ Feature: Submit a new answer
     And the database has the following table "groups":
       | id  | name | type |
       | 201 | team | Team |
-    And the database has the following table "groups_groups":
-      | parent_group_id | child_group_id |
-      | 22              | 13             |
-      | 201             | 101            |
     And the groups ancestors are computed
     And the database has the following table "items":
       | id | default_language_tag |

app/api/answers/submit.robustness.feature

@@ -3,9 +3,6 @@ Feature: Submit a new answer - robustness
     Given the database has the following users:
       | login | group_id |
       | john  | 101      |
-    And the database has the following table "groups_groups":
-      | parent_group_id | child_group_id |
-      | 22              | 13             |
     And the groups ancestors are computed
     And the database has the following table "items":
       | id | read_only | default_language_tag |

app/api/answers/update_current.robustness.feature

@@ -3,6 +3,10 @@ Feature: Update participant's current answer
     Given the database has the following users:
       | login | group_id |
       | john  | 101      |
+    And the database has the following table "groups":
+      | id  |
+      | 13  |
+      | 22  |
     And the database has the following table "groups_groups":
       | parent_group_id | child_group_id |
       | 22              | 13             |

app/api/currentuser/get_dump.feature

@@ -95,28 +95,32 @@ Feature: Export the short version of the current user's data
           "lock_membership_approved_at": null, "lock_membership_approved": 0,
           "personal_info_view_approved_at": null, "personal_info_view_approved": 0,
           "watch_approved_at": null, "watch_approved": 0,
-          "name": "Our Team", "expires_at": "9999-12-31T23:59:59Z"
+          "name": "Our Team", "expires_at": "9999-12-31T23:59:59Z",
+          "is_team_membership": 1
         },
         {
           "child_group_id": "11", "parent_group_id": "5",
           "lock_membership_approved_at": null, "lock_membership_approved": 0,
           "personal_info_view_approved_at": null, "personal_info_view_approved": 0,
           "watch_approved_at": null, "watch_approved": 0,
-          "name": "Other people", "expires_at": "9999-12-31T23:59:59Z"
+          "name": "Other people", "expires_at": "9999-12-31T23:59:59Z",
+          "is_team_membership": 0
         },
         {
           "child_group_id": "11", "parent_group_id": "6",
           "lock_membership_approved_at": null, "lock_membership_approved": 0,
           "personal_info_view_approved_at": null, "personal_info_view_approved": 0,
           "watch_approved_at": null, "watch_approved": 0,
-          "name": "Another Class", "expires_at": "9999-12-31T23:59:59Z"
+          "name": "Another Class", "expires_at": "9999-12-31T23:59:59Z",
+          "is_team_membership": 0
         },
         {
           "child_group_id": "11", "parent_group_id": "9",
           "lock_membership_approved_at": null, "lock_membership_approved": 0,
           "personal_info_view_approved_at": null, "personal_info_view_approved": 0,
           "watch_approved_at": null, "watch_approved": 0,
-          "name": "Some other friends", "expires_at": "9999-12-31T23:59:59Z"
+          "name": "Some other friends", "expires_at": "9999-12-31T23:59:59Z",
+          "is_team_membership": 0
         }
       ],
       "group_managers": [

app/api/currentuser/get_full_dump.feature

@@ -136,35 +136,40 @@ Feature: Export the current user's data
           "lock_membership_approved_at": null, "lock_membership_approved": 0,
           "personal_info_view_approved_at": null, "personal_info_view_approved": 0,
           "watch_approved_at": null, "watch_approved": 0,
-          "name": "Our Team", "expires_at": "9999-12-31T23:59:59Z"
+          "name": "Our Team", "expires_at": "9999-12-31T23:59:59Z",
+          "is_team_membership": 1
         },
         {
           "child_group_id": "11", "parent_group_id": "5",
           "lock_membership_approved_at": null, "lock_membership_approved": 0,
           "personal_info_view_approved_at": null, "personal_info_view_approved": 0,
           "watch_approved_at": null, "watch_approved": 0,
-          "name": "Other people", "expires_at": "9999-12-31T23:59:59Z"
+          "name": "Other people", "expires_at": "9999-12-31T23:59:59Z",
+          "is_team_membership": 0
         },
         {
           "child_group_id": "11", "parent_group_id": "6",
           "lock_membership_approved_at": null, "lock_membership_approved": 0,
           "personal_info_view_approved_at": null, "personal_info_view_approved": 0,
           "watch_approved_at": null, "watch_approved": 0,
-          "name": "Another Class", "expires_at": "9999-12-31T23:59:59Z"
+          "name": "Another Class", "expires_at": "9999-12-31T23:59:59Z",
+          "is_team_membership": 0
         },
         {
           "child_group_id": "11", "parent_group_id": "9",
           "lock_membership_approved_at": null, "lock_membership_approved": 0,
           "personal_info_view_approved_at": null, "personal_info_view_approved": 0,
           "watch_approved_at": null, "watch_approved": 0,
-          "name": "Some other friends", "expires_at": "9999-12-31T23:59:59Z"
+          "name": "Some other friends", "expires_at": "9999-12-31T23:59:59Z",
+          "is_team_membership": 0
         },
         {
           "child_group_id": "11", "parent_group_id": "10",
           "lock_membership_approved_at": null, "lock_membership_approved": 0,
           "personal_info_view_approved_at": null, "personal_info_view_approved": 0,
           "watch_approved_at": null, "watch_approved": 0,
-          "name": "Secret group", "expires_at": "9999-12-31T23:59:59Z"
+          "name": "Secret group", "expires_at": "9999-12-31T23:59:59Z",
+          "is_team_membership": 0
         }
       ],
       "group_managers": [

app/api/currentuser/get_full_dump.go

@@ -85,6 +85,7 @@ func (srv *Service) getDumpCommon(r *http.Request, w http.ResponseWriter, full b
 			service.MustNotBeError(store.Sessions().
 				Where("user_id = ?", user.GroupID).
 				Select(columns + ", '***' AS refresh_token").
+				Order("session_id").
 				ScanAndHandleMaps(streamerFunc(w)).Error())
 		})
 
@@ -95,6 +96,7 @@ func (srv *Service) getDumpCommon(r *http.Request, w http.ResponseWriter, full b
 				Joins("JOIN sessions ON sessions.session_id = access_tokens.session_id").
 				Where("sessions.user_id = ?", user.GroupID).
 				Select(columns + ", '***' AS token").
+				Order("session_id").
 				ScanAndHandleMaps(streamerFunc(w)).Error())
 		})
 	}
@@ -121,6 +123,7 @@ func (srv *Service) getDumpCommon(r *http.Request, w http.ResponseWriter, full b
 		writeComma(w)
 		writeJSONObjectArrayElement("answers", w, func(writer io.Writer) {
 			service.MustNotBeError(store.Answers().Where("author_id = ?", user.GroupID).
+				Order("id").
 				ScanAndHandleMaps(streamerFunc(w)).Error())
 		})
 
@@ -144,6 +147,7 @@ func (srv *Service) getDumpCommon(r *http.Request, w http.ResponseWriter, full b
 			Where("child_group_id = ?", user.GroupID).
 			Joins("JOIN `groups` ON `groups`.id = parent_group_id").
 			Select(columns + ", `groups`.name").
+			Order("parent_group_id").
 			ScanAndHandleMaps(streamerFunc(w)).Error())
 	})
 
@@ -154,6 +158,7 @@ func (srv *Service) getDumpCommon(r *http.Request, w http.ResponseWriter, full b
 			Where("manager_id = ?", user.GroupID).
 			Joins("JOIN `groups` ON `groups`.id = group_id").
 			Select(columns + ", `groups`.name").
+			Order("group_id").
 			ScanAndHandleMaps(streamerFunc(w)).Error())
 	})
 
@@ -176,6 +181,7 @@ func (srv *Service) getDumpCommon(r *http.Request, w http.ResponseWriter, full b
 				Where("member_id = ?", user.GroupID).
 				Joins("JOIN `groups` ON `groups`.id = group_id").
 				Select(columns + ", `groups`.name").
+				Order("group_id").
 				ScanAndHandleMaps(streamerFunc(w)).Error())
 		})
 	}
@@ -266,7 +272,7 @@ func buildQueryForGettingAttemptsOrResults(store *database.DataStore, user *data
 				Select(columns).
 				Where("participant_id IN (?)",
 					store.GroupGroups().WhereUserIsMember(user).
-						Select("`groups`.id").
-						Joins("JOIN `groups` ON `groups`.id = groups_groups.parent_group_id AND `groups`.type = 'Team'").
+						Where("groups_groups.is_team_membership = 1").
+						Select("groups_groups.parent_group_id AS id").
 						QueryExpr()))
 }

app/api/currentuser/get_group_activities.robustness.feature

@@ -2,6 +2,7 @@ Feature: Get root activities for a participant group - robustness
   Background:
     Given the database has the following table "groups":
       | id | name      | type  | root_activity_id | created_at          |
+      | 1  | Class     | Class | null             | 2019-01-30 08:26:46 |
       | 11 | jdoe      | User  | null             | 2019-01-30 08:26:48 |
       | 13 | Group B   | Team  | 230              | 2019-01-30 08:26:46 |
       | 14 | Group C   | Team  | 230              | 2019-01-30 08:26:46 |

app/api/currentuser/get_group_skills.robustness.feature

@@ -2,6 +2,7 @@ Feature: Get root skills for a participant group - robustness
   Background:
     Given the database has the following table "groups":
       | id | name      | type  | root_activity_id | created_at          |
+      | 1  | Class     | Class | null             | 2019-01-30 08:26:46 |
       | 13 | Group B   | Team  | 230              | 2019-01-30 08:26:46 |
       | 14 | Group C   | Team  | 230              | 2019-01-30 08:26:46 |
     And the database has the following table "languages":

app/api/currentuser/get_managed_groups.feature

@@ -1,11 +1,14 @@
 Feature: List groups managed by the current user
   Background:
     Given the database has the following table "groups":
-      | id | name          | type  | description |
-      | 5  | Group         | Class | null        |
-      | 13 | Our Class     | Class | null        |
-      | 14 | Our Friends   | Other | null        |
-      | 15 | Another Group | Other | Super Group |
+      | id | name          | type    | description |
+      | 1  | Friends       | Friends | null        |
+      | 5  | Group         | Class   | null        |
+      | 6  | Club          | Club    | null        |
+      | 9  | Other         | Other   | null        |
+      | 13 | Our Class     | Class   | null        |
+      | 14 | Our Friends   | Other   | null        |
+      | 15 | Another Group | Other   | Super Group |
     And the database has the following users:
       | group_id | login |
       | 21       | owner |

app/api/groups/create_invitations.go

@@ -211,15 +211,14 @@ func getOtherTeamsMembers(store *database.DataStore, parentGroupID int64, groups
 
 	var otherTeamsMembers []int64
 	service.MustNotBeError(store.ActiveGroupGroups().Where("child_group_id IN (?)", groupsToCheck).
-		Joins("JOIN `groups` ON groups.id = groups_groups_active.parent_group_id").
 		Joins("JOIN (?) AS teams_contests",
 			contestsQuery. // all the team's attempts (not only active ones)
 					Select(`
 					  root_item_id AS item_id,
 					  MAX(NOW() < attempts.allows_submissions_until AND attempts.ended_at IS NULL) AS is_active`).QueryExpr()).
 		Joins("JOIN items ON items.id = teams_contests.item_id").
-		Joins("JOIN attempts ON attempts.participant_id = groups.id AND attempts.root_item_id = items.id").
-		Where("groups.type = 'Team'").
+		Joins("JOIN attempts ON attempts.participant_id = groups_groups_active.parent_group_id AND attempts.root_item_id = items.id").
+		Where("groups_groups_active.is_team_membership = 1").
 		Where("parent_group_id != ?", parentGroupID).
 		Where(`
 			(teams_contests.is_active AND NOW() < attempts.allows_submissions_until AND attempts.ended_at IS NULL) OR

app/api/groups/get_current_user_team_by_item.go

@@ -54,7 +54,8 @@ func (srv *Service) getCurrentUserTeamByItem(w http.ResponseWriter, r *http.Requ
 
 	var teamID int64
 	user := srv.GetUser(r)
-	err = srv.GetStore(r).Groups().TeamGroupForTeamItemAndUser(itemID, user).PluckFirst("groups.id", &teamID).Error()
+	err = srv.GetStore(r).ActiveGroupGroups().TeamGroupForTeamItemAndUser(itemID, user).
+		PluckFirst("groups_groups_active.parent_group_id", &teamID).Error()
 	if gorm.IsRecordNotFoundError(err) {
 		return service.ErrNotFound(errors.New("no team for this item"))
 	}

app/api/groups/get_user_progress.go

@@ -206,10 +206,9 @@ func joinUserProgressResults(db *database.DB, userID interface{}) *database.DB {
 	return db.
 		Joins(`
 			LEFT JOIN LATERAL (
-				SELECT STRAIGHT_JOIN groups.id
+				SELECT STRAIGHT_JOIN groups_groups_active.parent_group_id AS id
 				FROM groups_groups_active
-				JOIN `+"`groups`"+` ON groups.id = groups_groups_active.parent_group_id
-				WHERE groups.type = 'Team' AND groups_groups_active.child_group_id = ?
+				WHERE groups_groups_active.is_team_membership = 1 AND groups_groups_active.child_group_id = ?
 			) teams ON 1`, userID).
 		Joins(`
 			LEFT JOIN LATERAL (

app/api/groups/get_user_progress_csv.go

@@ -266,10 +266,9 @@ func joinUserProgressResultsForCSV(db *database.DB, userID interface{}) *databas
 	return db.
 		Joins(`
 			LEFT JOIN LATERAL (
-				SELECT STRAIGHT_JOIN groups.id
+				SELECT STRAIGHT_JOIN groups_groups_active.parent_group_id AS id
 				FROM groups_groups_active
-				JOIN `+"`groups`"+` ON groups.id = groups_groups_active.parent_group_id
-				WHERE groups.type = 'Team' AND groups_groups_active.child_group_id = ?
+				WHERE groups_groups_active.is_team_membership = 1 AND groups_groups_active.child_group_id = ?
 			) teams ON 1`, userID).
 		Joins(`
 			LEFT JOIN LATERAL (

app/api/items/ask_hint.feature

@@ -8,7 +8,6 @@ Feature: Ask for a hint
       | 201 | team | Team |
     And the database has the following table "groups_groups":
       | parent_group_id | child_group_id |
-      | 22              | 13             |
       | 201             | 101            |
     And the groups ancestors are computed
     And the database has the following table "platforms":

app/api/items/ask_hint.robustness.feature

@@ -3,10 +3,6 @@ Feature: Ask for a hint - robustness
     Given the database has the following users:
       | login | group_id |
       | john  | 101      |
-    And the database has the following table "groups_groups":
-      | parent_group_id | child_group_id |
-      | 22              | 13             |
-    And the groups ancestors are computed
     And the database has the following table "platforms":
       | id | regexp                     | public_key                | priority |
       | 10 | https://platformwithkey    | {{taskPlatformPublicKey}} | 0        |

app/api/items/create_item.go

@@ -264,8 +264,6 @@ func validateAndInsertItem(srv *Service, r *http.Request) (itemID int64, apiErro
 
 		setNewItemAsRootActivityOrSkill(store, formData, &input, itemID)
 
-		service.MustNotBeError(store.ItemItems().CreateNewAncestors())
-
 		return nil
 	})
 	if err == nil {
@@ -617,6 +615,8 @@ func (srv *Service) insertItem(store *database.DataStore, user *database.User, f
 		parentChildSpec = append(parentChildSpec,
 			constructItemsItemsForChildren(newItemRequest.Children, itemID)...)
 		insertItemItems(store, parentChildSpec)
+
+		service.MustNotBeError(store.ItemItems().CreateNewAncestors())
 	}
 
 	return itemID, service.NoError

app/api/items/get_entry_state.go

@@ -306,11 +306,13 @@ func getEntryStateInfo(groupID, itemID int64, user *database.User, store *databa
 		membersCount = int32(len(otherMembers))
 
 		participatingSomewhereElseQuery := store.ActiveGroupGroups().Where("groups_groups_active.parent_group_id = ?", groupID).
-			Joins("JOIN groups_groups_active AS all_teams_relations ON all_teams_relations.child_group_id = groups_groups_active.child_group_id").
-			Joins("JOIN `groups` AS groups_to_check ON groups_to_check.id = all_teams_relations.parent_group_id AND groups_to_check.type = 'Team'").
+			Joins(`
+				JOIN groups_groups_active AS all_teams_relations
+				  ON all_teams_relations.child_group_id = groups_groups_active.child_group_id AND
+				     all_teams_relations.is_team_membership = 1`).
 			Joins("JOIN items ON items.id = ?", itemID).
-			Joins("JOIN attempts ON attempts.participant_id = groups_to_check.id AND attempts.root_item_id = items.id").
-			Where("groups_to_check.id != groups_groups_active.parent_group_id"). // except for this team
+			Joins("JOIN attempts ON attempts.participant_id = all_teams_relations.parent_group_id AND attempts.root_item_id = items.id").
+			Where("all_teams_relations.parent_group_id != groups_groups_active.parent_group_id"). // except for this team
 			Group("groups_groups_active.child_group_id").
 			Having("MAX(NOW() < attempts.allows_submissions_until) OR NOT MAX(items.allows_multiple_attempts)")
 		if lock {

app/api/items/save_grade.feature

@@ -8,7 +8,6 @@ Feature: Save grading result
       | 201 | team | Team |
     And the database has the following table "groups_groups":
       | parent_group_id | child_group_id |
-      | 22              | 13             |
       | 201             | 101            |
     And the groups ancestors are computed
     And the database has the following table "platforms":

app/api/items/save_grade.robustness.feature

@@ -3,10 +3,6 @@ Feature: Save grading result - robustness
     Given the database has the following users:
       | login | group_id |
       | john  | 101      |
-    And the database has the following table "groups_groups":
-      | parent_group_id | child_group_id |
-      | 22              | 13             |
-    And the groups ancestors are computed
     And the database has the following table "platforms":
       | id | regexp                                             | priority | public_key                |
       | 10 | http://taskplatform.mblockelet.info/task.html\?.*  | 2        | {{taskPlatformPublicKey}} |