From 574ff777b53a1f9dc3819713aa631f0aa42cd739 Mon Sep 17 00:00:00 2001
From: Carol Alexandru <alexandru@ifi.uzh.ch>
Date: Wed, 4 Sep 2024 16:45:56 +0900
Subject: [PATCH] Implement pruning of older submissions to save space

---
 .../uzh/ifi/access/config/SecurityConfig.kt   |  3 +-
 .../ifi/access/controller/CourseController.kt |  6 +++
 .../ch/uzh/ifi/access/model/Evaluation.kt     |  3 +-
 .../uzh/ifi/access/service/CourseService.kt   | 39 +++++++++++++++++++
 4 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt b/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt
index fcfa325..44644c4 100644
--- a/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt
+++ b/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt
@@ -83,7 +83,8 @@ class SecurityConfig(private val env: Environment) {
                 ).permitAll()
                 .requestMatchers(
                     "/courses/{course}/participants/**",
-                    "/courses/{course}/summary"
+                    "/courses/{course}/summary",
+                    "/pruneSubmissions"
                 ).access { _, context ->
                     AuthorityAuthorizationDecision(isAuthorizedAPIKey(context), parseAuthorities(listOf("supervisor")))
                 }
diff --git a/src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt b/src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt
index 9966618..42f9189 100644
--- a/src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt
+++ b/src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt
@@ -29,6 +29,12 @@ class CourseRootController(
         return courseService.editCourse(courseDTO).slug
     }
 
+    @GetMapping("/pruneSubmissions")
+    fun pruneSubmissions(): String? {
+        courseService.globalPruneSubmissions()
+        return "done"
+    }
+
     @PostMapping("/contact")
     fun sendMessage(@RequestBody contactDTO: ContactDTO?) {
         courseService.sendMessage(contactDTO!!)
diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/Evaluation.kt b/src/main/kotlin/ch/uzh/ifi/access/model/Evaluation.kt
index b1df689..07dbaf6 100644
--- a/src/main/kotlin/ch/uzh/ifi/access/model/Evaluation.kt
+++ b/src/main/kotlin/ch/uzh/ifi/access/model/Evaluation.kt
@@ -43,7 +43,8 @@ class Evaluation {
     fun addSubmission(newSubmission: Submission): Submission {
         submissions.add(newSubmission)
         newSubmission.evaluation = this
-        newSubmission.ordinalNum = countSubmissionsByType(newSubmission.command!!)
+        val latestOfType = submissions.filter { it.command == newSubmission.command }.maxByOrNull { it.ordinalNum ?: 0 }
+        newSubmission.ordinalNum = (latestOfType?.ordinalNum ?: 0) + 1
         return newSubmission
     }
 
diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt
index cc2702f..7f524a0 100644
--- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt
+++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt
@@ -338,6 +338,44 @@ class CourseService(
         submissionRepository.saveAndFlush(submission)
     }
 
+    // only necessary once to prune existing data in already-deployed ACCESS
+    fun globalPruneSubmissions() {
+        evaluationRepository.findAll().forEach { if (it != null) {pruneSubmissions(it) } }
+    }
+
+    fun pruneSubmissions(evaluation: Evaluation) {
+        // keep all GRADE submissions, but prune RUN and TEST submissions
+        listOf(Command.RUN, Command.TEST, Command.GRADE).map { command ->
+            Pair(command, evaluation.submissions.filter { it.command == command }.sortedByDescending { it.ordinalNum })
+        }.filter { it.second.isNotEmpty() }.forEach { (command, submissions) ->
+            val latest = submissions.first().ordinalNum!!
+            // always keep the latest 5
+            val keepOrdinalNums = submissions.take(5).map { it.ordinalNum!! }.toMutableSet()
+            // always keep the (latest) best GRADE submission
+            if (command == Command.GRADE) {
+                val best = submissions.maxWith(compareBy<Submission> { it.points }.thenBy { it.ordinalNum })
+                keepOrdinalNums.add(best.ordinalNum!!)
+            }
+            // keep increasingly fewer submissions the older they are (none older than 100 submissions ago)
+            var low = 5
+            listOf(10, 15, 25, 50, 75, 100).forEach { high ->
+                // find oldest in current low < distance_to_latest <= high bracket
+                val highestOrdinalNum = submissions.lastOrNull {
+                    val distance = latest - it.ordinalNum!!
+                    distance in low..<high
+                }?.ordinalNum
+                if (highestOrdinalNum != null) {
+                    keepOrdinalNums.add(highestOrdinalNum)
+                }
+                low = high
+            }
+            val prune = submissions.filter { !keepOrdinalNums.contains(it.ordinalNum!!) }
+            evaluation.submissions.removeAll(prune)
+            submissionRepository.deleteAll(prune)
+            logger.debug { " -->> Pruning ${evaluation.userId}/${evaluation.task?.assignment?.slug}/${evaluation.task?.slug} [${command}]: ${prune.map{it.ordinalNum}}" }
+        }
+    }
+
     @Caching(evict = [
         CacheEvict(value = ["getStudent"], key = "#courseSlug + '-' + #submissionDTO.userId"),
         CacheEvict(value = ["getStudentWithPoints"], key = "#courseSlug + '-' + #submissionDTO.userId"),
@@ -366,6 +404,7 @@ class CourseService(
             )
         }
         val submission = submissionRepository.saveAndFlush(newSubmission)
+        pruneSubmissions(evaluation)
         submissionDTO.files.stream().filter { fileDTO -> fileDTO.content != null }
             .forEach { fileDTO: SubmissionFileDTO -> createSubmissionFile(submission, fileDTO) }
         submission.valid = !submission.isGraded