From 1a1c3b4e057aa0fea0fe9cc870287fdadbb5f158 Mon Sep 17 00:00:00 2001 From: Carol Alexandru Date: Sun, 21 Jan 2024 20:01:24 +0900 Subject: [PATCH] Implement persistent result files --- .../ch/uzh/ifi/access/AccessApplication.kt | 1 - .../uzh/ifi/access/config/SecurityConfig.kt | 2 - .../ch/uzh/ifi/access/model/GlobalFile.kt | 2 +- .../ch/uzh/ifi/access/model/ResultFile.kt | 33 ++++++ .../ch/uzh/ifi/access/model/Submission.kt | 14 +-- .../kotlin/ch/uzh/ifi/access/model/Task.kt | 5 + .../ch/uzh/ifi/access/model/TaskFile.kt | 2 +- .../ch/uzh/ifi/access/model/dao/Results.kt | 2 +- .../access/model/dto/AssignmentProgressDTO.kt | 2 - .../ifi/access/model/dto/CourseProgressDTO.kt | 2 - .../uzh/ifi/access/model/dto/TaskFilesDTO.kt | 3 +- .../access/projections/AssignmentOverview.kt | 3 +- .../access/projections/AssignmentWorkspace.kt | 3 +- .../access/projections/SubmissionSummary.kt | 1 + .../ifi/access/projections/TaskOverview.kt | 3 +- .../access/repository/AssignmentRepository.kt | 1 - .../ifi/access/repository/CourseRepository.kt | 1 - .../access/repository/EvaluationRepository.kt | 1 - .../access/repository/TaskFileRepository.kt | 1 - .../ifi/access/repository/TaskRepository.kt | 1 - .../access/service/CourseConfigImporter.kt | 2 +- .../uzh/ifi/access/service/CourseLifecycle.kt | 26 +++-- .../uzh/ifi/access/service/CourseService.kt | 105 +++++++++++++++--- .../ch/uzh/ifi/access/service/FileService.kt | 40 +++++-- .../ch/uzh/ifi/access/service/RoleService.kt | 5 - .../V2_0__add_persistent_result_files.sql | 28 +++++ .../kotlin/ch/uzh/ifi/access/TestSuite.kt | 3 +- .../uzh/ifi/access/service/PublicAPITests.kt | 3 +- 28 files changed, 219 insertions(+), 76 deletions(-) create mode 100644 src/main/kotlin/ch/uzh/ifi/access/model/ResultFile.kt create mode 100644 src/main/resources/db/migration/V2_0__add_persistent_result_files.sql diff --git a/src/main/kotlin/ch/uzh/ifi/access/AccessApplication.kt b/src/main/kotlin/ch/uzh/ifi/access/AccessApplication.kt index 2384ea0..f38c7e6 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/AccessApplication.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/AccessApplication.kt @@ -3,7 +3,6 @@ package ch.uzh.ifi.access import org.springframework.boot.SpringApplication import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication -import org.springframework.cache.annotation.EnableCaching @SpringBootApplication diff --git a/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt b/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt index eb99269..b02b9e9 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt @@ -6,7 +6,6 @@ import io.github.oshai.kotlinlogging.KotlinLogging import lombok.AllArgsConstructor import org.keycloak.admin.client.Keycloak import org.keycloak.admin.client.resource.RealmResource -import org.springframework.boot.web.servlet.ServletListenerRegistrationBean import org.springframework.context.ApplicationListener import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -24,7 +23,6 @@ import org.springframework.security.oauth2.jwt.Jwt import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken import org.springframework.security.web.SecurityFilterChain import org.springframework.security.web.access.intercept.RequestAuthorizationContext -import org.springframework.security.web.session.HttpSessionEventPublisher import org.springframework.stereotype.Component import org.springframework.web.filter.CommonsRequestLoggingFilter import java.nio.file.Path diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/GlobalFile.kt b/src/main/kotlin/ch/uzh/ifi/access/model/GlobalFile.kt index 71451c6..1d059dc 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/GlobalFile.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/GlobalFile.kt @@ -26,7 +26,7 @@ class GlobalFile { @Column(nullable = false, columnDefinition = "text") var template: String? = null - @Column(nullable=true, name="template_binary", columnDefinition="bytea") + @Column(nullable=true, columnDefinition="bytea") var templateBinary: ByteArray? = null val binary: Boolean diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/ResultFile.kt b/src/main/kotlin/ch/uzh/ifi/access/model/ResultFile.kt new file mode 100644 index 0000000..56bb17c --- /dev/null +++ b/src/main/kotlin/ch/uzh/ifi/access/model/ResultFile.kt @@ -0,0 +1,33 @@ +package ch.uzh.ifi.access.model + +import com.fasterxml.jackson.annotation.JsonIgnore +import jakarta.persistence.* + + +@Entity +class ResultFile { + @Id + @GeneratedValue + var id: Long? = null + + @Column(nullable = false) + var path: String? = null + + @JsonIgnore + @ManyToOne + @JoinColumn(nullable = false, name = "submission_id") + var submission: Submission? = null + + @Column(nullable = false) + var mimeType: String? = null + + @Column(nullable=true, columnDefinition="text") + var content: String? = null + + @Column(nullable=true, columnDefinition="bytea") + var contentBinary: ByteArray? = null + + val binary: Boolean + get() = contentBinary != null + +} \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/Submission.kt b/src/main/kotlin/ch/uzh/ifi/access/model/Submission.kt index 635ba77..a6755cf 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/Submission.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/Submission.kt @@ -1,7 +1,6 @@ package ch.uzh.ifi.access.model import ch.uzh.ifi.access.model.constants.Command -import ch.uzh.ifi.access.model.dao.Results import com.fasterxml.jackson.annotation.JsonIgnore import jakarta.persistence.* import lombok.Getter @@ -9,7 +8,6 @@ import lombok.Setter import org.apache.commons.lang3.StringUtils import org.hibernate.annotations.CreationTimestamp import java.time.LocalDateTime -import java.util.* @Getter @Setter @@ -45,6 +43,9 @@ class Submission { @OneToMany(mappedBy = "submission", cascade = [CascadeType.ALL]) var files: MutableList = ArrayList() + @OneToMany(mappedBy = "submission", cascade = [CascadeType.ALL]) + var persistentResultFiles: MutableList = ArrayList() + @JsonIgnore @ManyToOne @JoinColumn(nullable = false, name = "evaluation_id") @@ -56,13 +57,4 @@ class Submission { val isGraded: Boolean get() = command!!.isGraded - fun parseResults(results: Results) { - output = results.hints?.filterNotNull()?.firstOrNull() - if (results.points != null) { - valid = true - // never go over 100%; the number of points is otherwise up to the test suite to determine correctly - points = minOf(results.points!!, maxPoints!!) - evaluation!!.update(points) - } - } } diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt b/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt index 714b84c..079833b 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt @@ -2,6 +2,8 @@ package ch.uzh.ifi.access.model import ch.uzh.ifi.access.model.constants.Command import jakarta.persistence.* +import org.hibernate.annotations.JdbcTypeCode +import org.hibernate.type.SqlTypes import java.time.Duration import java.util.* @@ -55,6 +57,9 @@ class Task { @OneToMany(mappedBy = "task", cascade = [CascadeType.ALL]) var files: MutableList = ArrayList() + @JdbcTypeCode(SqlTypes.JSON) + var persistentResultFilePaths: MutableList = ArrayList() + @OneToMany(mappedBy = "task", cascade = [CascadeType.ALL]) var evaluations: MutableList = ArrayList() diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/TaskFile.kt b/src/main/kotlin/ch/uzh/ifi/access/model/TaskFile.kt index dd020f8..e50db8f 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/TaskFile.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/TaskFile.kt @@ -26,7 +26,7 @@ class TaskFile { @Column(nullable=true, columnDefinition="text") var template: String? = null - @Column(nullable=true, name="template_binary", columnDefinition="bytea") + @Column(nullable=true, columnDefinition="bytea") var templateBinary: ByteArray? = null val binary: Boolean diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dao/Results.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dao/Results.kt index c959794..831b901 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dao/Results.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dao/Results.kt @@ -5,5 +5,5 @@ import lombok.Data @Data class Results( var points: Double? = null, - var hints: List? = null + var hints: MutableList = mutableListOf() ) \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssignmentProgressDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssignmentProgressDTO.kt index 21adb4a..af2bb82 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssignmentProgressDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssignmentProgressDTO.kt @@ -1,7 +1,5 @@ package ch.uzh.ifi.access.model.dto -import ch.uzh.ifi.access.model.AssignmentInformation -import ch.uzh.ifi.access.projections.AssignmentInformationPublic import com.fasterxml.jackson.annotation.JsonInclude @JsonInclude(JsonInclude.Include.NON_EMPTY) diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/CourseProgressDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/CourseProgressDTO.kt index 1a79d3a..61081b0 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/CourseProgressDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/CourseProgressDTO.kt @@ -1,7 +1,5 @@ package ch.uzh.ifi.access.model.dto -import ch.uzh.ifi.access.projections.CourseInformationPublic - class CourseProgressDTO ( val userId: String? = null, diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskFilesDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskFilesDTO.kt index 178c3e4..86d0869 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskFilesDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskFilesDTO.kt @@ -8,7 +8,8 @@ class TaskFilesDTO( var visible: List = ArrayList(), var editable: List = ArrayList(), var grading: List = ArrayList(), - var solution: List = ArrayList() + var solution: List = ArrayList(), + var persist: List = ArrayList() ) class CourseFilesDTO : TaskFilesDTO() \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/projections/AssignmentOverview.kt b/src/main/kotlin/ch/uzh/ifi/access/projections/AssignmentOverview.kt index c53e182..b4f3adc 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/projections/AssignmentOverview.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/projections/AssignmentOverview.kt @@ -1,6 +1,7 @@ package ch.uzh.ifi.access.projections -import ch.uzh.ifi.access.model.* +import ch.uzh.ifi.access.model.Assignment +import ch.uzh.ifi.access.model.AssignmentInformation import ch.uzh.ifi.access.model.dao.Timer import org.springframework.beans.factory.annotation.Value import org.springframework.data.rest.core.config.Projection diff --git a/src/main/kotlin/ch/uzh/ifi/access/projections/AssignmentWorkspace.kt b/src/main/kotlin/ch/uzh/ifi/access/projections/AssignmentWorkspace.kt index 09763da..78e42fe 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/projections/AssignmentWorkspace.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/projections/AssignmentWorkspace.kt @@ -1,6 +1,7 @@ package ch.uzh.ifi.access.projections -import ch.uzh.ifi.access.model.* +import ch.uzh.ifi.access.model.Assignment +import ch.uzh.ifi.access.model.AssignmentInformation import ch.uzh.ifi.access.model.dao.Timer import com.fasterxml.jackson.annotation.JsonFormat import org.springframework.beans.factory.annotation.Value diff --git a/src/main/kotlin/ch/uzh/ifi/access/projections/SubmissionSummary.kt b/src/main/kotlin/ch/uzh/ifi/access/projections/SubmissionSummary.kt index 32a2f40..401ce7f 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/projections/SubmissionSummary.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/projections/SubmissionSummary.kt @@ -15,5 +15,6 @@ interface SubmissionSummary { val output: String? val createdAt: LocalDateTime? val files: List? + val persistentResultFiles: List? } diff --git a/src/main/kotlin/ch/uzh/ifi/access/projections/TaskOverview.kt b/src/main/kotlin/ch/uzh/ifi/access/projections/TaskOverview.kt index a144c6c..e07e281 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/projections/TaskOverview.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/projections/TaskOverview.kt @@ -1,6 +1,7 @@ package ch.uzh.ifi.access.projections -import ch.uzh.ifi.access.model.* +import ch.uzh.ifi.access.model.Task +import ch.uzh.ifi.access.model.TaskInformation import org.springframework.beans.factory.annotation.Value import org.springframework.data.rest.core.config.Projection diff --git a/src/main/kotlin/ch/uzh/ifi/access/repository/AssignmentRepository.kt b/src/main/kotlin/ch/uzh/ifi/access/repository/AssignmentRepository.kt index d5816a3..b71fdf4 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/repository/AssignmentRepository.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/repository/AssignmentRepository.kt @@ -5,7 +5,6 @@ import ch.uzh.ifi.access.projections.AssignmentWorkspace import org.springframework.data.jpa.repository.JpaRepository import org.springframework.security.access.prepost.PostAuthorize import org.springframework.security.access.prepost.PostFilter -import java.util.* interface AssignmentRepository : JpaRepository { @PostFilter("filterObject.enabled and (hasRole(#courseSlug + '-assistant') or (hasRole(#courseSlug) and filterObject.isPublished) or hasRole(#courseSlug + '-supervisor'))") diff --git a/src/main/kotlin/ch/uzh/ifi/access/repository/CourseRepository.kt b/src/main/kotlin/ch/uzh/ifi/access/repository/CourseRepository.kt index decbd4f..547ee14 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/repository/CourseRepository.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/repository/CourseRepository.kt @@ -8,7 +8,6 @@ import ch.uzh.ifi.access.projections.MemberOverview import org.springframework.data.jpa.repository.JpaRepository import org.springframework.data.jpa.repository.Query import org.springframework.security.access.prepost.PostFilter -import java.util.* interface CourseRepository : JpaRepository { fun getBySlug(courseSlug: String?): Course? diff --git a/src/main/kotlin/ch/uzh/ifi/access/repository/EvaluationRepository.kt b/src/main/kotlin/ch/uzh/ifi/access/repository/EvaluationRepository.kt index 78e644a..4736b89 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/repository/EvaluationRepository.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/repository/EvaluationRepository.kt @@ -5,7 +5,6 @@ import ch.uzh.ifi.access.model.dao.Rank import ch.uzh.ifi.access.projections.EvaluationSummary import org.springframework.data.jpa.repository.JpaRepository import org.springframework.data.jpa.repository.Query -import java.util.* interface EvaluationRepository : JpaRepository { diff --git a/src/main/kotlin/ch/uzh/ifi/access/repository/TaskFileRepository.kt b/src/main/kotlin/ch/uzh/ifi/access/repository/TaskFileRepository.kt index da936b8..cbbb0d0 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/repository/TaskFileRepository.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/repository/TaskFileRepository.kt @@ -4,7 +4,6 @@ import ch.uzh.ifi.access.model.TaskFile import jakarta.transaction.Transactional import org.springframework.data.jpa.repository.JpaRepository import org.springframework.security.access.prepost.PostFilter -import java.util.* interface TaskFileRepository : JpaRepository { @Transactional diff --git a/src/main/kotlin/ch/uzh/ifi/access/repository/TaskRepository.kt b/src/main/kotlin/ch/uzh/ifi/access/repository/TaskRepository.kt index 5dd8e38..cab3207 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/repository/TaskRepository.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/repository/TaskRepository.kt @@ -4,7 +4,6 @@ import ch.uzh.ifi.access.model.Task import ch.uzh.ifi.access.projections.TaskWorkspace import org.springframework.data.jpa.repository.JpaRepository import org.springframework.security.access.prepost.PostAuthorize -import java.util.* interface TaskRepository : JpaRepository { // TODO: visibility based on date diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt index 9491196..96ffa87 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt @@ -4,7 +4,6 @@ import ch.uzh.ifi.access.model.dto.* import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.NullNode import com.fasterxml.jackson.dataformat.toml.TomlMapper -import org.apache.commons.compress.utils.FileNameUtils import org.springframework.stereotype.Service import java.nio.file.Files import java.nio.file.Path @@ -129,6 +128,7 @@ class CourseConfigImporter( "editable" -> files.editable = filenames "grading" -> files.grading = filenames "solution" -> files.solution = filenames + "persist" -> files.persist = filenames } } task.files = files diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseLifecycle.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseLifecycle.kt index bc86e2c..f30a8ee 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseLifecycle.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseLifecycle.kt @@ -27,6 +27,7 @@ class CourseLifecycle( private val workingDir: Path, private val roleService: RoleService, private val courseRepository: CourseRepository, +// private val persistentResultFilePathRepository: PersistentResultFilePathRepository, private val modelMapper: ModelMapper, private val dockerClient: DockerClient, private val cci: CourseConfigImporter, @@ -153,6 +154,11 @@ class CourseLifecycle( taskDTO.files?.solution?.forEach { filePath -> createOrUpdateTaskFile( task, taskPath, filePath ).solution = true } + // update persistent file paths + task.persistentResultFilePaths.clear() + taskDTO.files?.persist?.forEach { path -> + task.persistentResultFilePaths.add(path) + } } //assignment.setMaxPoints(assignment.getTasks().stream().filter(Task::enabled).mapToDouble(Task::getMaxPoints).sum()); //assignment.maxPoints = assignment.tasks.map { it.maxPoints!! }.sum() // TODO: safety @@ -169,12 +175,11 @@ class CourseLifecycle( .filter { existing: TaskFile -> existing.path == rootedFilePath }.findFirst() .orElseGet { task.createFile() } val taskFilePath = parentPath.resolve(unrootedFilePath) - val taskFileDTO = fileService.storeFile(taskFilePath, TaskFileDTO()) - modelMapper.map(taskFileDTO, taskFile) - taskFile.name = taskFilePath.fileName.toString() - taskFile.path = rootedFilePath - taskFile.enabled = true - return taskFile + val taskFileUpdated = fileService.storeFile(taskFilePath, taskFile) + taskFileUpdated.name = taskFilePath.fileName.toString() + taskFileUpdated.path = rootedFilePath + taskFileUpdated.enabled = true + return taskFileUpdated } private fun createOrUpdateGlobalFile(course: Course, parentPath: Path, path: String): GlobalFile { @@ -184,11 +189,10 @@ class CourseLifecycle( .filter { existing -> existing.path == rootedFilePath }.findFirst() .orElseGet { course.createFile() } val globalFilePath = parentPath.resolve(unrootedFilePath) - val globalFileDTO = fileService.storeFile(globalFilePath, TaskFileDTO()) - modelMapper.map(globalFileDTO, globalFile) - globalFile.name = globalFilePath.fileName.toString() - globalFile.path = rootedFilePath - globalFile.enabled = true + val globalFileUpdated = fileService.storeFile(globalFilePath, globalFile) + globalFileUpdated.name = globalFilePath.fileName.toString() + globalFileUpdated.path = rootedFilePath + globalFileUpdated.enabled = true return globalFile } diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt index dc8a4a6..1b3d28b 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt @@ -19,10 +19,9 @@ import jakarta.transaction.Transactional import jakarta.xml.bind.DatatypeConverter import org.apache.commons.collections4.ListUtils import org.apache.commons.io.FileUtils -import org.apache.logging.log4j.util.Strings +import org.apache.tika.Tika import org.keycloak.representations.idm.UserRepresentation import org.modelmapper.ModelMapper -import org.springframework.beans.factory.annotation.Autowired import org.springframework.cache.annotation.CacheEvict import org.springframework.cache.annotation.Cacheable import org.springframework.cache.annotation.Caching @@ -36,18 +35,14 @@ import java.nio.charset.Charset import java.nio.file.Files import java.nio.file.NoSuchFileException import java.nio.file.Path -import java.security.MessageDigest import java.time.LocalDateTime import java.time.format.DateTimeFormatter -import java.util.* import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.function.Consumer -import java.util.stream.Collectors import java.util.stream.Stream import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec -import kotlin.math.sign @Service class CourseServiceForCaching( @@ -103,6 +98,8 @@ class CourseService( private val jsonMapper: JsonMapper, private val courseLifecycle: CourseLifecycle, private val roleService: RoleService, + private val fileService: FileService, + private val tika: Tika ) { private val logger = KotlinLogging.logger {} @@ -317,9 +314,6 @@ class CourseService( }.joinToString(separator = "\n").replace("\u0000", "") } - private fun readResultsFile(path: Path): Results { - return jsonMapper.readValue(Files.readString(path.resolve("grade_results.json")), Results::class.java) - } private fun createEvaluation(taskId: Long, userId: String): Evaluation { val newEvaluation = getTaskById(taskId).createEvaluation(userId) @@ -405,6 +399,23 @@ class CourseService( val tmpfs: Map = mapOf( "/workspace" to "size=50M", ) + + // TODO: make this size configurable in task config.toml? + val resultFileSizeLimit = convertSizeToBytes("100K") + val persistentFileCopyCommands = task.persistentResultFilePaths.joinToString("\n") { path -> +""" +# Check if results file exceeds permissible size limit +if [[ -f "$path" ]]; then + actual_size=${'$'}(stat -c%s "$path") + if [[ ! ${'$'}actual_size -lt $resultFileSizeLimit ]]; then + exit 202 + fi +fi +file_dir=${'$'}(dirname "$path") +mkdir -p "/submission/${'$'}file_dir" +cp "$path" "/submission/${'$'}file_dir" +""" + } val command = ( """ # copy submitted files to tmpfs @@ -416,13 +427,14 @@ exit_code=${'$'}?; # write results and logs to submission volume /bin/cp /workspace/grade_results.json /submission/; /bin/cp /workspace/logs.txt /submission/; -# check if the tmpfs is full and if so, return 201, otherwise return command status code +# check if the tmpfs is full and if so, return 201 USAGE=${'$'}(df -h | grep /workspace | awk '{print ${'$'}5}' | sed 's/%//') if [ "${'$'}USAGE" -eq 100 ]; then exit 201 -else - exit ${'$'}exit_code; fi +# otherwise check and copy persistent results and return command status code +$persistentFileCopyCommands +exit ${'$'}exit_code; """ ) val container = containerCmd @@ -467,13 +479,13 @@ fi logger.debug { "Submission $submissionDir killed due to timeout" } Results( 0.0, - listOf("Your solution ran out of time. Check for infinite loops and ensure your solution is sufficiently fast even for challenging problem parameters.") + mutableListOf("Your solution ran out of time. Check for infinite loops and ensure your solution is sufficiently fast even for challenging problem parameters.") ) } else { logger.debug { "Submission $submissionDir out of memory" } Results( 0.0, - listOf("Your solution ran out of memory. Make sure you aren't creating gigantic data structures.") + mutableListOf("Your solution ran out of memory. Make sure you aren't creating gigantic data structures.") ) } } @@ -481,22 +493,50 @@ fi 201 -> { Results( 0.0, - listOf("Your solution wrote too much data, either to files, or by printing to the command line. Are you printing in an infinite loop?") + mutableListOf("Your solution wrote too much data, either to files, or by printing to the command line. Are you printing in an infinite loop?") + ) + } + // persistent result file too large + 202 -> { + Results( + 0.0, + mutableListOf("One or more files you're supposed to write exceeds the file size limit of ${bytesToHumanReadable(resultFileSizeLimit)}") ) } // none of the above, hopefully there are grading results else -> { try { - readResultsFile(submissionDir) + jsonMapper.readValue(Files.readString(submissionDir.resolve("grade_results.json")), Results::class.java) } catch (e: NoSuchFileException) { logger.debug { "Submission $submissionDir no grade_results.json" } Results(null, - listOf("No grading results. Please report this as a bug and provide as much detail as possible.") + mutableListOf("No grading results. Please report this as a bug and provide as much detail as possible.") ) } } } - newSubmission.parseResults(results) + val persistentResultFileErrors: MutableList = mutableListOf() + if (results.points != null) { + task.persistentResultFilePaths.forEach { path -> + try { + val resultFile = fileService.storeFile(submissionDir.resolve(path), ResultFile()) + resultFile.path = path + resultFile.submission = newSubmission + newSubmission.persistentResultFiles.add(resultFile) + } catch (e: Exception) { // TODO: are there other specific exceptions to catch? + persistentResultFileErrors.add("A file '$path' should have been created, but wasn't.") + } + } + } + results.hints.addAll(persistentResultFileErrors) + println(results.hints) + newSubmission.output = results.hints.filterNotNull().firstOrNull() + if (results.points != null) { + newSubmission.valid = true + // never go over 100%; the number of points is otherwise up to the test suite to determine correctly + newSubmission.points = minOf(results.points!!, newSubmission.maxPoints!!) + evaluation.update(newSubmission.points) + } } // TODO: move to finally? For the moment, we keep failed submission dirs around for debugging. FileUtils.deleteQuietly(submissionDir.toFile()) @@ -709,4 +749,33 @@ fi println() } + fun convertSizeToBytes(sizeStr: String): Long { + val regex = Regex("""^(\d+)([KMG]?)$""") + val matchResult = regex.matchEntire(sizeStr) + + return matchResult?.let { + val (number, suffix) = it.destructured + val sizeInBytes = when (suffix) { + "K" -> number.toLong() * 1024 + "M" -> number.toLong() * 1024 * 1024 + "G" -> number.toLong() * 1024 * 1024 * 1024 + "" -> number.toLong() + else -> 0 + } + sizeInBytes + } ?: 0 + } + + fun bytesToHumanReadable(sizeInBytes: Long): String { + val kilobyte = 1024.0 + val megabyte = kilobyte * 1024 + val gigabyte = megabyte * 1024 + + return when { + sizeInBytes < kilobyte -> "$sizeInBytes B" + sizeInBytes < megabyte -> String.format("%.2f KB", sizeInBytes / kilobyte) + sizeInBytes < gigabyte -> String.format("%.2f MB", sizeInBytes / megabyte) + else -> String.format("%.2f GB", sizeInBytes / gigabyte) + } + } } \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/FileService.kt b/src/main/kotlin/ch/uzh/ifi/access/service/FileService.kt index a2f5364..6c7e832 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/FileService.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/FileService.kt @@ -1,17 +1,19 @@ package ch.uzh.ifi.access.service +import ch.uzh.ifi.access.model.GlobalFile +import ch.uzh.ifi.access.model.ResultFile +import ch.uzh.ifi.access.model.TaskFile import ch.uzh.ifi.access.model.dto.TaskFileDTO +import org.apache.commons.codec.binary.Base64 import org.apache.commons.compress.utils.FileNameUtils import org.apache.tika.Tika +import org.apache.tika.config.TikaConfig import org.apache.tika.metadata.Metadata -import org.apache.tika.mime.MimeTypes import org.springframework.stereotype.Service import java.io.BufferedInputStream import java.nio.file.Files import java.nio.file.Path import java.util.* -import org.apache.commons.codec.binary.Base64 -import org.apache.tika.config.TikaConfig class FileData( var path: String? = null, @@ -57,25 +59,45 @@ class FileService(val tika: Tika) { val extension = FileNameUtils.getExtension(path.toString()) if (listOf("py", "r").contains(extension.lowercase(Locale.getDefault()))) { fileData.content = Files.readString(path) + fileData.contentBinary = null return fileData.validated() } // if the mimeType is text, that's what we assume if (mimeType.type == "text") { fileData.content = Files.readString(path) + fileData.contentBinary = null return fileData.validated() } // otherwise we store it as binary anyway fileData.contentBinary = Files.readAllBytes(path) + fileData.content = null return fileData.validated() } - fun storeFile(path: Path, dto: TaskFileDTO): TaskFileDTO { + fun storeFile(path: Path, taskFile: TaskFile): TaskFile { + val fileData = storeFile(path) + taskFile.template = fileData.content + taskFile.templateBinary = fileData.contentBinary + taskFile.path = fileData.path + taskFile.mimeType = fileData.mimeType + return taskFile + } + + fun storeFile(path: Path, globalFile: GlobalFile): GlobalFile { + val fileData = storeFile(path) + globalFile.template = fileData.content + globalFile.templateBinary = fileData.contentBinary + globalFile.path = fileData.path + globalFile.mimeType = fileData.mimeType + return globalFile + } + + fun storeFile(path: Path, resultFile: ResultFile): ResultFile { val fileData = storeFile(path) - dto.template = fileData.content - dto.templateBinary = fileData.contentBinary - dto.path = fileData.path - dto.mimeType = fileData.mimeType - return dto + resultFile.content = fileData.content + resultFile.contentBinary = fileData.contentBinary + resultFile.mimeType = fileData.mimeType + return resultFile } fun readToBase64(path: Path): String { diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/RoleService.kt b/src/main/kotlin/ch/uzh/ifi/access/service/RoleService.kt index 3b7091b..09be570 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/RoleService.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/RoleService.kt @@ -3,21 +3,16 @@ package ch.uzh.ifi.access.service import ch.uzh.ifi.access.model.Course import ch.uzh.ifi.access.model.constants.Role import ch.uzh.ifi.access.model.dto.MemberDTO -import ch.uzh.ifi.access.model.dto.StudentDTO import io.github.oshai.kotlinlogging.KotlinLogging import org.apache.commons.collections4.SetUtils -import org.hibernate.Hibernate import org.keycloak.admin.client.resource.RealmResource -import org.keycloak.admin.client.resource.UserResource import org.keycloak.representations.idm.RoleRepresentation import org.keycloak.representations.idm.RoleRepresentation.Composites import org.keycloak.representations.idm.UserRepresentation -import org.springframework.cache.annotation.CacheEvict import org.springframework.cache.annotation.Cacheable import org.springframework.security.core.Authentication import org.springframework.security.core.context.SecurityContextHolder import org.springframework.stereotype.Service -import java.time.LocalDate import java.time.LocalDateTime import java.util.* diff --git a/src/main/resources/db/migration/V2_0__add_persistent_result_files.sql b/src/main/resources/db/migration/V2_0__add_persistent_result_files.sql new file mode 100644 index 0000000..fa761af --- /dev/null +++ b/src/main/resources/db/migration/V2_0__add_persistent_result_files.sql @@ -0,0 +1,28 @@ +create table result_file ( + id bigint not null, + path varchar(255) not null, + submission_id bigint not null, + mime_type varchar(255) not null, + content text, + content_binary bytea, + primary key (id) +); + +alter table result_file + add constraint either_binary_or_not check ( + (content is not null and content_binary is null) or + (content is null and content_binary is not null) +); + +create sequence result_file_seq start with 1 increment by 50; + +alter table result_file add constraint FKfiecoo6Nu7gaidengae1chae2 foreign key (submission_id) references submission; + +alter table task + add column persistent_result_file_paths JSON; + +update task set persistent_result_file_paths = '[]'::json; + +alter table task + alter column persistent_result_file_paths set not null; + diff --git a/src/test/kotlin/ch/uzh/ifi/access/TestSuite.kt b/src/test/kotlin/ch/uzh/ifi/access/TestSuite.kt index ccc5239..4d8a61c 100644 --- a/src/test/kotlin/ch/uzh/ifi/access/TestSuite.kt +++ b/src/test/kotlin/ch/uzh/ifi/access/TestSuite.kt @@ -1,6 +1,7 @@ package ch.uzh.ifi.access -import ch.uzh.ifi.access.service.* +import ch.uzh.ifi.access.service.CourseLifecycleTests +import ch.uzh.ifi.access.service.PublicAPITests import org.junit.jupiter.api.ClassOrderer import org.junit.jupiter.api.TestClassOrder import org.junit.platform.suite.api.SelectClasses diff --git a/src/test/kotlin/ch/uzh/ifi/access/service/PublicAPITests.kt b/src/test/kotlin/ch/uzh/ifi/access/service/PublicAPITests.kt index 135ba30..08c5e3b 100644 --- a/src/test/kotlin/ch/uzh/ifi/access/service/PublicAPITests.kt +++ b/src/test/kotlin/ch/uzh/ifi/access/service/PublicAPITests.kt @@ -10,7 +10,8 @@ import org.springframework.security.test.web.servlet.request.SecurityMockMvcRequ import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post -import org.springframework.test.web.servlet.result.MockMvcResultMatchers.* +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status object JsonReference { fun get(filename: String): String {