Feature/improve automated backup

gradle/libs.versions.toml

@@ -128,6 +128,12 @@ twelvemonkeys-imageio-webp = { module = "com.twelvemonkeys.imageio:imageio-webp"
 # Testing
 mockk = "io.mockk:mockk:1.13.2"
 
+# cron scheduler
+cron4j = "it.sauronsoftware.cron4j:cron4j:2.2.5"
+
+# cron-utils
+cronUtils = "com.cronutils:cron-utils:9.2.0"
+
 [plugins]
 # Kotlin
 kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin"}

server/build.gradle.kts

@@ -70,6 +70,10 @@ dependencies {
     implementation("com.graphql-java:graphql-java-extended-scalars:20.0")
 
     testImplementation(libs.mockk)
+
+    implementation(libs.cron4j)
+
+    implementation(libs.cronUtils)
 }
 
 application {

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt

@@ -36,54 +36,50 @@ import suwayomi.tachidesk.manga.model.table.SourceTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
 import suwayomi.tachidesk.server.ApplicationDirs
 import suwayomi.tachidesk.server.serverConfig
+import suwayomi.tachidesk.util.HAScheduler
 import java.io.ByteArrayOutputStream
 import java.io.File
 import java.io.InputStream
 import java.text.SimpleDateFormat
 import java.util.Date
-import java.util.Timer
-import java.util.TimerTask
 import java.util.concurrent.TimeUnit
 import java.util.prefs.Preferences
 import kotlin.time.Duration.Companion.days
 
 object ProtoBackupExport : ProtoBackupBase() {
     private val logger = KotlinLogging.logger { }
     private val applicationDirs by DI.global.instance<ApplicationDirs>()
+    private var backupSchedulerJobId: String = ""
     private const val lastAutomatedBackupKey = "lastAutomatedBackupKey"
     private val preferences = Preferences.userNodeForPackage(ProtoBackupExport::class.java)
 
-    private val backupTimer = Timer()
-    private var currentAutomatedBackupTask: TimerTask? = null
-
     fun scheduleAutomatedBackupTask() {
+        HAScheduler.deschedule(backupSchedulerJobId)
+
         if (!serverConfig.automatedBackups) {
-            currentAutomatedBackupTask?.cancel()
             return
         }
 
-        val minInterval = 1.days
-        val interval = serverConfig.backupInterval.days
-        val backupInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds
-
-        val lastAutomatedBackup = preferences.getLong(lastAutomatedBackupKey, 0)
-        val initialDelay =
-            backupInterval - (System.currentTimeMillis() - lastAutomatedBackup) % backupInterval
+        val task = {
+            cleanupAutomatedBackups()
+            createAutomatedBackup()
+            preferences.putLong(lastAutomatedBackupKey, System.currentTimeMillis())
+        }
 
-        currentAutomatedBackupTask?.cancel()
-        currentAutomatedBackupTask = object : TimerTask() {
-            override fun run() {
-                cleanupAutomatedBackups()
-                createAutomatedBackup()
-                preferences.putLong(lastAutomatedBackupKey, System.currentTimeMillis())
-            }
+        val (hour, minute) = serverConfig.backupTime.split(":").map { it.toInt() }
+        val backupHour = hour.coerceAtLeast(0).coerceAtMost(23)
+        val backupMinute = minute.coerceAtLeast(0).coerceAtMost(59)
+        val backupInterval = serverConfig.backupInterval.days.coerceAtLeast(1.days)
+
+        // trigger last backup in case the server wasn't running on the scheduled time
+        val lastAutomatedBackup = preferences.getLong(lastAutomatedBackupKey, System.currentTimeMillis())
+        val wasPreviousBackupTriggered =
+            (System.currentTimeMillis() - lastAutomatedBackup) < backupInterval.inWholeMilliseconds
+        if (!wasPreviousBackupTriggered) {
+            task()
         }
 
-        backupTimer.scheduleAtFixedRate(
-            currentAutomatedBackupTask,
-            initialDelay,
-            backupInterval
-        )
+        HAScheduler.schedule(task, "$backupMinute $backupHour */${backupInterval.inWholeDays} * *", "backup")
     }
 
     private fun createAutomatedBackup() {
@@ -105,11 +101,15 @@ object ProtoBackupExport : ProtoBackupBase() {
 
             backupFile.outputStream().use { output -> input.copyTo(output) }
         }
-
     }
 
     private fun cleanupAutomatedBackups() {
-        logger.debug { "Cleanup automated backups" }
+        logger.debug { "Cleanup automated backups (ttl= ${serverConfig.backupTTL})" }
+
+        val isCleanupDisabled = serverConfig.backupTTL == 0
+        if (isCleanupDisabled) {
+            return
+        }
 
         val automatedBackupDir = File(applicationDirs.automatedBackupRoot)
         if (!automatedBackupDir.isDirectory) {
@@ -132,7 +132,7 @@ object ProtoBackupExport : ProtoBackupBase() {
 
         val lastAccessTime = file.lastModified()
         val isTTLReached =
-            System.currentTimeMillis() - lastAccessTime >= serverConfig.backupTTL.days.inWholeMilliseconds
+            System.currentTimeMillis() - lastAccessTime >= serverConfig.backupTTL.days.coerceAtLeast(1.days).inWholeMilliseconds
         if (isTTLReached) {
             file.delete()
         }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt

@@ -1,6 +1,8 @@
 package suwayomi.tachidesk.manga.impl.update
 
 import eu.kanade.tachiyomi.source.model.UpdateStrategy
+import it.sauronsoftware.cron4j.Task
+import it.sauronsoftware.cron4j.TaskExecutionContext
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
@@ -24,11 +26,13 @@ import org.kodein.di.instance
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.CategoryManga
 import suwayomi.tachidesk.manga.impl.Chapter
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport
 import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
 import suwayomi.tachidesk.manga.model.dataclass.IncludeInUpdate
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 import suwayomi.tachidesk.manga.model.table.MangaStatus
 import suwayomi.tachidesk.server.serverConfig
+import suwayomi.tachidesk.util.HAScheduler
 import java.util.Date
 import java.util.Timer
 import java.util.TimerTask
@@ -51,41 +55,46 @@ class Updater : IUpdater {
     private val lastAutomatedUpdateKey = "lastAutomatedUpdateKey"
     private val preferences = Preferences.userNodeForPackage(Updater::class.java)
 
-    private val updateTimer = Timer()
-    private var currentUpdateTask: TimerTask? = null
+    private var currentUpdateTaskId = ""
 
     init {
         scheduleUpdateTask()
     }
 
+
+    private fun autoUpdateTask() {
+        val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
+        preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
+
+        if (status.value.running) {
+            logger.debug { "Global update is already in progress" }
+            return
+        }
+
+        logger.info { "Trigger global update (interval= ${serverConfig.globalUpdateInterval}h, lastAutomatedUpdate= ${Date(lastAutomatedUpdate)})" }
+        addCategoriesToUpdateQueue(Category.getCategoryList(), true)
+    }
+
     private fun scheduleUpdateTask() {
+        HAScheduler.deschedule(currentUpdateTaskId)
+
         if (!serverConfig.automaticallyTriggerGlobalUpdate) {
             return
         }
 
         val minInterval = 6.hours
         val interval = serverConfig.globalUpdateInterval.hours
-        val updateInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds
-
-        val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
-        val initialDelay = updateInterval - (System.currentTimeMillis() - lastAutomatedUpdate) % updateInterval
-
-        currentUpdateTask?.cancel()
-        currentUpdateTask = object : TimerTask() {
-            override fun run() {
-                preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
-
-                if (status.value.running) {
-                    logger.debug { "Global update is already in progress, do not trigger global update" }
-                    return
-                }
-
-                logger.info { "Trigger global update (interval= ${serverConfig.globalUpdateInterval}h, lastAutomatedUpdate= ${Date(lastAutomatedUpdate)})" }
-                addCategoriesToUpdateQueue(Category.getCategoryList(), true)
-            }
+        val updateInterval = interval.coerceAtLeast(minInterval)
+        val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, System.currentTimeMillis())
+
+        // trigger update in case the server wasn't running on the scheduled time
+        val wasPreviousUpdateTriggered =
+            (System.currentTimeMillis() - lastAutomatedUpdate) < updateInterval.inWholeMilliseconds
+        if (!wasPreviousUpdateTriggered) {
+            autoUpdateTask()
         }
 
-        updateTimer.scheduleAtFixedRate(currentUpdateTask, initialDelay, updateInterval)
+        HAScheduler.schedule(::autoUpdateTask, "* */${updateInterval.inWholeHours} * * *", "global-update")
     }
 
     private fun getOrCreateUpdateChannelFor(source: String): Channel<UpdateJob> {

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -52,6 +52,7 @@ class ServerConfig(getConfig: () -> Config, moduleName: String = MODULE_NAME) :
 
     // backup
     var backupPath: String by overridableConfig
+    var backupTime: String by overridableConfig
     var backupInterval: Int by overridableConfig
     var automatedBackups: Boolean by overridableConfig
     var backupTTL: Int by overridableConfig

server/src/main/kotlin/suwayomi/tachidesk/util/HAScheduler.kt

@@ -0,0 +1,131 @@
+
+package suwayomi.tachidesk.util
+
+import com.cronutils.model.CronType.CRON4J
+import com.cronutils.model.definition.CronDefinitionBuilder
+import com.cronutils.model.time.ExecutionTime
+import com.cronutils.parser.CronParser
+import it.sauronsoftware.cron4j.Scheduler
+import it.sauronsoftware.cron4j.Task
+import it.sauronsoftware.cron4j.TaskExecutionContext
+import mu.KotlinLogging
+import java.time.ZonedDateTime
+import java.util.PriorityQueue
+import java.util.Timer
+import java.util.TimerTask
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.minutes
+import kotlin.time.Duration.Companion.seconds
+
+val cronParser = CronParser(CronDefinitionBuilder.instanceDefinitionFor(CRON4J))
+
+class HATask(val id: String, val cronExpr: String, val execute: () -> Unit, val name: String?) : Comparable<HATask> {
+    private val executionTime = ExecutionTime.forCron(cronParser.parse(cronExpr))
+
+    fun getLastExecutionTime(): Long {
+        return executionTime.lastExecution(ZonedDateTime.now()).get().toEpochSecond().seconds.inWholeMilliseconds
+    }
+
+    fun getNextExecutionTime(): Long {
+        return executionTime.nextExecution(ZonedDateTime.now()).get().toEpochSecond().seconds.inWholeMilliseconds
+    }
+
+    fun getTimeToNextExecution(): Long {
+        return executionTime.timeToNextExecution(ZonedDateTime.now()).get().toMillis()
+    }
+
+    override fun compareTo(other: HATask): Int {
+        return getTimeToNextExecution().compareTo(other.getTimeToNextExecution())
+    }
+}
+
+/**
+ * The "HAScheduler" ("HibernateAwareScheduler") is a scheduler that recognizes when the system was hibernating/suspended
+ * and triggers tasks that have missed their execution points.
+ */
+object HAScheduler {
+    private val logger = KotlinLogging.logger { }
+
+    private val scheduledTasks = PriorityQueue<HATask>()
+    private val scheduler = Scheduler()
+
+    private val HIBERNATION_THRESHOLD = 10.seconds.inWholeMilliseconds
+    private const val TASK_THRESHOLD = 0.1
+
+    init {
+        scheduleHibernateCheckerTask(1.minutes)
+    }
+
+    private fun scheduleHibernateCheckerTask(interval: Duration) {
+        val timer = Timer()
+        timer.scheduleAtFixedRate(
+            object : TimerTask() {
+                var lastExecutionTime = System.currentTimeMillis()
+
+                override fun run() {
+                    val currentTime = System.currentTimeMillis()
+                    val elapsedTime = currentTime - lastExecutionTime
+                    lastExecutionTime = currentTime
+
+                    val systemWasInHibernation = elapsedTime > interval.inWholeMilliseconds + HIBERNATION_THRESHOLD
+                    if (systemWasInHibernation) {
+                        logger.debug { "System hibernation detected, task was delayed by ${elapsedTime - interval.inWholeMilliseconds}ms" }
+                        scheduledTasks.forEach {
+                            val missedExecution = currentTime - it.getLastExecutionTime() - elapsedTime < 0
+                            val taskInterval = it.getNextExecutionTime() - it.getLastExecutionTime()
+                            // in case the next task execution doesn't take long the missed execution can be ignored to prevent a double execution
+                            val taskThresholdMet = taskInterval * TASK_THRESHOLD > it.getTimeToNextExecution()
+
+                            val triggerTask = missedExecution && taskThresholdMet
+                            if (triggerTask) {
+                                logger.debug { "Task \"${it.name ?: it.id}\" missed its execution, executing now..." }
+                                reschedule(it.id, it.cronExpr)
+                                it.execute()
+                            }
+
+                            // queue is ordered by next execution time, thus, loop can be exited early
+                            if (!missedExecution) {
+                                return@forEach
+                            }
+                        }
+                    }
+                }
+            },
+            interval.inWholeMilliseconds,
+            interval.inWholeMilliseconds
+        )
+    }
+
+    fun schedule(execute: () -> Unit, cronExpr: String, name: String?): String {
+        if (!scheduler.isStarted) {
+            scheduler.start()
+        }
+
+        val taskId = scheduler.schedule(
+            cronExpr,
+            object : Task() {
+                override fun execute(context: TaskExecutionContext?) {
+                    execute()
+                }
+            }
+        )
+
+        scheduledTasks.add(HATask(taskId, cronExpr, execute, name))
+
+        return taskId
+    }
+
+    fun deschedule(taskId: String) {
+        scheduler.deschedule(taskId)
+        scheduledTasks.removeIf { it.id == taskId }
+    }
+
+    fun reschedule(taskId: String, cronExpr: String) {
+        val task = scheduledTasks.find { it.id == taskId } ?: return
+
+        scheduledTasks.remove(task)
+        scheduledTasks.add(HATask(taskId, cronExpr, task.execute, task.name))
+
+        scheduler.reschedule(taskId, cronExpr)
+    }
+}

server/src/main/resources/server-reference.conf

@@ -37,6 +37,7 @@ server.systemTrayEnabled = true
 
 # backup
 server.backupPath = ""
+server.backupTime = "00:00" # range: hour: 0-23, minute: 0-59 - default: "00:00" - time of day at which the automated backup should be triggered
 server.backupInterval = 1 # time in days - range: 1 <= n < ∞ - default: 1 day - interval in which the server will automatically create a backup
 server.automatedBackups = true
-server.backupTTL = 14 # time in days - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted
+server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted

server/src/test/resources/server-reference.conf

@@ -30,6 +30,7 @@ server.electronPath = ""
 
 # backup
 server.backupPath = ""
+server.backupTime = "00:00"
 server.backupInterval = 1
 server.automatedBackups = true
 server.backupTTL = 14