Skip to content

Commit

Permalink
Feature/decouple thumbnail downloads and cache (#581)
Browse files Browse the repository at this point in the history
* Rename "DownloadedFilesProvider" to "ChaptersFilesProvider"

* Move files into sub packages

* Further abstract "DownloadedFilesProvider"

* Rename "getCachedImageResponse" to "getImageResponse"

* Extract getting cached image response into new function

* Decouple thumbnail cache and download

* Download and delete permanent thumbnails

When adding/removing manga from/to the library make sure the permanent thumbnail files will get handled properly

* Move thumbnail cache to actual temp folder

* Rename "mangaDownloadsRoot" to "downloadRoot"

* Move manga downloads into "mangas" subfolder

* Clear downloaded thumbnail
  • Loading branch information
schroda committed Aug 12, 2023
1 parent b8b92c8 commit f2dd67d
Show file tree
Hide file tree
Showing 19 changed files with 342 additions and 82 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import suwayomi.tachidesk.graphql.types.MangaMetaType
import suwayomi.tachidesk.graphql.types.MangaType
import suwayomi.tachidesk.manga.impl.Library
import suwayomi.tachidesk.manga.impl.Manga
import suwayomi.tachidesk.manga.model.table.MangaMetaTable
import suwayomi.tachidesk.manga.model.table.MangaTable
Expand Down Expand Up @@ -44,7 +45,7 @@ class MangaMutation {
val patch: UpdateMangaPatch
)

private fun updateMangas(ids: List<Int>, patch: UpdateMangaPatch) {
private suspend fun updateMangas(ids: List<Int>, patch: UpdateMangaPatch) {
transaction {
if (patch.inLibrary != null) {
MangaTable.update({ MangaTable.id inList ids }) { update ->
Expand All @@ -53,37 +54,47 @@ class MangaMutation {
}
}
}
}.apply {
if (patch.inLibrary != null) {
ids.forEach {
Library.handleMangaThumbnail(it, patch.inLibrary)
}
}
}
}

fun updateManga(input: UpdateMangaInput): UpdateMangaPayload {
fun updateManga(input: UpdateMangaInput): CompletableFuture<UpdateMangaPayload> {
val (clientMutationId, id, patch) = input

updateMangas(listOf(id), patch)
return future {
updateMangas(listOf(id), patch)
}.thenApply {
val manga = transaction {
MangaType(MangaTable.select { MangaTable.id eq id }.first())
}

val manga = transaction {
MangaType(MangaTable.select { MangaTable.id eq id }.first())
UpdateMangaPayload(
clientMutationId = clientMutationId,
manga = manga
)
}

return UpdateMangaPayload(
clientMutationId = clientMutationId,
manga = manga
)
}

fun updateMangas(input: UpdateMangasInput): UpdateMangasPayload {
fun updateMangas(input: UpdateMangasInput): CompletableFuture<UpdateMangasPayload> {
val (clientMutationId, ids, patch) = input

updateMangas(ids, patch)
return future {
updateMangas(ids, patch)
}.thenApply {
val mangas = transaction {
MangaTable.select { MangaTable.id inList ids }.map { MangaType(it) }
}

val mangas = transaction {
MangaTable.select { MangaTable.id inList ids }.map { MangaType(it) }
UpdateMangasPayload(
clientMutationId = clientMutationId,
mangas = mangas
)
}

return UpdateMangasPayload(
clientMutationId = clientMutationId,
mangas = mangas
)
}

data class FetchMangaInput(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package suwayomi.tachidesk.manga.impl

import kotlinx.coroutines.CoroutineScope
import suwayomi.tachidesk.manga.impl.download.ArchiveProvider
import suwayomi.tachidesk.manga.impl.download.DownloadedFilesProvider
import suwayomi.tachidesk.manga.impl.download.FolderProvider
import suwayomi.tachidesk.manga.impl.download.fileProvider.ChaptersFilesProvider
import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.ArchiveProvider
import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.FolderProvider
import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath
import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath
Expand All @@ -13,7 +13,7 @@ import java.io.InputStream

object ChapterDownloadHelper {
fun getImage(mangaId: Int, chapterId: Int, index: Int): Pair<InputStream, String> {
return provider(mangaId, chapterId).getImage(index)
return provider(mangaId, chapterId).getImage().execute(index)
}

fun delete(mangaId: Int, chapterId: Int): Boolean {
Expand All @@ -27,11 +27,11 @@ object ChapterDownloadHelper {
scope: CoroutineScope,
step: suspend (DownloadChapter?, Boolean) -> Unit
): Boolean {
return provider(mangaId, chapterId).download(download, scope, step)
return provider(mangaId, chapterId).download().execute(download, scope, step)
}

// return the appropriate provider based on how the download was saved. For the logic is simple but will evolve when new types of downloads are available
private fun provider(mangaId: Int, chapterId: Int): DownloadedFilesProvider {
private fun provider(mangaId: Int, chapterId: Int): ChaptersFilesProvider {
val chapterFolder = File(getChapterDownloadPath(mangaId, chapterId))
val cbzFile = File(getChapterCbzPath(mangaId, chapterId))
if (cbzFile.exists()) return ArchiveProvider(mangaId, chapterId)
Expand Down
24 changes: 24 additions & 0 deletions server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ package suwayomi.tachidesk.manga.impl
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */

import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.select
Expand All @@ -19,6 +23,8 @@ import suwayomi.tachidesk.manga.model.table.MangaTable
import java.time.Instant

object Library {
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)

suspend fun addMangaToLibrary(mangaId: Int) {
val manga = getManga(mangaId)
if (!manga.inLibrary) {
Expand All @@ -41,6 +47,8 @@ object Library {
}
}
}
}.apply {
handleMangaThumbnail(mangaId, true)
}
}
}
Expand All @@ -52,6 +60,22 @@ object Library {
MangaTable.update({ MangaTable.id eq manga.id }) {
it[inLibrary] = false
}
}.apply {
handleMangaThumbnail(mangaId, false)
}
}
}

fun handleMangaThumbnail(mangaId: Int, inLibrary: Boolean) {
scope.launch {
try {
if (inLibrary) {
ThumbnailDownloadHelper.download(mangaId)
} else {
ThumbnailDownloadHelper.delete(mangaId)
}
} catch (e: Exception) {
// ignore
}
}
}
Expand Down
35 changes: 26 additions & 9 deletions server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,13 @@ import org.kodein.di.conf.global
import org.kodein.di.instance
import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl
import suwayomi.tachidesk.manga.impl.Source.getSource
import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.MissingThumbnailException
import suwayomi.tachidesk.manga.impl.util.network.await
import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrNull
import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
import suwayomi.tachidesk.manga.impl.util.source.StubSource
import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage
import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
Expand Down Expand Up @@ -126,7 +127,7 @@ object Manga {
if (!sManga.thumbnail_url.isNullOrEmpty() && sManga.thumbnail_url != mangaEntry[MangaTable.thumbnail_url]) {
it[MangaTable.thumbnail_url] = sManga.thumbnail_url
it[MangaTable.thumbnailUrlLastFetched] = Instant.now().epochSecond
clearMangaThumbnailCache(mangaId)
clearThumbnail(mangaId)
}

it[MangaTable.realUrl] = runCatching {
Expand Down Expand Up @@ -232,15 +233,16 @@ object Manga {

private val applicationDirs by DI.global.instance<ApplicationDirs>()
private val network: NetworkHelper by injectLazy()
suspend fun getMangaThumbnail(mangaId: Int): Pair<InputStream, String> {
val cacheSaveDir = applicationDirs.thumbnailsRoot

suspend fun fetchMangaThumbnail(mangaId: Int): Pair<InputStream, String> {
val cacheSaveDir = applicationDirs.tempThumbnailCacheRoot
val fileName = mangaId.toString()

val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
val sourceId = mangaEntry[MangaTable.sourceReference]

return when (val source = getCatalogueSourceOrStub(sourceId)) {
is HttpSource -> getCachedImageResponse(cacheSaveDir, fileName) {
is HttpSource -> getImageResponse(cacheSaveDir, fileName) {
val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
?: if (!mangaEntry[MangaTable.initialized]) {
// initialize then try again
Expand Down Expand Up @@ -272,7 +274,7 @@ object Manga {
imageFile.inputStream() to contentType
}

is StubSource -> getCachedImageResponse(cacheSaveDir, fileName) {
is StubSource -> getImageResponse(cacheSaveDir, fileName) {
val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
?: throw NullPointerException("No thumbnail found")
network.client.newCall(
Expand All @@ -284,10 +286,25 @@ object Manga {
}
}

private fun clearMangaThumbnailCache(mangaId: Int) {
val saveDir = applicationDirs.thumbnailsRoot
suspend fun getMangaThumbnail(mangaId: Int): Pair<InputStream, String> {
val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }

if (mangaEntry[MangaTable.inLibrary]) {
return try {
ThumbnailDownloadHelper.getImage(mangaId)
} catch (_: MissingThumbnailException) {
ThumbnailDownloadHelper.download(mangaId)
ThumbnailDownloadHelper.getImage(mangaId)
}
}

return fetchMangaThumbnail(mangaId)
}

private fun clearThumbnail(mangaId: Int) {
val fileName = mangaId.toString()

clearCachedImage(saveDir, fileName)
clearCachedImage(applicationDirs.tempThumbnailCacheRoot, fileName)
clearCachedImage(applicationDirs.thumbnailDownloadsRoot, fileName)
}
}
4 changes: 2 additions & 2 deletions server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import org.jetbrains.exposed.sql.update
import suwayomi.tachidesk.manga.impl.util.getChapterCachePath
import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
import suwayomi.tachidesk.manga.model.table.ChapterTable
import suwayomi.tachidesk.manga.model.table.MangaTable
Expand Down Expand Up @@ -99,7 +99,7 @@ object Page {
val cacheSaveDir = getChapterCachePath(mangaId, chapterId)

// Note: don't care about invalidating cache because OS cache is not permanent
return getCachedImageResponse(cacheSaveDir, fileName) {
return getImageResponse(cacheSaveDir, fileName) {
source.fetchImage(tachiyomiPage).awaitSingle()
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package suwayomi.tachidesk.manga.impl

import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.ThumbnailFileProvider
import java.io.InputStream

object ThumbnailDownloadHelper {
fun getImage(mangaId: Int): Pair<InputStream, String> {
return provider(mangaId).getImage().execute()
}

fun delete(mangaId: Int): Boolean {
return provider(mangaId).delete()
}

suspend fun download(mangaId: Int): Boolean {
return provider(mangaId).download().execute()
}

// return the appropriate provider based on how the download was saved. For the logic is simple but will evolve when new types of downloads are available
private fun provider(mangaId: Int): ThumbnailFileProvider {
return ThumbnailFileProvider(mangaId)
}
}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package suwayomi.tachidesk.manga.impl.download.fileProvider

import kotlinx.coroutines.CoroutineScope
import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
import java.io.InputStream

/*
* Base class for downloaded chapter files provider, example: Folder, Archive
* */
abstract class ChaptersFilesProvider(val mangaId: Int, val chapterId: Int) : DownloadedFilesProvider {
abstract fun getImageImpl(index: Int): Pair<InputStream, String>

override fun getImage(): RetrieveFile1Args<Int> {
return object : RetrieveFile1Args<Int> {
override fun execute(a: Int): Pair<InputStream, String> {
return getImageImpl(a)
}
}
}

abstract suspend fun downloadImpl(
download: DownloadChapter,
scope: CoroutineScope,
step: suspend (DownloadChapter?, Boolean) -> Unit
): Boolean

override fun download(): FileDownload3Args<DownloadChapter, CoroutineScope, suspend (DownloadChapter?, Boolean) -> Unit> {
return object : FileDownload3Args<DownloadChapter, CoroutineScope, suspend (DownloadChapter?, Boolean) -> Unit> {
override suspend fun execute(
a: DownloadChapter,
b: CoroutineScope,
c: suspend (DownloadChapter?, Boolean) -> Unit
): Boolean {
return downloadImpl(a, b, c)
}
}
}

abstract override fun delete(): Boolean
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package suwayomi.tachidesk.manga.impl.download.fileProvider

interface DownloadedFilesProvider : FileDownloader, FileRetriever {
fun delete(): Boolean
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package suwayomi.tachidesk.manga.impl.download.fileProvider

@FunctionalInterface
interface FileDownload {
suspend fun executeDownload(vararg args: Any): Boolean
}

@FunctionalInterface
interface FileDownload0Args : FileDownload {
suspend fun execute(): Boolean

override suspend fun executeDownload(vararg args: Any): Boolean {
return execute()
}
}

@FunctionalInterface
interface FileDownload3Args<A, B, C> : FileDownload {
suspend fun execute(a: A, b: B, c: C): Boolean

override suspend fun executeDownload(vararg args: Any): Boolean {
return execute(args[0] as A, args[1] as B, args[2] as C)
}
}

@FunctionalInterface
interface FileDownloader {
fun download(): FileDownload
}
Loading

0 comments on commit f2dd67d

Please sign in to comment.