From f2dd67d87f38c30c8df6f3718ce392197afbff9a Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sat, 12 Aug 2023 17:14:43 +0200 Subject: [PATCH] Feature/decouple thumbnail downloads and cache (#581) * Rename "DownloadedFilesProvider" to "ChaptersFilesProvider" * Move files into sub packages * Further abstract "DownloadedFilesProvider" * Rename "getCachedImageResponse" to "getImageResponse" * Extract getting cached image response into new function * Decouple thumbnail cache and download * Download and delete permanent thumbnails When adding/removing manga from/to the library make sure the permanent thumbnail files will get handled properly * Move thumbnail cache to actual temp folder * Rename "mangaDownloadsRoot" to "downloadRoot" * Move manga downloads into "mangas" subfolder * Clear downloaded thumbnail --- .../graphql/mutations/MangaMutation.kt | 49 +++++++---- .../manga/impl/ChapterDownloadHelper.kt | 12 +-- .../suwayomi/tachidesk/manga/impl/Library.kt | 24 +++++ .../suwayomi/tachidesk/manga/impl/Manga.kt | 35 ++++++-- .../suwayomi/tachidesk/manga/impl/Page.kt | 4 +- .../manga/impl/ThumbnailDownloadHelper.kt | 23 +++++ .../impl/download/DownloadedFilesProvider.kt | 20 ----- .../fileProvider/ChaptersFilesProvider.kt | 40 +++++++++ .../fileProvider/DownloadedFilesProvider.kt | 5 ++ .../download/fileProvider/FileDownloader.kt | 29 +++++++ .../download/fileProvider/FileRetriever.kt | 31 +++++++ .../impl}/ArchiveProvider.kt | 11 +-- .../{ => fileProvider/impl}/FolderProvider.kt | 9 +- .../impl/ThumbnailFileProvider.kt | 87 +++++++++++++++++++ .../manga/impl/extension/Extension.kt | 4 +- .../tachidesk/manga/impl/util/DirName.kt | 8 +- .../manga/impl/util/storage/ImageResponse.kt | 16 ++-- .../suwayomi/tachidesk/server/ServerSetup.kt | 13 +-- .../tachidesk/test/ApplicationTest.kt | 4 +- 19 files changed, 342 insertions(+), 82 deletions(-) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ThumbnailDownloadHelper.kt delete mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/DownloadedFilesProvider.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/FileDownloader.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/FileRetriever.kt rename server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/{ => fileProvider/impl}/ArchiveProvider.kt (87%) rename server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/{ => fileProvider/impl}/FolderProvider.kt (90%) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ThumbnailFileProvider.kt diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt index 569180cb4..8c6f21b38 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt @@ -8,6 +8,7 @@ import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.graphql.types.MangaMetaType import suwayomi.tachidesk.graphql.types.MangaType +import suwayomi.tachidesk.manga.impl.Library import suwayomi.tachidesk.manga.impl.Manga import suwayomi.tachidesk.manga.model.table.MangaMetaTable import suwayomi.tachidesk.manga.model.table.MangaTable @@ -44,7 +45,7 @@ class MangaMutation { val patch: UpdateMangaPatch ) - private fun updateMangas(ids: List, patch: UpdateMangaPatch) { + private suspend fun updateMangas(ids: List, patch: UpdateMangaPatch) { transaction { if (patch.inLibrary != null) { MangaTable.update({ MangaTable.id inList ids }) { update -> @@ -53,37 +54,47 @@ class MangaMutation { } } } + }.apply { + if (patch.inLibrary != null) { + ids.forEach { + Library.handleMangaThumbnail(it, patch.inLibrary) + } + } } } - fun updateManga(input: UpdateMangaInput): UpdateMangaPayload { + fun updateManga(input: UpdateMangaInput): CompletableFuture { val (clientMutationId, id, patch) = input - updateMangas(listOf(id), patch) + return future { + updateMangas(listOf(id), patch) + }.thenApply { + val manga = transaction { + MangaType(MangaTable.select { MangaTable.id eq id }.first()) + } - val manga = transaction { - MangaType(MangaTable.select { MangaTable.id eq id }.first()) + UpdateMangaPayload( + clientMutationId = clientMutationId, + manga = manga + ) } - - return UpdateMangaPayload( - clientMutationId = clientMutationId, - manga = manga - ) } - fun updateMangas(input: UpdateMangasInput): UpdateMangasPayload { + fun updateMangas(input: UpdateMangasInput): CompletableFuture { val (clientMutationId, ids, patch) = input - updateMangas(ids, patch) + return future { + updateMangas(ids, patch) + }.thenApply { + val mangas = transaction { + MangaTable.select { MangaTable.id inList ids }.map { MangaType(it) } + } - val mangas = transaction { - MangaTable.select { MangaTable.id inList ids }.map { MangaType(it) } + UpdateMangasPayload( + clientMutationId = clientMutationId, + mangas = mangas + ) } - - return UpdateMangasPayload( - clientMutationId = clientMutationId, - mangas = mangas - ) } data class FetchMangaInput( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt index b423c89f2..907a217a5 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt @@ -1,9 +1,9 @@ package suwayomi.tachidesk.manga.impl import kotlinx.coroutines.CoroutineScope -import suwayomi.tachidesk.manga.impl.download.ArchiveProvider -import suwayomi.tachidesk.manga.impl.download.DownloadedFilesProvider -import suwayomi.tachidesk.manga.impl.download.FolderProvider +import suwayomi.tachidesk.manga.impl.download.fileProvider.ChaptersFilesProvider +import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.ArchiveProvider +import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.FolderProvider import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath @@ -13,7 +13,7 @@ import java.io.InputStream object ChapterDownloadHelper { fun getImage(mangaId: Int, chapterId: Int, index: Int): Pair { - return provider(mangaId, chapterId).getImage(index) + return provider(mangaId, chapterId).getImage().execute(index) } fun delete(mangaId: Int, chapterId: Int): Boolean { @@ -27,11 +27,11 @@ object ChapterDownloadHelper { scope: CoroutineScope, step: suspend (DownloadChapter?, Boolean) -> Unit ): Boolean { - return provider(mangaId, chapterId).download(download, scope, step) + return provider(mangaId, chapterId).download().execute(download, scope, step) } // return the appropriate provider based on how the download was saved. For the logic is simple but will evolve when new types of downloads are available - private fun provider(mangaId: Int, chapterId: Int): DownloadedFilesProvider { + private fun provider(mangaId: Int, chapterId: Int): ChaptersFilesProvider { val chapterFolder = File(getChapterDownloadPath(mangaId, chapterId)) val cbzFile = File(getChapterCbzPath(mangaId, chapterId)) if (cbzFile.exists()) return ArchiveProvider(mangaId, chapterId) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt index fc6a68f95..1a3bdb074 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt @@ -7,6 +7,10 @@ package suwayomi.tachidesk.manga.impl * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.select @@ -19,6 +23,8 @@ import suwayomi.tachidesk.manga.model.table.MangaTable import java.time.Instant object Library { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + suspend fun addMangaToLibrary(mangaId: Int) { val manga = getManga(mangaId) if (!manga.inLibrary) { @@ -41,6 +47,8 @@ object Library { } } } + }.apply { + handleMangaThumbnail(mangaId, true) } } } @@ -52,6 +60,22 @@ object Library { MangaTable.update({ MangaTable.id eq manga.id }) { it[inLibrary] = false } + }.apply { + handleMangaThumbnail(mangaId, false) + } + } + } + + fun handleMangaThumbnail(mangaId: Int, inLibrary: Boolean) { + scope.launch { + try { + if (inLibrary) { + ThumbnailDownloadHelper.download(mangaId) + } else { + ThumbnailDownloadHelper.delete(mangaId) + } + } catch (e: Exception) { + // ignore } } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt index fa05127f9..13296f4c0 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt @@ -25,12 +25,13 @@ import org.kodein.di.conf.global import org.kodein.di.instance import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl import suwayomi.tachidesk.manga.impl.Source.getSource +import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.MissingThumbnailException import suwayomi.tachidesk.manga.impl.util.network.await import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrNull import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub import suwayomi.tachidesk.manga.impl.util.source.StubSource import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage -import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse +import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass @@ -126,7 +127,7 @@ object Manga { if (!sManga.thumbnail_url.isNullOrEmpty() && sManga.thumbnail_url != mangaEntry[MangaTable.thumbnail_url]) { it[MangaTable.thumbnail_url] = sManga.thumbnail_url it[MangaTable.thumbnailUrlLastFetched] = Instant.now().epochSecond - clearMangaThumbnailCache(mangaId) + clearThumbnail(mangaId) } it[MangaTable.realUrl] = runCatching { @@ -232,15 +233,16 @@ object Manga { private val applicationDirs by DI.global.instance() private val network: NetworkHelper by injectLazy() - suspend fun getMangaThumbnail(mangaId: Int): Pair { - val cacheSaveDir = applicationDirs.thumbnailsRoot + + suspend fun fetchMangaThumbnail(mangaId: Int): Pair { + val cacheSaveDir = applicationDirs.tempThumbnailCacheRoot val fileName = mangaId.toString() val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() } val sourceId = mangaEntry[MangaTable.sourceReference] return when (val source = getCatalogueSourceOrStub(sourceId)) { - is HttpSource -> getCachedImageResponse(cacheSaveDir, fileName) { + is HttpSource -> getImageResponse(cacheSaveDir, fileName) { val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url] ?: if (!mangaEntry[MangaTable.initialized]) { // initialize then try again @@ -272,7 +274,7 @@ object Manga { imageFile.inputStream() to contentType } - is StubSource -> getCachedImageResponse(cacheSaveDir, fileName) { + is StubSource -> getImageResponse(cacheSaveDir, fileName) { val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url] ?: throw NullPointerException("No thumbnail found") network.client.newCall( @@ -284,10 +286,25 @@ object Manga { } } - private fun clearMangaThumbnailCache(mangaId: Int) { - val saveDir = applicationDirs.thumbnailsRoot + suspend fun getMangaThumbnail(mangaId: Int): Pair { + val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() } + + if (mangaEntry[MangaTable.inLibrary]) { + return try { + ThumbnailDownloadHelper.getImage(mangaId) + } catch (_: MissingThumbnailException) { + ThumbnailDownloadHelper.download(mangaId) + ThumbnailDownloadHelper.getImage(mangaId) + } + } + + return fetchMangaThumbnail(mangaId) + } + + private fun clearThumbnail(mangaId: Int) { val fileName = mangaId.toString() - clearCachedImage(saveDir, fileName) + clearCachedImage(applicationDirs.tempThumbnailCacheRoot, fileName) + clearCachedImage(applicationDirs.thumbnailDownloadsRoot, fileName) } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt index 944f99cb2..27d172486 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt @@ -19,7 +19,7 @@ import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.manga.impl.util.getChapterCachePath import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub -import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse +import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil import suwayomi.tachidesk.manga.model.table.ChapterTable import suwayomi.tachidesk.manga.model.table.MangaTable @@ -99,7 +99,7 @@ object Page { val cacheSaveDir = getChapterCachePath(mangaId, chapterId) // Note: don't care about invalidating cache because OS cache is not permanent - return getCachedImageResponse(cacheSaveDir, fileName) { + return getImageResponse(cacheSaveDir, fileName) { source.fetchImage(tachiyomiPage).awaitSingle() } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ThumbnailDownloadHelper.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ThumbnailDownloadHelper.kt new file mode 100644 index 000000000..7fb040100 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ThumbnailDownloadHelper.kt @@ -0,0 +1,23 @@ +package suwayomi.tachidesk.manga.impl + +import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.ThumbnailFileProvider +import java.io.InputStream + +object ThumbnailDownloadHelper { + fun getImage(mangaId: Int): Pair { + return provider(mangaId).getImage().execute() + } + + fun delete(mangaId: Int): Boolean { + return provider(mangaId).delete() + } + + suspend fun download(mangaId: Int): Boolean { + return provider(mangaId).download().execute() + } + + // return the appropriate provider based on how the download was saved. For the logic is simple but will evolve when new types of downloads are available + private fun provider(mangaId: Int): ThumbnailFileProvider { + return ThumbnailFileProvider(mangaId) + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt deleted file mode 100644 index 47c7db610..000000000 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt +++ /dev/null @@ -1,20 +0,0 @@ -package suwayomi.tachidesk.manga.impl.download - -import kotlinx.coroutines.CoroutineScope -import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter -import java.io.InputStream - -/* -* Base class for downloaded chapter files provider, example: Folder, Archive -* */ -abstract class DownloadedFilesProvider(val mangaId: Int, val chapterId: Int) { - abstract fun getImage(index: Int): Pair - - abstract suspend fun download( - download: DownloadChapter, - scope: CoroutineScope, - step: suspend (DownloadChapter?, Boolean) -> Unit - ): Boolean - - abstract fun delete(): Boolean -} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt new file mode 100644 index 000000000..02a70af99 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt @@ -0,0 +1,40 @@ +package suwayomi.tachidesk.manga.impl.download.fileProvider + +import kotlinx.coroutines.CoroutineScope +import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter +import java.io.InputStream + +/* +* Base class for downloaded chapter files provider, example: Folder, Archive +* */ +abstract class ChaptersFilesProvider(val mangaId: Int, val chapterId: Int) : DownloadedFilesProvider { + abstract fun getImageImpl(index: Int): Pair + + override fun getImage(): RetrieveFile1Args { + return object : RetrieveFile1Args { + override fun execute(a: Int): Pair { + return getImageImpl(a) + } + } + } + + abstract suspend fun downloadImpl( + download: DownloadChapter, + scope: CoroutineScope, + step: suspend (DownloadChapter?, Boolean) -> Unit + ): Boolean + + override fun download(): FileDownload3Args Unit> { + return object : FileDownload3Args Unit> { + override suspend fun execute( + a: DownloadChapter, + b: CoroutineScope, + c: suspend (DownloadChapter?, Boolean) -> Unit + ): Boolean { + return downloadImpl(a, b, c) + } + } + } + + abstract override fun delete(): Boolean +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/DownloadedFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/DownloadedFilesProvider.kt new file mode 100644 index 000000000..0b45f06af --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/DownloadedFilesProvider.kt @@ -0,0 +1,5 @@ +package suwayomi.tachidesk.manga.impl.download.fileProvider + +interface DownloadedFilesProvider : FileDownloader, FileRetriever { + fun delete(): Boolean +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/FileDownloader.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/FileDownloader.kt new file mode 100644 index 000000000..5c46cfaae --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/FileDownloader.kt @@ -0,0 +1,29 @@ +package suwayomi.tachidesk.manga.impl.download.fileProvider + +@FunctionalInterface +interface FileDownload { + suspend fun executeDownload(vararg args: Any): Boolean +} + +@FunctionalInterface +interface FileDownload0Args : FileDownload { + suspend fun execute(): Boolean + + override suspend fun executeDownload(vararg args: Any): Boolean { + return execute() + } +} + +@FunctionalInterface +interface FileDownload3Args : FileDownload { + suspend fun execute(a: A, b: B, c: C): Boolean + + override suspend fun executeDownload(vararg args: Any): Boolean { + return execute(args[0] as A, args[1] as B, args[2] as C) + } +} + +@FunctionalInterface +interface FileDownloader { + fun download(): FileDownload +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/FileRetriever.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/FileRetriever.kt new file mode 100644 index 000000000..bf585f341 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/FileRetriever.kt @@ -0,0 +1,31 @@ +package suwayomi.tachidesk.manga.impl.download.fileProvider + +import java.io.InputStream + +@FunctionalInterface +interface RetrieveFile { + fun executeGetImage(vararg args: Any): Pair +} + +@FunctionalInterface +interface RetrieveFile0Args : RetrieveFile { + fun execute(): Pair + + override fun executeGetImage(vararg args: Any): Pair { + return execute() + } +} + +@FunctionalInterface +interface RetrieveFile1Args : RetrieveFile { + fun execute(a: A): Pair + + override fun executeGetImage(vararg args: Any): Pair { + return execute(args[0] as A) + } +} + +@FunctionalInterface +interface FileRetriever { + fun getImage(): RetrieveFile +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/ArchiveProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ArchiveProvider.kt similarity index 87% rename from server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/ArchiveProvider.kt rename to server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ArchiveProvider.kt index d02d33c13..fd3c60e68 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/ArchiveProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ArchiveProvider.kt @@ -1,4 +1,4 @@ -package suwayomi.tachidesk.manga.impl.download +package suwayomi.tachidesk.manga.impl.download.fileProvider.impl import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -7,14 +7,15 @@ import org.apache.commons.compress.archivers.zip.ZipArchiveEntry import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream import org.apache.commons.compress.archivers.zip.ZipFile +import suwayomi.tachidesk.manga.impl.download.fileProvider.ChaptersFilesProvider import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath import java.io.File import java.io.InputStream -class ArchiveProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(mangaId, chapterId) { - override fun getImage(index: Int): Pair { +class ArchiveProvider(mangaId: Int, chapterId: Int) : ChaptersFilesProvider(mangaId, chapterId) { + override fun getImageImpl(index: Int): Pair { val cbzPath = getChapterCbzPath(mangaId, chapterId) val zipFile = ZipFile(cbzPath) val zipEntry = zipFile.entries.toList().sortedWith(compareBy({ it.name }, { it.name }))[index] @@ -23,7 +24,7 @@ class ArchiveProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(ma return Pair(inputStream.buffered(), "image/$fileType") } - override suspend fun download( + override suspend fun downloadImpl( download: DownloadChapter, scope: CoroutineScope, step: suspend (DownloadChapter?, Boolean) -> Unit @@ -33,7 +34,7 @@ class ArchiveProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(ma val chapterFolder = File(chapterDir) if (outputFile.exists()) handleExistingCbzFile(outputFile, chapterFolder) - FolderProvider(mangaId, chapterId).download(download, scope, step) + FolderProvider(mangaId, chapterId).download().execute(download, scope, step) withContext(Dispatchers.IO) { outputFile.createNewFile() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/FolderProvider.kt similarity index 90% rename from server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt rename to server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/FolderProvider.kt index d0aef8932..7ae570232 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/FolderProvider.kt @@ -1,4 +1,4 @@ -package suwayomi.tachidesk.manga.impl.download +package suwayomi.tachidesk.manga.impl.download.fileProvider.impl import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.FlowPreview @@ -9,6 +9,7 @@ import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample import suwayomi.tachidesk.manga.impl.Page import suwayomi.tachidesk.manga.impl.Page.getPageName +import suwayomi.tachidesk.manga.impl.download.fileProvider.ChaptersFilesProvider import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse @@ -19,8 +20,8 @@ import java.io.InputStream /* * Provides downloaded files when pages were downloaded into folders * */ -class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(mangaId, chapterId) { - override fun getImage(index: Int): Pair { +class FolderProvider(mangaId: Int, chapterId: Int) : ChaptersFilesProvider(mangaId, chapterId) { + override fun getImageImpl(index: Int): Pair { val chapterDir = getChapterDownloadPath(mangaId, chapterId) val folder = File(chapterDir) folder.mkdirs() @@ -30,7 +31,7 @@ class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(man } @OptIn(FlowPreview::class) - override suspend fun download( + override suspend fun downloadImpl( download: DownloadChapter, scope: CoroutineScope, step: suspend (DownloadChapter?, Boolean) -> Unit diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ThumbnailFileProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ThumbnailFileProvider.kt new file mode 100644 index 000000000..a2c2126d5 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ThumbnailFileProvider.kt @@ -0,0 +1,87 @@ +package suwayomi.tachidesk.manga.impl.download.fileProvider.impl + +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance +import suwayomi.tachidesk.manga.impl.Manga +import suwayomi.tachidesk.manga.impl.download.fileProvider.DownloadedFilesProvider +import suwayomi.tachidesk.manga.impl.download.fileProvider.FileDownload0Args +import suwayomi.tachidesk.manga.impl.download.fileProvider.RetrieveFile0Args +import suwayomi.tachidesk.manga.impl.util.getThumbnailDownloadPath +import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse +import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse +import suwayomi.tachidesk.server.ApplicationDirs +import java.io.File +import java.io.InputStream + +class MissingThumbnailException : Exception("No thumbnail found") + +private val applicationDirs by DI.global.instance() + +class ThumbnailFileProvider(val mangaId: Int) : DownloadedFilesProvider { + private fun getFilePath(): String? { + val thumbnailDir = applicationDirs.thumbnailDownloadsRoot + val fileName = mangaId.toString() + return ImageResponse.findFileNameStartingWith(thumbnailDir, fileName) + } + + fun getImageImpl(): Pair { + val filePathWithoutExt = getThumbnailDownloadPath(mangaId) + val filePath = getFilePath() + + if (filePath.isNullOrEmpty()) { + throw MissingThumbnailException() + } + + return getCachedImageResponse(filePath, filePathWithoutExt) + } + + override fun getImage(): RetrieveFile0Args { + return object : RetrieveFile0Args { + override fun execute(): Pair { + return getImageImpl() + } + } + } + + suspend fun downloadImpl(): Boolean { + val isExistingFile = getFilePath() != null + if (isExistingFile) { + return true + } + + Manga.fetchMangaThumbnail(mangaId).first.use { image -> + makeSureDownloadDirExists() + val filePath = getThumbnailDownloadPath(mangaId) + ImageResponse.saveImage(filePath, image) + } + + return true + } + + override fun download(): FileDownload0Args { + return object : FileDownload0Args { + override suspend fun execute(): Boolean { + return downloadImpl() + } + } + } + + override fun delete(): Boolean { + val filePath = getFilePath() + if (filePath.isNullOrEmpty()) { + return true + } + + return File(filePath).delete() + } + + private fun makeSureDownloadDirExists() { + val downloadDirPath = applicationDirs.thumbnailDownloadsRoot + val downloadDir = File(downloadDirPath) + + if (!downloadDir.exists()) { + downloadDir.mkdir() + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt index eb6f5964e..3ca50a4cb 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt @@ -40,7 +40,7 @@ import suwayomi.tachidesk.manga.impl.util.PackageTools.getPackageInfo import suwayomi.tachidesk.manga.impl.util.PackageTools.loadExtensionSources import suwayomi.tachidesk.manga.impl.util.network.await import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource -import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse +import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse import suwayomi.tachidesk.manga.model.table.ExtensionTable import suwayomi.tachidesk.manga.model.table.SourceTable import suwayomi.tachidesk.server.ApplicationDirs @@ -329,7 +329,7 @@ object Extension { val cacheSaveDir = "${applicationDirs.extensionsRoot}/icon" - return getCachedImageResponse(cacheSaveDir, apkName) { + return getImageResponse(cacheSaveDir, apkName) { network.client.newCall( GET(iconUrl) ).await() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt index a9ba78efc..cc15edb5f 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt @@ -44,6 +44,10 @@ private fun getChapterDir(mangaId: Int, chapterId: Int): String { return getMangaDir(mangaId) + "/$chapterDir" } +fun getThumbnailDownloadPath(mangaId: Int): String { + return applicationDirs.thumbnailDownloadsRoot + "/$mangaId" +} + fun getChapterDownloadPath(mangaId: Int, chapterId: Int): String { return applicationDirs.mangaDownloadsRoot + "/" + getChapterDir(mangaId, chapterId) } @@ -66,8 +70,8 @@ fun updateMangaDownloadDir(mangaId: Int, newTitle: String): Boolean { val newMangaDir = SafePath.buildValidFilename(newTitle) - val oldDir = "${applicationDirs.mangaDownloadsRoot}/$sourceDir/$mangaDir" - val newDir = "${applicationDirs.mangaDownloadsRoot}/$sourceDir/$newMangaDir" + val oldDir = "${applicationDirs.downloadsRoot}/$sourceDir/$mangaDir" + val newDir = "${applicationDirs.downloadsRoot}/$sourceDir/$newMangaDir" val oldDirFile = File(oldDir) val newDirFile = File(newDir) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt index c0e7c8c1a..8a41d7a26 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt @@ -30,6 +30,14 @@ object ImageResponse { return null } + fun getCachedImageResponse(cachedFile: String, filePath: String): Pair { + val fileType = cachedFile.substringAfter("$filePath.") + return Pair( + pathToInputStream(cachedFile), + "image/$fileType" + ) + } + /** * Get a cached image response * @@ -38,7 +46,7 @@ object ImageResponse { * @param cacheSavePath where to save the cached image. Caller should decide to use perma cache or temp cache (OS temp dir) * @param fileName what the saved cache file should be named */ - suspend fun getCachedImageResponse( + suspend fun getImageResponse( saveDir: String, fileName: String, fetcher: suspend () -> Response @@ -50,11 +58,7 @@ object ImageResponse { // in case the cached file is a ".tmp" file something went wrong with the previous download, and it has to be downloaded again if (cachedFile != null && !cachedFile.endsWith(".tmp")) { - val fileType = cachedFile.substringAfter("$filePath.") - return Pair( - pathToInputStream(cachedFile), - "image/$fileType" - ) + return getCachedImageResponse(cachedFile, filePath) } val response = fetcher() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt index 2c97ae67e..f504afb32 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt @@ -45,13 +45,16 @@ class ApplicationDirs( ) { val cacheRoot = System.getProperty("java.io.tmpdir") + "/tachidesk" val extensionsRoot = "$dataRoot/extensions" - val thumbnailsRoot = "$dataRoot/thumbnails" - val mangaDownloadsRoot = serverConfig.downloadsPath.ifBlank { "$dataRoot/downloads" } + val downloadsRoot = serverConfig.downloadsPath.ifBlank { "$dataRoot/downloads" } val localMangaRoot = serverConfig.localSourcePath.ifBlank { "$dataRoot/local" } val webUIRoot = "$dataRoot/webUI" val automatedBackupRoot = serverConfig.backupPath.ifBlank { "$dataRoot/backups" } + val tempThumbnailCacheRoot = "$tempRoot/thumbnails" val tempMangaCacheRoot = "$tempRoot/manga-cache" + + val thumbnailDownloadsRoot = "$downloadsRoot/thumbnails" + val mangaDownloadsRoot = "$downloadsRoot/mangas" } val serverConfig: ServerConfig by lazy { GlobalConfigManager.module() } @@ -96,7 +99,7 @@ fun applicationSetup() { logger.debug("Data Root directory is set to: ${applicationDirs.dataRoot}") // Migrate Directories from old versions - File("$ApplicationRootDir/manga-thumbnails").renameTo(applicationDirs.thumbnailsRoot) + File("$ApplicationRootDir/manga-thumbnails").renameTo(applicationDirs.tempThumbnailCacheRoot) File("$ApplicationRootDir/manga-local").renameTo(applicationDirs.localMangaRoot) File("$ApplicationRootDir/anime-thumbnails").delete() @@ -105,8 +108,8 @@ fun applicationSetup() { applicationDirs.dataRoot, applicationDirs.extensionsRoot, applicationDirs.extensionsRoot + "/icon", - applicationDirs.thumbnailsRoot, - applicationDirs.mangaDownloadsRoot, + applicationDirs.tempThumbnailCacheRoot, + applicationDirs.downloadsRoot, applicationDirs.localMangaRoot ).forEach { File(it).mkdirs() diff --git a/server/src/test/kotlin/suwayomi/tachidesk/test/ApplicationTest.kt b/server/src/test/kotlin/suwayomi/tachidesk/test/ApplicationTest.kt index 69171a190..f723e0ff0 100644 --- a/server/src/test/kotlin/suwayomi/tachidesk/test/ApplicationTest.kt +++ b/server/src/test/kotlin/suwayomi/tachidesk/test/ApplicationTest.kt @@ -74,8 +74,8 @@ open class ApplicationTest { applicationDirs.dataRoot, applicationDirs.extensionsRoot, applicationDirs.extensionsRoot + "/icon", - applicationDirs.thumbnailsRoot, - applicationDirs.mangaDownloadsRoot, + applicationDirs.tempThumbnailCacheRoot, + applicationDirs.downloadsRoot, applicationDirs.localMangaRoot ).forEach { File(it).mkdirs()