Merge pull request #172 from mbret/develop

release

packages/api/src/functions/covers/handler.ts

@@ -11,8 +11,7 @@ const s3 = new S3Client({
 })
 
 const lambda: ValidatedEventAPIGatewayProxyEvent = async (event) => {
-  const coverId = event.pathParameters?.id ?? ``
-  const objectKey = `cover-${coverId}`
+  const objectKey = event.pathParameters?.id ?? ``
   const format = event.queryStringParameters?.format || "image/webp"
 
   const userCover = await getCover(s3, objectKey)

packages/api/src/functions/refreshMetadataCollectionLongProcess/src/refreshMetadata.ts

@@ -4,11 +4,12 @@ import {
   directives
 } from "@oboku/shared"
 import { fetchMetadata } from "./fetchMetadata"
-import { atomicUpdate } from "@libs/couch/dbHelpers"
+import { atomicUpdate, findOne } from "@libs/couch/dbHelpers"
 import nano from "nano"
 import { Logger } from "@libs/logger"
 import { pluginFacade } from "@libs/plugins/facade"
 import { computeMetadata } from "@libs/collections/computeMetadata"
+import { saveOrUpdateCover } from "./saveOrUpdateCover"
 
 export const refreshMetadata = async (
   collection: CollectionDocType,
@@ -108,39 +109,55 @@ export const refreshMetadata = async (
     )
 
     const title = directives.removeDirectiveFromString(
-      linkMetadataInfo?.name ?? userTitle ?? ""
+      directivesFromLink.metadataTitle ??
+        linkMetadataInfo?.name ??
+        userTitle ??
+        ""
     )
+
     const year = directivesFromLink.year ?? userStartYear
 
-    const updatedMetadataList = await fetchMetadata(
+    const externalMetadatas = await fetchMetadata(
       { title, year: year ? String(year) : undefined },
       { withGoogle: true, googleApiKey, comicVineApiKey }
     )
 
-    await atomicUpdate(db, "obokucollection", collection._id, (old) => {
-      const persistentMetadataList =
-        old.metadata?.filter((entry) =>
-          (["user"] as CollectionMetadata["type"][]).includes(entry.type)
-        ) ?? []
-
-      const linkMetadata: CollectionMetadata = {
-        type: "link",
-        ...old.metadata?.find((item) => item.type === "link"),
-        title: linkMetadataInfo?.name
-      }
+    const linkMetadata: CollectionMetadata = {
+      type: "link",
+      ...collection.metadata?.find((item) => item.type === "link"),
+      title: linkMetadataInfo?.name
+    }
 
-      return {
-        ...old,
-        lastMetadataUpdatedAt: new Date().toISOString(),
-        metadataUpdateStatus: "idle",
-        lastMetadataUpdateError: null,
-        metadata: [
-          ...persistentMetadataList,
-          ...updatedMetadataList,
-          linkMetadata
-        ]
-      } satisfies CollectionDocType
+    // try to get latest collection to stay as fresh as possible
+    const currentCollection = await findOne(db, "obokucollection", {
+      selector: { _id: collection._id }
     })
+
+    if (!currentCollection) throw new Error("Unable to find collection")
+
+    const userMetadata =
+      currentCollection.metadata?.filter((entry) => entry.type === "user") ?? []
+    const metadata = [...userMetadata, ...externalMetadatas, linkMetadata]
+
+    // cannot be done in // since metadata status will trigger cover refresh
+    await saveOrUpdateCover(currentCollection, {
+      _id: currentCollection._id,
+      metadata
+    })
+
+    await atomicUpdate(
+      db,
+      "obokucollection",
+      collection._id,
+      (old) =>
+        ({
+          ...old,
+          lastMetadataUpdatedAt: new Date().toISOString(),
+          metadataUpdateStatus: "idle",
+          lastMetadataUpdateError: null,
+          metadata
+        }) satisfies CollectionDocType
+    )
   } catch (error) {
     await atomicUpdate(
       db,

packages/api/src/functions/refreshMetadataCollectionLongProcess/src/saveOrUpdateCover.ts

@@ -0,0 +1,37 @@
+import { isCoverExist } from "@libs/books/covers/isCoverExist"
+import { saveCoverFromExternalLinkToBucket } from "@libs/books/covers/saveCoverFromExternalLinkToBucket"
+import { CollectionDocType, getCollectionCoverKey } from "@oboku/shared"
+
+export const saveOrUpdateCover = async (
+  prevCollection: Pick<CollectionDocType, "_id" | "metadata">,
+  currentCollection: Pick<CollectionDocType, "_id" | "metadata">
+) => {
+  const existingCover = prevCollection.metadata?.find(
+    (metadata) => metadata.cover
+  )?.cover
+  const cover = currentCollection.metadata?.find(
+    (metadata) => metadata.cover
+  )?.cover
+
+  if (!cover) return
+
+  const coverKey = getCollectionCoverKey(currentCollection._id)
+
+  if (
+    existingCover &&
+    cover.uri === existingCover.uri &&
+    (await isCoverExist(coverKey))
+  ) {
+    console.log(`Already have cover ${coverKey} for ${cover.uri}`)
+
+    return
+  }
+
+  try {
+    await saveCoverFromExternalLinkToBucket(coverKey, cover.uri)
+
+    console.log(`Successfully saved cover ${cover.uri} at ${coverKey}`)
+  } catch (e) {
+    console.error(e)
+  }
+}

packages/api/src/functions/refreshMetadataLongProcess/src/updateCover.ts

@@ -4,7 +4,7 @@ import { Extractor } from "node-unrar-js"
 import { saveCoverFromRarArchiveToBucket } from "@libs/books/covers/saveCoverFromRarArchiveToBucket"
 import { Context } from "./types"
 import { saveCoverFromExternalLinkToBucket } from "@libs/books/covers/saveCoverFromExternalLinkToBucket"
-import { isBookCoverExist } from "@libs/books/covers/isBookCoverExist"
+import { isCoverExist } from "@libs/books/covers/isCoverExist"
 
 export const updateCover = async ({
   metadataList,
@@ -29,7 +29,7 @@ export const updateCover = async ({
     metadataForCover?.type === currentMetadaForCover?.type &&
     metadataForCover?.coverLink &&
     metadataForCover.coverLink === currentMetadaForCover?.coverLink &&
-    (await isBookCoverExist(coverObjectKey))
+    (await isCoverExist(coverObjectKey))
   ) {
     console.log(
       `Skipping cover update for ${book._id} since the current and new cover link are equals`
@@ -70,9 +70,10 @@ export const updateCover = async ({
     metadataForCover?.type === "googleBookApi" &&
     metadataForCover.coverLink
   ) {
+    const objectKey = `cover-${ctx.userNameHex}-${ctx.book._id}`
+
     await saveCoverFromExternalLinkToBucket(
-      ctx,
-      ctx.book._id,
+      objectKey,
       metadataForCover.coverLink
     )
 

packages/api/src/libs/books/covers/isBookCoverExist.ts → packages/api/src/libs/books/covers/isCoverExist.ts

@@ -2,7 +2,7 @@ import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3"
 
 const s3 = new S3Client()
 
-export const isBookCoverExist = async (coverObjectKey: string) => {
+export const isCoverExist = async (coverObjectKey: string) => {
   try {
     await s3.send(
       new HeadObjectCommand({

packages/api/src/libs/books/covers/saveCoverFromExternalLinkToBucket.ts

@@ -4,30 +4,21 @@ import { saveCoverFromBufferToBucket } from "./saveCoverFromBufferToBucket"
 
 const logger = Logger.child({ module: "saveCoverFromExternalLinkToBucket" })
 
-type Context = {
-  userNameHex: string
-}
-
 export const saveCoverFromExternalLinkToBucket = async (
-  ctx: Context,
-  bookId: string,
+  coverKey: string,
   coverUrl: string
 ) => {
-  const objectKey = `cover-${ctx.userNameHex}-${bookId}`
-
-  Logger.info(`prepare to save cover ${objectKey}`)
+  Logger.info(`prepare to save cover ${coverKey}`)
 
   try {
-    // @todo request is deprecated, switch to something else
-    // @see https://github.com/request/request/issues/3143
     const response = await axios.get<ArrayBuffer>(coverUrl, {
       responseType: "arraybuffer"
     })
     const entryAsBuffer = Buffer.from(response.data)
 
-    await saveCoverFromBufferToBucket(entryAsBuffer, objectKey)
+    await saveCoverFromBufferToBucket(entryAsBuffer, coverKey)
 
-    Logger.info(`cover ${objectKey} has been saved/updated`)
+    Logger.info(`cover ${coverKey} has been saved/updated`)
   } catch (e) {
     logger.error(e)
   }

packages/api/src/libs/metadata/comicvine/getSeriesMetadata.ts

@@ -1,4 +1,4 @@
-import { CollectionMetadata } from "@oboku/shared"
+  import { CollectionMetadata } from "@oboku/shared"
 import { Logger } from "@libs/logger"
 import { URL } from "url"
 import axios from "axios"

packages/api/src/libs/metadata/mangadex/getSeriesMetadata.ts

@@ -17,6 +17,9 @@ export const getSeriesMetadata = async (metadata: {
     const statisticsResponse = await getStatistics([result.id])
     const statistics = statisticsResponse.data.statistics ?? {}
     const seriesStastistics = statistics[result.id]
+    const cover = result.relationships.find(
+      (relationship) => relationship.type === "cover_art"
+    )
 
     if (result) {
       return {
@@ -27,7 +30,12 @@ export const getSeriesMetadata = async (metadata: {
           en: result.attributes.title.en
         },
         // x/10
-        rating: seriesStastistics?.rating?.bayesian
+        rating: seriesStastistics?.rating?.bayesian,
+        ...(cover && {
+          cover: {
+            uri: `https://mangadex.org/covers/${result.id}/${cover.attributes.fileName}`
+          }
+        })
       } satisfies CollectionMetadata
     }
   } catch (e) {

packages/api/src/libs/metadata/mangadex/searchManga.ts

@@ -1,4 +1,5 @@
 import axios from "axios"
+import { parse } from "url"
 
 type Response = {
   result: "ok" | "unknown"
@@ -56,26 +57,51 @@ type Response = {
       availableTranslatedLanguages: string[]
       latestUploadedChapter: string
     }
-    relationships: Array<{
-      id: "string"
-      type: "author" | "artist" | "cover_art" | "manga"
-      related?: "prequel" | "unknown"
-    }>
+    relationships: Array<
+      | {
+          id: string
+          type: "author" | "artist" | "manga"
+          related?: "prequel" | "unknown"
+        }
+      | {
+          id: string
+          type: "cover_art"
+          attributes: {
+            fileName: string
+            description: string
+            volume: string
+            locale: string
+            version: number
+            createdAt: string
+            updatedAt: string
+          }
+        }
+    >
   }>
   limit: number
   offset: number
   total: number
 }
 
 export const searchManga = async (title: string) => {
-  return axios<Response>({
+  const response = await axios<Response>({
     method: "get",
     url: "https://api.mangadex.org/manga",
     headers: {
       "Content-Type": "application/json"
     },
     params: {
-      title
+      title,
+      includes: ["cover_art"],
+      order: {
+        relevance: "desc"
+      }
     }
   })
+
+  const { url, params } = response.config
+
+  console.log(`[mangadex.searchManga] url: ${JSON.stringify({ url, params })}`)
+
+  return response
 }

packages/api/src/libs/sync/books/createOrUpdateBook.ts

@@ -15,7 +15,7 @@ import {
   addTagsToBookIfNotExist,
   createBook
 } from "@libs/couch/dbHelpers"
-import { isBookCoverExist } from "@libs/books/covers/isBookCoverExist"
+import { isCoverExist } from "@libs/books/covers/isCoverExist"
 import { Context } from "../types"
 
 type Helpers = Parameters<NonNullable<DataSourcePlugin["sync"]>>[1]
@@ -225,7 +225,7 @@ export const createOrUpdateBook = async ({
 
       if (
         metadataAreOlderThanModifiedDate ||
-        !(await isBookCoverExist(coverObjectKey))
+        !(await isCoverExist(coverObjectKey))
       ) {
         await helpers
           .refreshBookMetadata({ bookId: existingBook?._id })

packages/shared/src/collections/index.ts

@@ -0,0 +1,3 @@
+export const getCollectionCoverKey = (collectionId: string) => {
+  return `collection-${collectionId}`
+}

packages/shared/src/directives.ts

@@ -21,6 +21,7 @@ export const extractDirectivesFromName = (
   year?: string
   ignoreMetadata?: string | undefined
   isWebtoon: boolean
+  metadataTitle?: string
 } => {
   let isNotACollection = false
   let tags: string[] = []
@@ -30,6 +31,7 @@ export const extractDirectivesFromName = (
   let isbn = undefined
   let series: boolean | undefined = undefined
   let ignoreMetadata: string | undefined = undefined
+  let metadataTitle: string | undefined = undefined
   let isWebtoon: boolean = false
 
   const directives = resourceId
@@ -50,6 +52,11 @@ export const extractDirectivesFromName = (
       ignoreMetadata = value
     }
 
+    if (directive.startsWith("metadata-title~")) {
+      const value = directive.replace(/metadata-title\~/, "")
+      metadataTitle = value
+    }
+
     if (directive === "ignore") {
       isIgnored = true
     }
@@ -92,7 +99,8 @@ export const extractDirectivesFromName = (
     isbn,
     year,
     ignoreMetadata,
-    isWebtoon
+    isWebtoon,
+    metadataTitle
   }
 }
 

packages/shared/src/index.ts

@@ -92,3 +92,4 @@ export * from "./utils/truncate"
 export * from "./utils/intersection"
 export * from "./utils/groupBy"
 export * from "./utils/mergeWith"
+export * from "./collections"

packages/shared/src/metadata/index.ts

@@ -43,6 +43,11 @@ export type CollectionMetadata = {
   startYear?: number
   publisherName?: string
   rating?: number
+  cover?: {
+    uri: string
+    createdAt?: string
+    updatedAt?: string
+  }
   status?: "completed" | "ongoing" | "unknown"
   /**
    * googleBookApi: Metadata scrapped through google book api