Merge pull request #94 from mbret/develop

feat: added refresh metadata for collections
mbret · Mar 12, 2024 · d9fadf0 · d9fadf0
2 parents 56370cb + d81add3
commit d9fadf0
Show file tree

Hide file tree

Showing 21 changed files with 402 additions and 169 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -70,7 +70,7 @@
     "@react-rxjs/utils": "^0.9.7",
     "arg": "^5.0.2",
     "react-markdown": "^9.0.1",
-    "reactjrx": "^1.74.1",
+    "reactjrx": "^1.79.1",
     "remark-gfm": "^4.0.0",
     "sharp": "^0.33.2"
   }

diff --git a/packages/api/src/functions/refreshMetadataCollection/handler.ts b/packages/api/src/functions/refreshMetadataCollection/handler.ts
@@ -4,10 +4,9 @@ import { getNormalizedHeader } from "@libs/utils"
 import schema from "./schema"
 import { InvokeCommand } from "@aws-sdk/client-lambda"
 import { STAGE } from "src/constants"
+import { COLLECTION_METADATA_LOCK_MN } from "@oboku/shared"
 import { lock } from "@libs/supabase/lock"
 
-const LOCK_MAX_DURATION_MN = 5
-
 const lambda: ValidatedEventAPIGatewayProxyEvent<typeof schema> = async (
   event
 ) => {
@@ -28,7 +27,7 @@ const lambda: ValidatedEventAPIGatewayProxyEvent<typeof schema> = async (
 
   const lockId = `metadata-collection_${event.body.collectionId}`
 
-  const { alreadyLocked } = await lock(lockId, LOCK_MAX_DURATION_MN)
+  const { alreadyLocked } = await lock(lockId, COLLECTION_METADATA_LOCK_MN)
 
   if (!alreadyLocked) {
     await client.send(command)

diff --git a/packages/api/src/libs/collections/refreshMetadata.ts b/packages/api/src/libs/collections/refreshMetadata.ts
@@ -29,92 +29,125 @@ export const refreshMetadata = async (
     return
   }
 
-  // we always get updated link data
-  // we take a chance to update the collection from its resource
-  const metadataLink =
-    collection.linkResourceId && collection.linkType
-      ? await pluginFacade.getMetadata({
-          resourceId: collection.linkResourceId,
-          linkType: collection.linkType,
-          credentials
-        })
+  try {
+    await atomicUpdate(db, "obokucollection", collection._id, (old) => {
+      const wasAlreadyInitialized =
+        old.metadataUpdateStatus === "fetching" && old.lastMetadataStartedAt
+
+      if (wasAlreadyInitialized) return old
+
+      return {
+        ...old,
+        metadataUpdateStatus: "fetching" as const,
+        lastMetadataStartedAt: new Date().toISOString()
+      }
+    })
+
+    // we always get updated link data
+    // we take a chance to update the collection from its resource
+    const metadataLink =
+      collection.linkResourceId && collection.linkType
+        ? await pluginFacade.getMetadata({
+            resourceId: collection.linkResourceId,
+            linkType: collection.linkType,
+            credentials
+          })
+        : undefined
+
+    const linkModifiedAt = metadataLink?.modifiedAt
+      ? new Date(metadataLink.modifiedAt)
+      : undefined
+    const collectionMetadataUpdatedAt = collection?.lastMetadataUpdatedAt
+      ? new Date(collection?.lastMetadataUpdatedAt)
       : undefined
 
-  const linkModifiedAt = metadataLink?.modifiedAt
-    ? new Date(metadataLink.modifiedAt)
-    : undefined
-  const collectionMetadataUpdatedAt = collection?.lastMetadataUpdatedAt
-    ? new Date(collection?.lastMetadataUpdatedAt)
-    : undefined
-
-  const isCollectionAlreadyUpdatedFromLink =
-    linkModifiedAt &&
-    collectionMetadataUpdatedAt &&
-    linkModifiedAt.getTime() < collectionMetadataUpdatedAt.getTime()
-
-  /**
-   * @important
-   * In case of soft refresh, we only update if the link is updated
-   * or if there is no link but the collection has not yet fetched metadata.
-   * This soft mode is mostly used during sync.
-   */
-
-  if (soft && isCollectionAlreadyUpdatedFromLink) {
-    Logger.log(`${collection._id} already has metadata, ignoring it!`)
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify({})
+    const isCollectionAlreadyUpdatedFromLink =
+      linkModifiedAt &&
+      collectionMetadataUpdatedAt &&
+      linkModifiedAt.getTime() < collectionMetadataUpdatedAt.getTime()
+
+    /**
+     * @important
+     * In case of soft refresh, we only update if the link is updated
+     * or if there is no link but the collection has not yet fetched metadata.
+     * This soft mode is mostly used during sync.
+     */
+
+    if (soft && isCollectionAlreadyUpdatedFromLink) {
+      Logger.log(`${collection._id} already has metadata, ignoring it!`)
+
+      return {
+        statusCode: 200,
+        body: JSON.stringify({})
+      }
     }
-  }
 
-  if (soft && !metadataLink && collection.lastMetadataUpdatedAt) {
-    Logger.log(
-      `${collection._id} does not have link and is already refreshed, ignoring it!`
-    )
+    if (soft && !metadataLink && collection.lastMetadataUpdatedAt) {
+      Logger.log(
+        `${collection._id} does not have link and is already refreshed, ignoring it!`
+      )
 
-    return {
-      statusCode: 200,
-      body: JSON.stringify({})
+      return {
+        statusCode: 200,
+        body: JSON.stringify({})
+      }
     }
-  }
 
-  const metadataUser = collection.metadata?.find((item) => item.type === "user")
-
-  const directivesFromLink = directives.extractDirectivesFromName(
-    metadataLink?.name ?? ""
-  )
+    const metadataUser = collection.metadata?.find(
+      (item) => item.type === "user"
+    )
 
-  const title = directives.removeDirectiveFromString(
-    metadataLink?.name ?? metadataUser?.title ?? ""
-  )
-  const year = directivesFromLink.year ?? metadataUser?.startYear
+    const directivesFromLink = directives.extractDirectivesFromName(
+      metadataLink?.name ?? ""
+    )
 
-  const updatedMetadataList = await fetchMetadata(
-    { title, year: year ? String(year) : undefined },
-    { withGoogle: true, googleApiKey }
-  )
+    const title = directives.removeDirectiveFromString(
+      metadataLink?.name ?? metadataUser?.title ?? ""
+    )
+    const year = directivesFromLink.year ?? metadataUser?.startYear
 
-  await atomicUpdate(db, "obokucollection", collection._id, (old) => {
-    const persistentMetadataList =
-      old.metadata?.filter((entry) =>
-        (["user"] as CollectionMetadata["type"][]).includes(entry.type)
-      ) ?? []
+    const updatedMetadataList = await fetchMetadata(
+      { title, year: year ? String(year) : undefined },
+      { withGoogle: true, googleApiKey }
+    )
 
-    const linkMetadata: CollectionMetadata = {
-      type: "link",
-      ...old.metadata?.find((item) => item.type === "link"),
-      title: metadataLink?.name
-    }
+    await atomicUpdate(db, "obokucollection", collection._id, (old) => {
+      const persistentMetadataList =
+        old.metadata?.filter((entry) =>
+          (["user"] as CollectionMetadata["type"][]).includes(entry.type)
+        ) ?? []
+
+      const linkMetadata: CollectionMetadata = {
+        type: "link",
+        ...old.metadata?.find((item) => item.type === "link"),
+        title: metadataLink?.name
+      }
+
+      return {
+        ...old,
+        lastMetadataUpdatedAt: new Date().toISOString(),
+        metadataUpdateStatus: "idle",
+        lastMetadataUpdateError: null,
+        metadata: [
+          ...persistentMetadataList,
+          ...updatedMetadataList,
+          linkMetadata
+        ]
+      } satisfies CollectionDocType
+    })
+  } catch (error) {
+    await atomicUpdate(
+      db,
+      "obokucollection",
+      collection._id,
+      (old) =>
+        ({
+          ...old,
+          metadataUpdateStatus: "idle",
+          lastMetadataUpdateError: "unknown"
+        }) satisfies CollectionDocType
+    )
 
-    return {
-      ...old,
-      lastMetadataUpdatedAt: new Date().toISOString(),
-      metadata: [
-        ...persistentMetadataList,
-        ...updatedMetadataList,
-        linkMetadata
-      ]
-    }
-  })
+    throw error
+  }
 }
diff --git a/packages/shared/src/docTypes.ts b/packages/shared/src/docTypes.ts
@@ -117,6 +117,9 @@ export type CollectionDocType = CommonBase & {
   syncAt?: string | null
   createdAt: string
   lastMetadataUpdatedAt?: string
+  lastMetadataStartedAt?: string
+  metadataUpdateStatus?: "fetching" | "idle"
+  lastMetadataUpdateError?: null | string
   type?: "series" | "shelve"
   metadata?: CollectionMetadata[]
 }

diff --git a/packages/shared/src/metadata/index.ts b/packages/shared/src/metadata/index.ts
@@ -47,3 +47,5 @@ export type CollectionMetadata = {
    */
   type: "googleBookApi" | "link" | "user" | "biblioreads" | "comicvine" | "mangaupdates"
 }
+
+export const COLLECTION_METADATA_LOCK_MN = 5
diff --git a/packages/web/src/books/drawer/BookActionsDrawer.tsx b/packages/web/src/books/drawer/BookActionsDrawer.tsx
@@ -16,7 +16,7 @@ import {
 import { useNavigate } from "react-router-dom"
 import { useRemoveDownloadFile } from "../../download/useRemoveDownloadFile"
 import { ROUTES } from "../../constants"
-import { useAtomicUpdateBook, useRefreshBookMetadata } from "../helpers"
+import { useAtomicUpdateBook } from "../helpers"
 import {
   Drawer,
   Divider,
@@ -41,6 +41,7 @@ import {
 import { signal, useLiveRef, useSignalValue } from "reactjrx"
 import { useRemoveHandler } from "./useRemoveHandler"
 import { getMetadataFromBook } from "../getMetadataFromBook"
+import { useRefreshBookMetadata } from "../useRefreshBookMetadata"
 
 type SignalState = {
   openedWith: undefined | string

diff --git a/packages/web/src/books/effects.ts b/packages/web/src/books/effects.ts
@@ -20,9 +20,9 @@ import {
   upsertBookLinkEnd,
   upsertBookLinkEnd$
 } from "./triggers"
-import { useRefreshBookMetadata } from "./helpers"
 import { isDefined, useSubscribeEffect } from "reactjrx"
 import { latestDatabase$ } from "../rxdb/useCreateDatabase"
+import { useRefreshBookMetadata } from "./useRefreshBookMetadata"
 
 const useUpsertBookLinkActionEffect = () => {
   const { db: database } = useDatabase()

diff --git a/packages/web/src/books/helpers.ts b/packages/web/src/books/helpers.ts
@@ -13,14 +13,12 @@ import { AtomicUpdateFunction } from "rxdb"
 import { useLock } from "../common/BlockingBackdrop"
 import { useNetworkState } from "react-use"
 import { useDialogManager } from "../dialog"
-import { useSyncReplicate } from "../rxdb/replication/useSyncReplicate"
-import { catchError, EMPTY, from, map, switchMap } from "rxjs"
+import { from } from "rxjs"
 import { useRemoveBookFromDataSource } from "../plugins/useRemoveBookFromDataSource"
-import { usePluginRefreshMetadata } from "../plugins/usePluginRefreshMetadata"
 import { useMutation } from "reactjrx"
 import { isPluginError } from "../plugins/plugin-front"
-import { httpClient } from "../http/httpClient"
 import { getMetadataFromBook } from "./getMetadataFromBook"
+import { useRefreshBookMetadata } from "./useRefreshBookMetadata"
 
 export const useRemoveBook = () => {
   const removeDownload = useRemoveDownloadFile()
@@ -110,79 +108,6 @@ export const useAtomicUpdateBook = () => {
   return [updater] as [typeof updater]
 }
 
-export const useRefreshBookMetadata = () => {
-  const { db: database } = useDatabase()
-  const [updateBook] = useAtomicUpdateBook()
-  const dialog = useDialogManager()
-  const network = useNetworkState()
-  const { mutateAsync: sync } = useSyncReplicate()
-  const refreshPluginMetadata = usePluginRefreshMetadata()
-
-  return async (bookId: string) => {
-    try {
-      if (!network.online) {
-        return dialog({ preset: "OFFLINE" })
-      }
-
-      const book = await database?.book
-        .findOne({ selector: { _id: bookId } })
-        .exec()
-
-      const firstLink = await database?.link
-        .findOne({ selector: { _id: book?.links[0] } })
-        .exec()
-
-      if (!firstLink) {
-        Report.warn(`Trying to refresh metadata of file item ${bookId}`)
-
-        return
-      }
-
-      const { data: pluginMetadata } = await refreshPluginMetadata({
-        linkType: firstLink.type
-      })
-
-      if (!database) return
-
-      from(
-        updateBook(bookId, (old) => ({
-          ...old,
-          metadataUpdateStatus: "fetching"
-        }))
-      )
-        .pipe(
-          switchMap(() => from(sync([database.link, database.book]))),
-          switchMap(() =>
-            from(httpClient.refreshBookMetadata(bookId, pluginMetadata))
-          ),
-          catchError((e) =>
-            from(
-              updateBook(bookId, (old) => ({
-                ...old,
-                metadataUpdateStatus: null,
-                lastMetadataUpdateError: "unknown"
-              }))
-            ).pipe(
-              map((_) => {
-                throw e
-              })
-            )
-          ),
-          catchError((e) => {
-            Report.error(e)
-
-            return EMPTY
-          })
-        )
-        .subscribe()
-    } catch (e) {
-      if (isPluginError(e) && e.code === "cancelled") return
-
-      Report.error(e)
-    }
-  }
-}
-
 export const useAddCollectionToBook = () => {
   const { db } = useDatabase()