feat: outline error workarounds (#33)

* feat: outline 429 errors indicate retrying
* feat: outline 503 errors indicate retrying max 3 times
* feat: outline 429 and 503 errors delay next import by 1 second

.changeset/perfect-baboons-attack.md

@@ -0,0 +1,5 @@
+---
+"coda-mover": patch
+---
+
+More meaningful indications to users with Outline errors

src/modules/simple-mover/MoverClient.ts

@@ -10,6 +10,7 @@ import {
   ITEM_STATUS_DONE,
   ITEM_STATUS_ERROR,
   ITEM_STATUS_PENDING,
+  ITEM_STATUS_RETRYING,
   ITEM_STATUS_SKIPPED,
   SERVER_IMPORT_RETURN_ISSUES,
   SERVER_RETURN_DOCS,
@@ -23,6 +24,7 @@ import type {
   IItemStatus,
   IItemStatuses,
   IImportLog,
+  IImportLogLevel,
 } from './interfaces'
 
 declare const envs: {
@@ -136,16 +138,25 @@ export class MoverClient implements IClient {
   private reportImportProgress () {
     const importLogs = Object.values(this.itemStatuses).map(item => {
       const isImportLog = item.id.startsWith('import::')
-      const isTrackedStatus = item.status === ITEM_STATUS_ERROR ||
-        item.status === ITEM_STATUS_DONE ||
+      const isStatusSuccess = item.status === ITEM_STATUS_DONE ||
         item.status === ITEM_STATUS_SKIPPED
+      const isStatusError = item.status === ITEM_STATUS_ERROR
+      const isStatusRetrying = item.status === ITEM_STATUS_RETRYING
+      const isTrackedStatus = isStatusError || isStatusSuccess || isStatusRetrying
       const message = item.message
+      let logLevel: IImportLogLevel = 'error'
+
+      if (isStatusRetrying) {
+        logLevel = 'info'
+      } else if (!isStatusError) {
+        logLevel = 'success'
+      }
 
       if (isImportLog && isTrackedStatus && message) {
         return {
           id: item.id,
           name: item.name,
-          level: item.status === ITEM_STATUS_ERROR ? 'error' : 'success',
+          level: logLevel,
           message,
         } satisfies IImportLog
       }

src/modules/simple-mover/events.ts

@@ -27,6 +27,7 @@ export const ITEM_STATUS_IMPORTING = 'importing'
 export const ITEM_STATUS_WAITING = 'waiting'
 export const ITEM_STATUS_SKIPPED = 'skipped'
 export const ITEM_STATUS_ARCHIVING = 'archiving'
+export const ITEM_STATUS_RETRYING = 'retrying'
 export const ITEM_STATUS_CANCELLED = 'cancelled'
 
 export const ItemStatuses = [
@@ -43,4 +44,5 @@ export const ItemStatuses = [
   ITEM_STATUS_ARCHIVING,
   ITEM_STATUS_CONFIRMING,
   ITEM_STATUS_CANCELLED,
+  ITEM_STATUS_RETRYING,
 ] as const

src/modules/simple-mover/interfaces.ts

@@ -92,10 +92,12 @@ export interface IClientHandlers {
   onImportLogs?: (logs: IImportLog[]) => void
 }
 
+export type IImportLogLevel = 'success' | 'error' | 'info'
+
 export interface IImportLog {
   id: string
   name?: string
-  level: 'success' | 'error' | 'info'
+  level: IImportLogLevel
   message: string
 }
 

src/modules/simple-mover/lib/server/helpers.ts

@@ -35,3 +35,9 @@ export const getParentDir = (item: ICodaItem, items: Record<string, ICodaItem>)
 
   return `${codaDocsPath}/${parentDirSubPath}`
 }
+
+export const waitForMsAsync = async (milliseconds: number) => {
+  log.info('[wait]', milliseconds, 'ms')
+
+  return await new Promise(resolve => setTimeout(resolve, milliseconds))
+}

src/modules/simple-mover/transfers/OutlineImporter.ts

@@ -7,6 +7,7 @@ import {
   ITEM_STATUS_IMPORTING,
   ITEM_STATUS_LISTING,
   ITEM_STATUS_PENDING,
+  ITEM_STATUS_RETRYING,
   ITEM_STATUS_VALIDATING,
   ITEM_STATUS_WAITING,
 } from '../events'
@@ -23,24 +24,49 @@ import type {
   IStatus,
 } from '../interfaces'
 import { isAxiosError } from 'axios'
-import { trimSlashes } from '../lib'
+import { trimSlashes, waitForMsAsync } from '../lib'
 import { stat } from 'fs-extra'
 
 const DEFAULT_COLLECTION_NAME = 'Coda'
+const MAX_SERVICE_UNAVAILABLE_RETRIES = 3
 
 export class OutlineImporter implements IImporter {
   private readonly tasks = new TaskEmitter({
     concurrency: 1,
     onItemError: (item, error) => {
-      if (isAxiosError(error)) {
+      const isRequestError = isAxiosError(error)
+      const isRequestError429 = isRequestError && error.response?.status === 429
+      const isRequestError503 = isRequestError && error.response?.status === 503
+      const serviceUnavailableRetryCount = this.serviceUnavailableRetries[item.id!] || 0
+      const shouldRetryOn503Error = isRequestError503 && serviceUnavailableRetryCount < MAX_SERVICE_UNAVAILABLE_RETRIES
+      const shouldRetry = isRequestError429 || shouldRetryOn503Error
+
+      if (isRequestError429) {
+        this.shouldDelayImport = true
+        this.setStatus(item.id!, ITEM_STATUS_RETRYING, 'Retrying, rate limit exceeded')
+      } else if (shouldRetryOn503Error) {
+        this.shouldDelayImport = true
+        this.setStatus(
+          item.id!, ITEM_STATUS_RETRYING,
+          `Retrying (${serviceUnavailableRetryCount + 1}), service unavailable`
+        )
+        this.serviceUnavailableRetries[item.id!] = serviceUnavailableRetryCount + 1
+      } else {
+        this.shouldDelayImport = false
+      }
+
+      if (shouldRetry) {
         this.tasks.add({ ...item, priority: TaskPriority.LOW })
         this.tasks.next()
+
+        return
       }
 
-      this.setStatus(item.id!, ITEM_STATUS_ERROR, error.message)
+      this.setStatus(item.id!, ITEM_STATUS_ERROR, `Should import manually, ${error.message}`)
       this.checkDoneStatus()
     },
     onItemDone: () => {
+      this.shouldDelayImport = false
       this.checkDoneStatus()
     },
   })
@@ -50,6 +76,9 @@ export class OutlineImporter implements IImporter {
   private waitingExports: Array<{ id: string, outlineTreePath: string }> = []
   // coda id => corresponding outline index (for ordering)
   private readonly codaOrderingIndexes: Record<string, number> = {}
+  // item id => number of retries on 503 error service unavailable
+  private readonly serviceUnavailableRetries: Record<string, number> = {}
+  private shouldDelayImport = false
 
   constructor (
     private readonly mover: IMover,
@@ -148,6 +177,8 @@ export class OutlineImporter implements IImporter {
   }
 
   private async importDoc (doc: ICodaDoc) {
+    if (this.shouldDelayImport) await waitForMsAsync(1000)
+
     const outlineTreePath = '/'
     if (this.mover.itemStatuses[doc.id]?.status === ITEM_STATUS_LISTING) {
       this.setStatus(doc.id, ITEM_STATUS_WAITING, `Waiting for listing ${doc.name}`)
@@ -190,6 +221,8 @@ export class OutlineImporter implements IImporter {
   }
 
   private async importPage (page: ICodaPage, outlineTreePath = '/') {
+    if (this.shouldDelayImport) await waitForMsAsync(1000)
+
     if (this.mover.itemStatuses[page.id]?.status !== ITEM_STATUS_DONE) {
       this.setStatus(page.id, ITEM_STATUS_WAITING, `Waiting for syncing ${page.name}`)
       this.waitingExports.push({ id: page.id, outlineTreePath })