fix: add gateway fetch timeout and signal error when upload has no shards

Author: alanshaw

src/lib/migrations/gateway.ts

@@ -6,19 +6,27 @@ import { LinkIndexer } from 'linkdex'
 import { carCode } from './constants'
 import { Shard } from './api'
 
-export const fetchCAR = async (root: UnknownLink): Promise<Shard> => {
-  const res = await fetch(`https://w3s.link/ipfs/${root}?format=car`)
-  if (!res.ok) throw new Error('failed to get DAG as CAR', { cause: { status: res.status } })
-  const bytes = new Uint8Array(await res.arrayBuffer())
-  // Verify CAR is complete
-  const iterator = await CarBlockIterator.fromBytes(bytes)
-  const index = new LinkIndexer()
-  for await (const block of iterator) {
-    index.decodeAndIndex(block)
-  }
-  if (!index.isCompleteDag()) {
-    throw new Error('CAR does not contain a complete DAG')
+export const fetchCAR = async (root: UnknownLink, options?: { timeout?: number }): Promise<Shard> => {
+  const controller = new AbortController()
+  const timeoutID = setTimeout(() => controller.abort(), options?.timeout ?? 30_000)
+  try {
+    const res = await fetch(`https://w3s.link/ipfs/${root}?format=car`, { signal: controller.signal })
+    if (!res.ok) throw new Error('failed to get DAG as CAR', { cause: { status: res.status } })
+    clearTimeout(timeoutID)
+
+    const bytes = new Uint8Array(await res.arrayBuffer())
+    // Verify CAR is complete
+    const iterator = await CarBlockIterator.fromBytes(bytes)
+    const index = new LinkIndexer()
+    for await (const block of iterator) {
+      index.decodeAndIndex(block)
+    }
+    if (!index.isCompleteDag()) {
+      throw new Error('CAR does not contain a complete DAG')
+    }
+    const link = Link.create(carCode, await sha256.digest(bytes))
+    return { link, size: async () => bytes.length, bytes: async () => bytes }
+  } finally {
+    clearTimeout(timeoutID)
   }
-  const link = Link.create(carCode, await sha256.digest(bytes))
-  return { link, size: async () => bytes.length, bytes: async () => bytes }
 }

src/lib/migrations/index.ts

@@ -104,8 +104,15 @@ export const migrate = async ({
       }
     }
 
+    // signal that this upload failed if it has no shards
+    if (upload.shards.length === 0) {
+      await onError(new Error('upload has no shards'), upload)
+    }
+
     // do no register an upload if not all the shards uploaded successfully
-    if (!allShardsStored) continue
+    if (!allShardsStored) {
+      continue
+    }
 
     try {
       const receipt = await UploadCapabilities.add.invoke({

src/lib/migrations/web3-storage.ts

@@ -76,7 +76,6 @@ class Reader {
 
       // Add a synthetic shard that is the entire DAG.
       // Attempt to download from gateway.
-      // TODO: fetch from /complete?
       if (!shards.length) {
         try {
           const shard = await fetchCAR(root)