feat: add old web3 storage migrator (#129)

<img width="1356" alt="Screenshot 2024-09-25 at 18 34 57"
src="https://github.com/user-attachments/assets/eaefb7b7-9231-4823-a2f2-b31d87d94457">

---------

Co-authored-by: Alan Shaw <[email protected]>

Author: alanshaw

pnpm-lock.yaml

@@ -8,9 +8,10 @@ import { useMigrations } from '@/components/MigrationsProvider'
 import { DidIcon } from '@/components/DidIcon'
 import CopyIcon from '@/components/CopyIcon'
 import { CheckCircleIcon, ClockIcon, FlagIcon } from '@heroicons/react/20/solid'
-import { Migration, MigrationProgress } from '@/lib/migrations/api'
+import { Migration, Progress } from '@/lib/migrations/api'
 import { useRouter } from 'next/navigation'
 import { UnknownLink } from '@w3ui/react'
+import { dataSources } from '@/app/migration/data-sources'
 
 interface PageProps {
   params: {
@@ -49,14 +50,17 @@ export default function MigrationPage ({ params }: PageProps): JSX.Element {
   const migration = migrations.find(m => m.id === params.id)
   if (!migration) return <H1>Migration not found</H1>
 
+  const ds = dataSources.find(({ source }) => source.id === migration.source)
+  if (!ds) return <H1>Unknown data source</H1>
+
   const handleRemove = () => {
     removeMigration(migration.id)
     router.replace('/')
   }
 
   return (
     <div className='max-w-6xl'>
-      <H1>Migrating from {migration.source}</H1>
+      <H1>Migrating from {ds.name}</H1>
       <div className='bg-white my-4 p-4 rounded-2xl border border-hot-red'>
         <div className='flex mb-4'>
           <div className='flex-auto'>
@@ -122,7 +126,7 @@ const LogLines = ({ lines }: { lines: string[] }) => {
   )
 }
 
-const ProgressBar = ({ progress }: { progress?: MigrationProgress }) => {
+const ProgressBar = ({ progress }: { progress?: Progress }) => {
   const attempted = progress ? progress.succeeded + progress.failed.length : 0
   const failed = progress?.failed.length
   const total = progress ? attempted + progress.pending : 0
@@ -142,7 +146,7 @@ const ProgressBar = ({ progress }: { progress?: MigrationProgress }) => {
   )
 }
 
-const RemoveButton = ({ onRemove, progress }: { onRemove: () => void, progress?: MigrationProgress }) => {
+const RemoveButton = ({ onRemove, progress }: { onRemove: () => void, progress?: Progress }) => {
   const [isRemoveConfirmModalOpen, setRemoveConfirmModalOpen] = useState(false)
 
   if (progress && progress.pending <= 0) {

public/web3storage-psa-logo.png

@@ -2,14 +2,13 @@
 
 import { MouseEventHandler, useState } from 'react'
 import { useRouter } from 'next/navigation'
-import { ChevronLeftIcon, ChevronRightIcon } from '@heroicons/react/20/solid'
+import { ArrowPathIcon, ChevronLeftIcon, ChevronRightIcon } from '@heroicons/react/20/solid'
 import { H1, H2 } from '@/components/Text'
 import { useMigrations } from '@/components/MigrationsProvider'
 import { DIDKey, useW3 } from '@w3ui/react'
-import * as NFTStorageMigrator from '@/lib/migrations/nft-storage'
-import * as Web3StorageMigrator from '@/lib/migrations/web3-storage'
 import { DidIcon } from '@/components/DidIcon'
-import { MigrationConfiguration, MigrationSource } from '@/lib/migrations/api'
+import { MigrationConfiguration, DataSourceID } from '@/lib/migrations/api'
+import { dataSources } from '@/app/migration/data-sources'
 
 interface WizardProps {
   config: Partial<MigrationConfiguration>
@@ -46,7 +45,7 @@ export default function CreateMigrationPage (): JSX.Element {
 }
 
 function ChooseSource ({ config, onNext }: WizardProps) {
-  const [source, setSource] = useState<MigrationSource|undefined>(config.source)
+  const [source, setSource] = useState<DataSourceID|undefined>(config.source)
   const handleNextClick: MouseEventHandler = e => {
     e.preventDefault()
     if (!source) return
@@ -58,16 +57,14 @@ function ChooseSource ({ config, onNext }: WizardProps) {
       <H1>Create a new migration</H1>
       <div className='bg-white my-4 p-5 rounded-2xl border border-hot-red font-epilogue'>
         <p className='mb-8'>This tool allows data to be migrated from a previous provider to one of your spaces.</p>
-
         <H2>Where from?</H2>
         <p className='mb-4'>Pick a storage service you want to migrate data from.</p>
-        <div className='mb-4'>
-          <button className={`bg-white/60 rounded-lg shadow-md p-8 hover:outline mr-4 ${source === 'classic.nft.storage' ? 'outline' : ''}`} type='button' onClick={() => setSource('classic.nft.storage')} title='Migrate from NFT.Storage (Classic)'>
-            <img src='/nftstorage-logo.png' width='350' />
-          </button>
-          <button className={`bg-white/60 opacity-60 rounded-lg shadow-md p-8 ${source === 'old.web3.storage' ? 'outline' : ''}`} type='button' onClick={() => setSource('old.web3.storage')} title='COMING SOON! Migrate from Web3.Storage (Old)' disabled={true}>
-            <img src='/web3storage-logo.png' width='350' />
-          </button>
+        <div className='mb-4 text-center'>
+          {dataSources.map(({ name, logo, source: { id } }) => (
+            <button key={id} className={`bg-white/60 rounded-lg shadow-md p-8 border border-black hover:outline ml-4 first:ml-0 mb-4 ${source === id ? 'outline' : ''}`} type='button' onClick={() => setSource(id)} title={`Migrate from ${name}`}>
+              {logo}
+            </button>
+          ))}
         </div>
         <button onClick={handleNextClick} className={`inline-block bg-hot-red border border-hot-red font-epilogue text-white uppercase text-sm px-6 py-2 rounded-full whitespace-nowrap ${source ? 'hover:bg-white hover:text-hot-red' : 'opacity-10'}`} disabled={!source}>
           Next <ChevronRightIcon className='h-5 w-5 inline-block ml-1 align-middle' style={{marginTop: -4}} />
@@ -79,32 +76,33 @@ function ChooseSource ({ config, onNext }: WizardProps) {
 
 function AddSourceToken ({ config, onNext, onPrev }: WizardProps) {
   const [token, setToken] = useState<string|undefined>(config.token)
+  const [checking, setChecking] = useState(false)
   const [error, setError] = useState('')
 
+  const ds = dataSources.find(({ source }) => source.id === config.source)
+  if (!ds) return
+
   const handleNextClick: MouseEventHandler = async e => {
     e.preventDefault()
-    if (!token) return
+    if (!token || checking) return
     setError('')
+    setChecking(true)
 
     try {
-      if (config.source === 'classic.nft.storage') {
-        await NFTStorageMigrator.checkToken(token)
-      } else if (config.source === 'old.web3.storage') {
-        await Web3StorageMigrator.checkToken(token)
-      } else {
-        throw new Error(`unknown data source: ${config.source}`)
-      }
+      await ds.source.checkToken(token)
     } catch (err: any) {
       console.error(err)
       return setError(`Error using token: ${err.message}`)
+    } finally {
+      setChecking(false)
     }
     onNext({ ...config, token })
   }
   return (
     <div className='max-w-4xl'>
       <H1>Add data source token</H1>
       <div className='bg-white my-4 p-5 rounded-2xl border border-hot-red font-epilogue'>
-        <p className='mb-8'>Add your <strong>{config.source}</strong> API token. Note: the key never leaves this device, it is for local use only by the migration tool.</p>
+        <p className='mb-8'>Add your API token for <strong>{ds.name}</strong>. Note: the key never leaves this device, it is for local use only by the migration tool.</p>
         <H2>API Token</H2>
         <div className='max-w-xl mb-4'>
           <input
@@ -120,8 +118,10 @@ function AddSourceToken ({ config, onNext, onPrev }: WizardProps) {
         <button onClick={e => { e.preventDefault(); onPrev() }} className={`inline-block bg-hot-red border border-hot-red font-epilogue text-white uppercase text-sm mr-2 px-6 py-2 rounded-full whitespace-nowrap hover:bg-white hover:text-hot-red`}>
           <ChevronLeftIcon className='h-5 w-5 inline-block mr-1 align-middle' style={{marginTop: -4}}/> Previous
         </button>
-        <button onClick={handleNextClick} className={`inline-block bg-hot-red border border-hot-red font-epilogue text-white uppercase text-sm px-6 py-2 rounded-full whitespace-nowrap ${token ? 'hover:bg-white hover:text-hot-red' : 'opacity-10'}`} disabled={!token}>
-          Next <ChevronRightIcon className='h-5 w-5 inline-block ml-1 align-middle' style={{marginTop: -4}}/>
+        <button onClick={handleNextClick} className={`inline-block bg-hot-red border border-hot-red font-epilogue text-white uppercase text-sm px-6 py-2 rounded-full whitespace-nowrap ${token ? 'hover:bg-white hover:text-hot-red' : 'opacity-10'}`} disabled={!token || checking}>
+          {checking
+            ? <><ArrowPathIcon className={`h-5 w-5 animate-spin inline-block mr-1 align-middle`}/>Checking...</>
+            : <>Next <ChevronRightIcon className='h-5 w-5 inline-block ml-1 align-middle' style={{marginTop: -4}}/></>}
         </button>
       </div>
     </div>
@@ -176,6 +176,9 @@ function Confirmation ({ config, onNext, onPrev }: WizardProps) {
   const space = spaces.find(s => s.did() === config.space)
   if (!space) return
 
+  const ds = dataSources.find(({ source }) => source.id === config.source)
+  if (!ds) return
+
   const handleNextClick: MouseEventHandler = async e => {
     e.preventDefault()
     onNext(config)
@@ -186,8 +189,8 @@ function Confirmation ({ config, onNext, onPrev }: WizardProps) {
       <div className='bg-white my-4 p-5 rounded-2xl border border-hot-red font-epilogue'>
         <p className='mb-8'>Make sure these details are correct before starting the migration.</p>
         <H2>Source</H2>
-        <div className={`bg-white/60 rounded-lg shadow-md p-8 mb-4 inline-block`} title='Web3.Storage (Old)'>
-          <img src={config.source === 'old.web3.storage' ? '/web3storage-logo.png' : '/nftstorage-logo.png'} width='360' />
+        <div className={`bg-white/60 rounded-lg shadow-md p-8 mb-4 inline-block border border-black`} title={ds.name}>
+          {ds.logo}
         </div>
         <H2>Target</H2>
         <div className='max-w-lg border rounded-2xl border-hot-red bg-white mb-4'>

src/app/migration/[id]/page.tsx

@@ -0,0 +1,21 @@
+import * as NFTStorage from '@/lib/migrations/nft-storage'
+import * as Web3Storage from '@/lib/migrations/web3-storage'
+import * as Web3StoragePSA from '@/lib/migrations/web3-storage-psa'
+
+export const dataSources = [
+  {
+    name: 'NFT.Storage (Classic)',
+    logo: <img src='/nftstorage-logo.png' width='350' />,
+    source: NFTStorage
+  },
+  {
+    name: 'Web3.Storage (Old)',
+    logo: <img src='/web3storage-logo.png' width='350' />,
+    source: Web3Storage
+  },
+  {
+    name: 'Web3.Storage Pinning Service API (Old)',
+    logo: <img src='/web3storage-psa-logo.png' width='274' style={{ margin: 'auto 38px' }} />,
+    source: Web3StoragePSA
+  }
+]

src/app/migration/create/page.tsx

@@ -1,7 +1,7 @@
 'use client'
 
 import React, { createContext, useContext, ReactNode, useState, useEffect } from 'react'
-import { Migration, MigrationConfiguration, MigrationID, MigrationProgress, MigrationSource, UploadsSource } from '@/lib/migrations/api'
+import { Migration, MigrationConfiguration, MigrationID, Progress } from '@/lib/migrations/api'
 import { useW3 } from '@w3ui/react'
 import * as Migrations from '@/lib/migrations'
 import { MigrationsStorage } from '@/lib/migrations/store'
@@ -80,15 +80,15 @@ export function Provider ({ children }: ProviderProps): ReactNode {
     const controller = new AbortController()
     runningMigrations[id] = controller
 
-    const uploads = Migrations.create(migration.source, {
+    const uploads = Migrations.createReader(migration.source, {
       token: migration.token,
       cursor: migration.progress?.cursor
     })
     const initProgress = async () => ({
       pending: await uploads.count(),
       succeeded: 0,
       failed: []
-    }) as MigrationProgress
+    }) as Progress
 
     Migrations.migrate({
       signal: controller.signal,
@@ -127,7 +127,7 @@ export function Provider ({ children }: ProviderProps): ReactNode {
       },
       onError: async (err, upload, shard) => {
         console.error(err)
-        log(id, `migration failed ${upload.root}${shard ? ` (shard: ${shard.link})` : ''}: ${err.stack}`)
+        log(id, `failed migration ${upload.root}${shard ? ` (shard: ${shard.link})` : ''}: ${err.stack}`)
         const migration = migrationsStore.read(id)
         migration.progress = migration.progress ?? await initProgress()
         migration.progress.failed.push(upload.root)
@@ -138,6 +138,16 @@ export function Provider ({ children }: ProviderProps): ReactNode {
         }
         migrationsStore.update(migration)
         setMigrations(() => migrationsStore.load())
+      },
+      onComplete: async () => {
+        log(id, 'finished migration')
+        const migration = migrationsStore.read(id)
+        // there will be no progress if there are 0 items to migrate
+        if (!migration.progress) {
+          migration.progress = migration.progress ?? await initProgress()
+          migrationsStore.update(migration)
+          setMigrations(() => migrationsStore.load())
+        }
       }
     })
   }

src/app/migration/data-sources.tsx

@@ -87,7 +87,7 @@ export function SpaceCreatorForm({
     return (
       <div className={className}>
         <div className='max-w-3xl border border-hot-red rounded-2xl'>
-          <SpacePreview did={space.did()} name={space.name} />
+          <SpacePreview did={space.did()} name={space.name} capabilities={['*']} />
         </div>
       </div>
     )

src/components/MigrationsProvider.tsx

@@ -1,26 +1,48 @@
-import { CARLink, DIDKey, Link, UnknownLink } from '@w3ui/react'
+import { CARLink, DIDKey, UnknownLink } from '@w3ui/react'
 
-export type MigrationID = string
+/** A data source for a migration. */
+export interface DataSource {
+  /** Identifier for the migration. */
+  id: DataSourceID
+  /** Checks a token can be used, throws if invalid. */
+  checkToken: (token: string) => Promise<void>
+  /** Creates a new reader instance for this migration source */
+  createReader: (config: DataSourceConfiguration) => Reader
+}
 
-export type MigrationSource = 'classic.nft.storage' | 'old.web3.storage'
+export type DataSourceID = 'classic.nft.storage' | 'old.web3.storage' | 'psa.old.web3.storage'
 
-export interface MigrationSourceConfiguration {
+export interface DataSourceConfiguration {
   /** API token for data source */
   token: string
   /** Cursor for resuming migration. */
   cursor?: string
 }
 
+export interface Reader extends AsyncIterable<Upload> {
+  /** The total number of uploads to be migrated. */
+  count: () => Promise<number>
+}
+
 export interface MigrationConfiguration {
   /** Data source */
-  source: MigrationSource
+  source: DataSourceID
   /** API token for data source */
   token: string
   /** Target space to migrate data to */
   space: DIDKey
 }
 
-export interface MigrationProgress {
+export interface Migration extends MigrationConfiguration {
+  id: MigrationID
+  /** The progress of the migration. */
+  progress?: Progress
+}
+
+/** A opaque string used to identify the migration instance. */
+export type MigrationID = string
+
+export interface Progress {
   /** The current item being migrated. */
   current?: UnknownLink
   /** Cursor for resuming migration. */
@@ -33,12 +55,6 @@ export interface MigrationProgress {
   failed: UnknownLink[]
 }
 
-export interface Migration extends MigrationConfiguration {
-  id: MigrationID
-  /** The progress of the migration. */
-  progress?: MigrationProgress
-}
-
 export interface Shard {
   link: CARLink
   size: () => Promise<number>
@@ -49,5 +65,3 @@ export interface Upload {
   root: UnknownLink
   shards: Shard[]
 }
-
-export interface UploadsSource extends AsyncIterable<Upload> {}

src/components/SpaceCreator.tsx

@@ -0,0 +1 @@
+export const carCode = 0x0202

src/lib/migrations/api.ts

@@ -0,0 +1,32 @@
+import { UnknownLink } from 'multiformats'
+import * as Link from 'multiformats/link'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { CarBlockIterator } from '@ipld/car'
+import { LinkIndexer } from 'linkdex'
+import { carCode } from './constants'
+import { Shard } from './api'
+
+export const fetchCAR = async (root: UnknownLink, options?: { timeout?: number }): Promise<Shard> => {
+  const controller = new AbortController()
+  const timeoutID = setTimeout(() => controller.abort(), options?.timeout ?? 30_000)
+  try {
+    const res = await fetch(`https://w3s.link/ipfs/${root}?format=car`, { signal: controller.signal })
+    if (!res.ok) throw new Error('failed to get DAG as CAR', { cause: { status: res.status } })
+    clearTimeout(timeoutID)
+
+    const bytes = new Uint8Array(await res.arrayBuffer())
+    // Verify CAR is complete
+    const iterator = await CarBlockIterator.fromBytes(bytes)
+    const index = new LinkIndexer()
+    for await (const block of iterator) {
+      index.decodeAndIndex(block)
+    }
+    if (!index.isCompleteDag()) {
+      throw new Error('CAR does not contain a complete DAG')
+    }
+    const link = Link.create(carCode, await sha256.digest(bytes))
+    return { link, size: async () => bytes.length, bytes: async () => bytes }
+  } finally {
+    clearTimeout(timeoutID)
+  }
+}