Merge master into dev (#2975)

* chore: Update ckb node assume valid target for rc/v0.112.0.

* chore: update versions and changelogs

* feat: Update Neuron compatibility table

* fix: Fix light client sync tx type. (#2965)

* fix: Fix light client sync tx type.

* feat: update ckb client versions (#2970)

* chore: Update ckb node assume valid target (#2968)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update Neuron compatibility table (#2971)

Co-authored-by: Keith-CY <[email protected]>

* chore: Update ckb node assume valid target for rc/v0.112.0. (#2972)

* chore: update release date

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Keith-CY <[email protected]>
Co-authored-by: 严国宇 <[email protected]>

.ckb-light-version

@@ -1 +1 @@
-v0.3.1
+v0.3.2

CHANGELOG.md

@@ -1,3 +1,34 @@
+# 0.112.0 (2023-12-07)
+
+### CKB Node & Light Client
+
+- [[email protected]](https://github.com/nervosnetwork/ckb/releases/tag/v0.112.1) was released on Nov. 21st, 2023. This version of CKB node is now bundled and preconfigured in Neuron.
+- [CKB Light [email protected]](https://github.com/nervosnetwork/ckb-light-client/releases/tag/v0.3.2) was released on Dec. 5th, 2023. This version of CKB Light Client is now bundled and preconfigured in Neuron
+
+### Assumed valid target
+
+Block before `0x1d46fe5bb62d19a004eadd7ba40564c45620905ab260d8a546a9e4b9d7cc0f85`(at height `11,511,944`) will be skipped in validation.(https://github.com/nervosnetwork/neuron/pull/2968)
+
+---
+
+## New features
+
+- #2810: Support Spore Protocol.(@homura)
+- #2916: Add light client in compatibility table.(@yanguoyu)
+- #2926: Refactor switch between light client mainnet and light client testnet to make it more intuitive.(@yanguoyu)
+
+## Bug fixes
+
+- #2928: Fix condition of compatibility alert.(@yanguoyu)
+- #2935: Fix migration settings.(@yanguoyu)
+- #2945: Fix Nervos DAO withdrawal on light client mode.(@homura)
+- #2944: Fix requests jam when a synced transaction includes cellbase cells.(@yanguoyu)
+- #2965: Fix transaction type recognition on light client mode.(@yanguoyu)
+
+
+**Full Changelog**: https://github.com/nervosnetwork/neuron/compare/v0.111.1...v0.112.0
+
+
 # 0.111.1 (2023-11-08)
 
 ### CKB Node & Light Client

compatible.json

@@ -55,6 +55,18 @@
         "0.103"
       ],
       "light": []
+    },
+    "0.112": {
+      "full": [
+        "0.112",
+        "0.111",
+        "0.110",
+        "0.109"
+      ],
+      "light": [
+        "0.3",
+        "0.2"
+      ]
     }
   }
 }

lerna.json

@@ -1,6 +1,6 @@
 {
   "packages": ["packages/*"],
-  "version": "0.111.1",
+  "version": "0.112.0",
   "npmClient": "yarn",
   "$schema": "node_modules/lerna/schemas/lerna-schema.json"
 }

package.json

@@ -2,7 +2,7 @@
   "name": "neuron",
   "productName": "Neuron",
   "description": "CKB Neuron Wallet",
-  "version": "0.111.1",
+  "version": "0.112.0",
   "private": true,
   "author": {
     "name": "Nervos Core Dev",

packages/neuron-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "neuron-ui",
-  "version": "0.111.1",
+  "version": "0.112.0",
   "private": true,
   "author": {
     "name": "Nervos Core Dev",

packages/neuron-wallet/.env

@@ -117,4 +117,4 @@ DAO_CODE_HASH=0x82d76d1b75fe2fd9a27dfbaa65a039221a380d76c926f378d3f81cf3e7e13f2e
 MULTISIG_CODE_HASH=0x5c5069eb0857efc65e1bca0c07df34c31663b3622fd3876c876320fc9634e2a8
 
 # CKB NODE OPTIONS
-CKB_NODE_ASSUME_VALID_TARGET='0x79cecdd6f41361e2474290224751284312a018528d1d92f4e18dd6d542feddfe'
+CKB_NODE_ASSUME_VALID_TARGET='0x1d46fe5bb62d19a004eadd7ba40564c45620905ab260d8a546a9e4b9d7cc0f85'

packages/neuron-wallet/package.json

@@ -3,7 +3,7 @@
   "productName": "Neuron",
   "description": "CKB Neuron Wallet",
   "homepage": "https://www.nervos.org/",
-  "version": "0.111.1",
+  "version": "0.112.0",
   "private": true,
   "author": {
     "name": "Nervos Core Dev",
@@ -97,7 +97,7 @@
     "electron-builder": "23.6.0",
     "electron-devtools-installer": "3.2.0",
     "jest-when": "3.5.2",
-    "neuron-ui": "0.111.1",
+    "neuron-ui": "0.112.0",
     "typescript": "5.3.2"
   }
 }

packages/neuron-wallet/src/block-sync-renderer/sync/connector.ts

@@ -1,5 +1,13 @@
-import { SyncAddressType } from '../../database/chain/entities/sync-progress'
 import { Subject } from 'rxjs'
+import { queue, QueueObject } from 'async'
+import { Indexer as CkbIndexer, CellCollector } from '@ckb-lumos/ckb-indexer'
+import { QueryOptions } from '@ckb-lumos/base'
+import AddressMeta from '../../database/address/meta'
+import { Address } from '../../models/address'
+import { SyncAddressType } from '../../database/chain/entities/sync-progress'
+import IndexerCacheService from './indexer-cache-service'
+import logger from '../../utils/logger'
+import IndexerTxHashCache from '../../database/chain/entities/indexer-tx-hash-cache'
 
 export interface BlockTips {
   cacheTipNumber: number
@@ -41,15 +49,146 @@ export interface AppendScript {
   scriptType: CKBRPC.ScriptType
 }
 
-export abstract class Connector<TransactionsSubjectParam = unknown> {
-  abstract blockTipsSubject: Subject<BlockTips>
-  abstract transactionsSubject: Subject<{ txHashes: CKBComponents.Hash[]; params: TransactionsSubjectParam }>
+export abstract class Connector {
+  public readonly blockTipsSubject: Subject<BlockTips> = new Subject<BlockTips>()
+  public readonly transactionsSubject = new Subject<{ txHashes: CKBComponents.Hash[]; params: string }>()
+  protected indexer: CkbIndexer
+  protected processNextBlockNumberQueue: QueueObject<void>
+  protected processingBlockNumber?: string
+  protected addressesByWalletId: Map<string, AddressMeta[]> = new Map()
+  protected pollingIndexer: boolean = false
+  private indexerQueryQueue: QueueObject<LumosCellQuery> | undefined
 
   abstract connect(): Promise<void>
-  abstract notifyCurrentBlockNumberProcessed(param: TransactionsSubjectParam): void
-  abstract stop(): void
-  abstract getLiveCellsByScript(query: LumosCellQuery): Promise<unknown>
+  abstract processTxsInNextBlockNumber(): Promise<void>
+  protected abstract upsertTxHashes(): Promise<unknown>
+  public abstract notifyCurrentBlockNumberProcessed(blockNumber: string): Promise<void>
   async appendScript(_scripts: AppendScript[]) {
     // do nothing
   }
+
+  constructor({ addresses, nodeUrl, indexerUrl }: { addresses: Address[]; nodeUrl: string; indexerUrl: string }) {
+    this.indexer = new CkbIndexer(nodeUrl, indexerUrl)
+    this.addressesByWalletId = addresses
+      .map(address => AddressMeta.fromObject(address))
+      .reduce((addressesByWalletId, addressMeta) => {
+        if (!addressesByWalletId.has(addressMeta.walletId)) {
+          addressesByWalletId.set(addressMeta.walletId, [])
+        }
+
+        const addressMetas = addressesByWalletId.get(addressMeta.walletId)
+        addressMetas!.push(addressMeta)
+
+        return addressesByWalletId
+      }, new Map<string, AddressMeta[]>())
+
+    this.processNextBlockNumberQueue = queue(async () => this.processTxsInNextBlockNumber(), 1)
+    this.processNextBlockNumberQueue.error((err: any) => {
+      logger.error(`Connector: \tError in processing next block number queue: ${err}`)
+    })
+
+    this.indexerQueryQueue = queue(async (query: any) => {
+      return await this.collectLiveCellsByScript(query)
+    })
+  }
+
+  public stop(): void {
+    this.pollingIndexer = false
+  }
+
+  protected async processNextBlockNumber() {
+    // the processNextBlockNumberQueue is a queue to ensure that ONLY one
+    // block processing task runs at a time to avoid the data conflict while syncing
+    this.processNextBlockNumberQueue?.push()
+    await this.processNextBlockNumberQueue?.drain()
+  }
+
+  protected async getTxHashesWithNextUnprocessedBlockNumber(): Promise<[string | undefined, string[]]> {
+    const txHashCachesByNextBlockNumberAndAddress = await Promise.all(
+      [...this.addressesByWalletId.keys()].map(async walletId =>
+        IndexerCacheService.nextUnprocessedTxsGroupedByBlockNumber(walletId)
+      )
+    )
+    const groupedTxHashCaches = txHashCachesByNextBlockNumberAndAddress.flat().reduce((grouped, txHashCache) => {
+      if (!grouped.get(txHashCache.blockNumber.toString())) {
+        grouped.set(txHashCache.blockNumber.toString(), [])
+      }
+      grouped.get(txHashCache.blockNumber.toString())!.push(txHashCache)
+
+      return grouped
+    }, new Map<string, Array<IndexerTxHashCache>>())
+
+    const nextUnprocessedBlockNumber = [...groupedTxHashCaches.keys()].sort((a, b) => parseInt(a) - parseInt(b)).shift()
+
+    if (!nextUnprocessedBlockNumber) {
+      return [undefined, []]
+    }
+
+    const txHashCachesInNextUnprocessedBlockNumber = groupedTxHashCaches.get(nextUnprocessedBlockNumber)
+
+    return [nextUnprocessedBlockNumber, txHashCachesInNextUnprocessedBlockNumber!.map(({ txHash }) => txHash)]
+  }
+
+  protected async notifyAndSyncNext(indexerTipNumber: number) {
+    const nextUnprocessedBlockNumber = await IndexerCacheService.nextUnprocessedBlock([
+      ...this.addressesByWalletId.keys(),
+    ])
+    if (nextUnprocessedBlockNumber) {
+      this.blockTipsSubject.next({
+        cacheTipNumber: parseInt(nextUnprocessedBlockNumber),
+        indexerTipNumber,
+      })
+      if (!this.processingBlockNumber) {
+        await this.processNextBlockNumber()
+      }
+      return true
+    }
+    this.blockTipsSubject.next({
+      cacheTipNumber: indexerTipNumber,
+      indexerTipNumber,
+    })
+    return false
+  }
+
+  public async getLiveCellsByScript(query: LumosCellQuery) {
+    return new Promise((resolve, reject) => {
+      this.indexerQueryQueue!.push(query, (err: any, result: unknown) => {
+        if (err) {
+          return reject(err)
+        }
+        resolve(result)
+      })
+    })
+  }
+
+  private async collectLiveCellsByScript(query: LumosCellQuery) {
+    const { lock, type, data } = query
+    if (!lock && !type) {
+      throw new Error('at least one parameter is required')
+    }
+
+    const queries: QueryOptions = {
+      ...(lock ? { lock } : {}),
+      ...(type ? { type } : {}),
+      data: data || 'any',
+    }
+
+    const collector = new CellCollector(this.indexer, queries)
+
+    const result = []
+    for await (const cell of collector.collect()) {
+      //somehow the lumos indexer returns an invalid hash type "lock" for hash type "data"
+      //for now we have to fix it here
+      const cellOutput = cell.cellOutput
+      // FIXME
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-expect-error
+      if (cellOutput.type?.hashType === 'lock') {
+        console.error('Unexpected hash type "lock" found with the query', JSON.stringify(queries))
+        cellOutput.type.hashType = 'data'
+      }
+      result.push(cell)
+    }
+    return result
+  }
 }