From f035265210de1e1c42454cc19f84ee3504aff7a6 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Sat, 13 May 2023 12:55:50 -0400 Subject: [PATCH 01/21] feat: Inititalise Neutrino Client --- bin/bcoin | 3 + bin/neutrino | 37 +++++++++++ lib/blockchain/chain.js | 1 + lib/net/peer.js | 17 +++++ lib/node/neutrino.js | 133 ++++++++++++++++++++++++++++++++++++++++ lib/wallet/index.js | 1 + lib/wallet/wallet.js | 2 + 7 files changed, 194 insertions(+) create mode 100755 bin/neutrino create mode 100644 lib/node/neutrino.js diff --git a/bin/bcoin b/bin/bcoin index 0df74c54f..700c8c757 100755 --- a/bin/bcoin +++ b/bin/bcoin @@ -46,6 +46,9 @@ for arg in "$@"; do --spv) cmd='spvnode' ;; + --neutrino) + cmd='neutrino' + ;; esac done diff --git a/bin/neutrino b/bin/neutrino new file mode 100755 index 000000000..a649e3763 --- /dev/null +++ b/bin/neutrino @@ -0,0 +1,37 @@ +#!/usr/bin/env node + +'use strict'; + +console.log('Starting bcoin'); +process.title = 'bcoin'; +const Neutrino = require('../lib/node/neutrino'); + +// Doubt in db +const node = new Neutrino({ + file: true, + argv: true, + env: true, + logFile: true, + logConsole: true, + logLevel: 'debug', + db: 'leveldb', + memory: false, + persistent: true, + workers: true, + listen: true, + loader: require +}); + +(async () => { + await node.ensure(); + await node.open(); + await node.connect(); + node.startSync(); + + node.on("full", () => { + console.log("Full node"); + }); +})().catch((err) => { + console.error(err.stack); + process.exit(1); +}); diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 9cd0a312f..8c9ed2953 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2616,6 +2616,7 @@ class ChainOptions { this.compression = true; this.spv = false; + this.neutrino = false; this.bip91 = false; this.bip148 = false; this.prune = false; diff --git a/lib/net/peer.js b/lib/net/peer.js index 2271e7896..f63092633 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -1009,6 +1009,12 @@ class Peer extends EventEmitter { case packetTypes.GETHEADERS: this.request(packetTypes.HEADERS, timeout * 2); break; + case packetTypes.GETCFHEADERS: + this.request(packetTypes.CFHEADERS, timeout); + break; + case packetTypes.GETCFILTERS: + this.request(packetTypes.CFILTER, timeout); + break; case packetTypes.GETDATA: this.request(packetTypes.DATA, timeout * 2); break; @@ -1448,6 +1454,12 @@ class Peer extends EventEmitter { if (this.outbound) { if (!(this.services & services.NETWORK)) throw new Error('Peer does not support network services.'); + + if(this.options.neutrino) { + if(!(this.services & services.NODE_COMPACT_FILTERS)){ + throw new Error('Peer does not support Neutrino.'); + } + } if (this.options.headers) { if (this.version < common.HEADERS_VERSION) @@ -2143,6 +2155,11 @@ class PeerOptions { this.spv = options.spv; } + if(options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + if (options.compact != null) { assert(typeof options.compact === 'boolean'); this.compact = options.compact; diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js new file mode 100644 index 000000000..5d932025e --- /dev/null +++ b/lib/node/neutrino.js @@ -0,0 +1,133 @@ +'use strict'; + +const Chain = require('../blockchain/chain'); +const {SPVNode} = require('./index'); +const {Pool} = require('../net'); +const {RPC} = require('bweb'); +const HTTP = require('http'); +const blockstore = require('../blockstore'); + +/** + * Neutrino Node + * Create a neutrino node which only maintains + * a chain, a pool, and an http server. + * @alias module:node.Neutrino + * @extends Node + */ + +class Neutrino extends SPVNode { + /** + * Create Neutrino node. + * @constructor + * @param {Object?} options + * @param {Buffer?} options.sslKey + * @param {Buffer?} options.sslCert + * @param {Number?} options.httpPort + * @param {String?} options.httpHost + */ + constructor(options) { + super('bcoin', 'bcoin.conf', 'debug.log', options); + + this.opened = false; + + // SPV flag. + this.spv = true; + + this.blocks = blockstore.create({ + network: this.network, + logger: this.logger, + prefix: this.config.prefix, + cacheSize: this.config.mb('block-cache-size'), + memory: this.memory + }); + + this.chain = new Chain({ + network: this.network, + logger: this.logger, + prefix: this.config.prefix, + memory: this.memory, + maxFiles: this.config.uint('max-files'), + cacheSize: this.config.mb('cache-size'), + entryCache: this.config.uint('entry-cache'), + forceFlags: this.config.bool('force-flags'), + checkpoints: this.config.bool('checkpoints'), + bip91: this.config.bool('bip91'), + bip148: this.config.bool('bip148'), + spv: true + }); + + this.pool = new Pool({ + network: this.network, + logger: this.logger, + chain: this.chain, + prefix: this.config.prefix, + proxy: this.config.str('proxy'), + onion: this.config.bool('onion'), + upnp: this.config.bool('upnp'), + seeds: this.config.array('seeds'), + nodes: this.config.array('nodes'), + only: this.config.array('only'), + maxOutbound: this.config.uint('max-outbound'), + createSocket: this.config.func('create-socket'), + memory: this.memory, + selfish: true, + listen: false + }); + + this.rpc = new RPC(this); + + this.http = new HTTP({ + network: this.network, + logger: this.logger, + node: this, + prefix: this.config.prefix, + ssl: this.config.bool('ssl'), + keyFile: this.config.path('ssl-key'), + certFile: this.config.path('ssl-cert'), + host: this.config.str('http-host'), + port: this.config.uint('http-port'), + apiKey: this.config.str('api-key'), + noAuth: this.config.bool('no-auth'), + cors: this.config.bool('cors') + }); + + this.init(); + } + + init() { + // Bind to errors + this.chain.on('error', err => this.error(err)); + this.pool.on('error', err => this.error(err)); + + if (this.http) + this.http.on('error', err => this.error(err)); + + this.pool.on('tx', (tx) => { + this.emit('tx', tx); + }); + + this.chain.on('block', (block) => { + this.emit('block', block); + }); + + this.chain.on('connect', async (entry, block) => { + this.emit('connect', entry, block); + }); + + this.chain.on('disconnect', (entry, block) => { + this.emit('disconnect', entry, block); + }); + + this.chain.on('reorganize', (tip, competitor) => { + this.emit('reorganize', tip, competitor); + }); + + this.chain.on('reset', (tip) => { + this.emit('reset', tip); + }); + + this.loadPlugins(); + } +} + +module.exports = Neutrino; diff --git a/lib/wallet/index.js b/lib/wallet/index.js index 304662ab7..3e49f823a 100644 --- a/lib/wallet/index.js +++ b/lib/wallet/index.js @@ -12,6 +12,7 @@ exports.Account = require('./account'); exports.Client = require('./client'); +exports.NeutrinoClient = require('./neutrinoclient'); exports.common = require('./common'); exports.HTTP = require('./http'); exports.layout = require('./layout'); diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index 3703deaf9..32bdb4b38 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -59,6 +59,8 @@ class Wallet extends EventEmitter { this.writeLock = new Lock(); this.fundLock = new Lock(); + this.neutrino = false; + this.wid = 0; this.id = null; this.watchOnly = false; From 23da8ac7db652e7fc076c3f4d883c0564305235f Mon Sep 17 00:00:00 2001 From: masterchief164 <63920595+masterchief164@users.noreply.github.com> Date: Tue, 30 May 2023 03:58:06 +0530 Subject: [PATCH 02/21] feat: added methods to send get packets for compact filters --- lib/net/peer.js | 68 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/lib/net/peer.js b/lib/net/peer.js index f63092633..077f0d53a 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -1454,9 +1454,9 @@ class Peer extends EventEmitter { if (this.outbound) { if (!(this.services & services.NETWORK)) throw new Error('Peer does not support network services.'); - - if(this.options.neutrino) { - if(!(this.services & services.NODE_COMPACT_FILTERS)){ + + if (this.options.neutrino) { + if (!(this.services & services.NODE_COMPACT_FILTERS)) { throw new Error('Peer does not support Neutrino.'); } } @@ -1757,6 +1757,26 @@ class Peer extends EventEmitter { this.send(packet); } + /** + * @param {Number} filterType - `0` = basic + * @param {Number} startHeight - Height to start at. + * @param {Hash} stopHash - Hash to stop at. + * @returns {void} + * @description Send `getcfilters` to peer. + */ + sendGetCFilters(filterType, startHeight, stopHash) { + const packet = new packets.GetCFiltersPacket( + filterType, + startHeight, + stopHash); + + this.logger.debug( + 'Sending getcfilters (type=%d, startHeight=%d, stopHash=%h).', + filterType, startHeight, stopHash); + + this.send(packet); + } + /** * Send `cfheaders` to peer. * @param {Number} filterType @@ -1779,6 +1799,27 @@ class Peer extends EventEmitter { this.send(packet); } + /** + * @param {Number} filterType + * @param {Hash} startHash + * @param {Hash} stopHash + * @returns {void} + * @description Send `getcfheaders` to peer. + */ + + sendGetCFHeaders(filterType, startHash, stopHash) { + const packet = new packets.GetCFHeadersPacket( + filterType, + startHash, + stopHash); + + this.logger.debug( + 'Sending getcfheaders (type=%d, start=%h, stop=%h).', + filterType, startHash, stopHash); + + this.send(packet); + } + /** * send `cfcheckpt` to peer. * @param {Number} filterType @@ -1799,6 +1840,25 @@ class Peer extends EventEmitter { this.send(packet); } + /** + * Send `getcfcheckpt` to peer. + * @param {Number} filterType + * @param {Hash} stopHash + * @returns {void} + */ + + sendGetCFCheckpt(filterType, stopHash) { + const packet = new packets.GetCFCheckptPacket( + filterType, + stopHash); + + this.logger.debug( + 'Sending getcfcheckpt (type=%d, stop=%h).', + filterType, stopHash); + + this.send(packet); + } + /** * Send `mempool` to peer. */ @@ -2155,7 +2215,7 @@ class PeerOptions { this.spv = options.spv; } - if(options.neutrino != null) { + if (options.neutrino != null) { assert(typeof options.neutrino === 'boolean'); this.neutrino = options.neutrino; } From 6ab4b668c34cea81008c5f4e850eebd27c281471 Mon Sep 17 00:00:00 2001 From: masterchief164 <63920595+masterchief164@users.noreply.github.com> Date: Tue, 30 May 2023 04:11:27 +0530 Subject: [PATCH 03/21] feat: initialize filter sync --- bin/bcoin | 8 +-- bin/neutrino | 2 + lib/bcoin-browser.js | 1 + lib/bcoin.js | 1 + lib/net/peer.js | 3 +- lib/net/pool.js | 52 +++++++++++++- lib/node/neutrino.js | 168 ++++++++++++++++++++++++++++++++++++++++--- 7 files changed, 221 insertions(+), 14 deletions(-) diff --git a/bin/bcoin b/bin/bcoin index 700c8c757..56842efa5 100755 --- a/bin/bcoin +++ b/bin/bcoin @@ -43,12 +43,12 @@ for arg in "$@"; do --daemon) daemon=1 ;; - --spv) - cmd='spvnode' - ;; - --neutrino) + --neutrino) cmd='neutrino' ;; + --spv) + cmd='spvnode' + ;; esac done diff --git a/bin/neutrino b/bin/neutrino index a649e3763..16e1d48ee 100755 --- a/bin/neutrino +++ b/bin/neutrino @@ -5,6 +5,8 @@ console.log('Starting bcoin'); process.title = 'bcoin'; const Neutrino = require('../lib/node/neutrino'); +const Outpoint = require('../lib/primitives/outpoint'); +const assert = require('assert'); // Doubt in db const node = new Neutrino({ diff --git a/lib/bcoin-browser.js b/lib/bcoin-browser.js index 1f7254be8..8b2d46cb5 100644 --- a/lib/bcoin-browser.js +++ b/lib/bcoin-browser.js @@ -89,6 +89,7 @@ bcoin.node = require('./node'); bcoin.Node = require('./node/node'); bcoin.FullNode = require('./node/fullnode'); bcoin.SPVNode = require('./node/spvnode'); +bcoin.Neutrino = require('./node/neutrino'); // Primitives bcoin.primitives = require('./primitives'); diff --git a/lib/bcoin.js b/lib/bcoin.js index 3e795f6f6..3e4adb7b5 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -108,6 +108,7 @@ bcoin.define('node', './node'); bcoin.define('Node', './node/node'); bcoin.define('FullNode', './node/fullnode'); bcoin.define('SPVNode', './node/spvnode'); +bcoin.define('Neutrino', './node/neutrino'); // Primitives bcoin.define('primitives', './primitives'); diff --git a/lib/net/peer.js b/lib/net/peer.js index 077f0d53a..8f6c7f3c6 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -1466,7 +1466,7 @@ class Peer extends EventEmitter { throw new Error('Peer does not support getheaders.'); } - if (this.options.spv) { + if (this.options.spv && !this.options.neutrino) { if (!(this.services & services.BLOOM)) throw new Error('Peer does not support BIP37.'); @@ -2152,6 +2152,7 @@ class PeerOptions { this.agent = common.USER_AGENT; this.noRelay = false; this.spv = false; + this.neutrino = false; this.compact = false; this.headers = false; this.banScore = common.BAN_SCORE; diff --git a/lib/net/pool.js b/lib/net/pool.js index 234b23bc2..323da2e5c 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -68,6 +68,7 @@ class Pool extends EventEmitter { this.connected = false; this.disconnecting = false; this.syncing = false; + this.filterSyncing = false; this.discovering = false; this.spvFilter = null; this.txFilter = null; @@ -79,6 +80,7 @@ class Pool extends EventEmitter { this.pendingRefill = null; this.checkpoints = false; + this.neutrino = false; this.headerChain = new List(); this.headerNext = null; this.headerTip = null; @@ -704,6 +706,18 @@ class Pool extends EventEmitter { this.compactBlocks.clear(); } + /** + * Start the filters sync. + */ + + startFilterSync() { + if (!this.opened || !this.connected) + return; + + this.filterSyncing = true; + this.refill(); + } + /** * Send a sync to each peer. * @private @@ -794,6 +808,17 @@ class Pool extends EventEmitter { return this.sendLocator(locator, peer); } + /** + * Sync the filter headers from peer. + * @method + * @param {Peer} peer + * @returns {Promise} + */ + + async syncCompactFilters(peer) { + peer.sendGetCFCheckpt(common.FILTERS.BASIC, this.chain.tip.hash); + } + /** * Send a chain locator and start syncing from peer. * @method @@ -2257,6 +2282,11 @@ class Pool extends EventEmitter { await this.addBlock(peer, packet.block, flags); } + async handleFilter(peer, packet) { + const flags = chainCommon.flags.DEFAULT_FLAGS; + await this.addFilter(peer, packet.filter, flags); + } + /** * Attempt to add block to chain. * @method @@ -2353,6 +2383,20 @@ class Pool extends EventEmitter { await this.resolveChain(peer, hash); } + async addFilter(peer, filter, flags) { + const hash = filter.hash(); + const unlock = await this.locker.lock(hash); + try { + return await this._addFilter(peer, filter, flags); + } finally { + unlock(); + } + } + + async _addFilter(peer, filter, flags) { + + } + /** * Resolve header chain. * @method @@ -3690,6 +3734,7 @@ class PoolOptions { this.prefix = null; this.checkpoints = true; this.spv = false; + this.neutrino = false; this.bip37 = false; this.bip157 = false; this.listen = false; @@ -3778,6 +3823,11 @@ class PoolOptions { this.spv = this.chain.options.spv; } + if(options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + if (options.bip37 != null) { assert(typeof options.bip37 === 'boolean'); this.bip37 = options.bip37; @@ -3961,7 +4011,7 @@ class PoolOptions { if (this.bip37) this.services |= common.services.BLOOM; - if (this.bip157) + if (this.bip157 || this.neutrino) this.services |= common.services.NODE_COMPACT_FILTERS; if (this.proxy) diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 5d932025e..6bf51b532 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -1,10 +1,18 @@ +/*! + * neutrino.js - spv node for bcoin + * Copyright (c) 2014-2015, Fedor Indutny (MIT License) + * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). + * https://github.com/bcoin-org/bcoin + */ + 'use strict'; +const assert = require('bsert'); const Chain = require('../blockchain/chain'); -const {SPVNode} = require('./index'); -const {Pool} = require('../net'); -const {RPC} = require('bweb'); -const HTTP = require('http'); +const Pool = require('../net/pool'); +const Node = require('./node'); +const HTTP = require('./http'); +const RPC = require('./rpc'); const blockstore = require('../blockstore'); /** @@ -15,7 +23,7 @@ const blockstore = require('../blockstore'); * @extends Node */ -class Neutrino extends SPVNode { +class Neutrino extends Node { /** * Create Neutrino node. * @constructor @@ -25,14 +33,17 @@ class Neutrino extends SPVNode { * @param {Number?} options.httpPort * @param {String?} options.httpHost */ + constructor(options) { super('bcoin', 'bcoin.conf', 'debug.log', options); this.opened = false; // SPV flag. - this.spv = true; + this.spv = false; + this.neutrino = true; + // Instantiate block storage. this.blocks = blockstore.create({ network: this.network, logger: this.logger, @@ -53,7 +64,8 @@ class Neutrino extends SPVNode { checkpoints: this.config.bool('checkpoints'), bip91: this.config.bool('bip91'), bip148: this.config.bool('bip148'), - spv: true + spv: true, + neutrino: true, }); this.pool = new Pool({ @@ -71,7 +83,8 @@ class Neutrino extends SPVNode { createSocket: this.config.func('create-socket'), memory: this.memory, selfish: true, - listen: false + listen: false, + neutrino: true }); this.rpc = new RPC(this); @@ -94,7 +107,13 @@ class Neutrino extends SPVNode { this.init(); } + /** + * Initialize the node. + * @private + */ + init() { + console.log('Initializing Neutrino Node (spv).'); // Bind to errors this.chain.on('error', err => this.error(err)); this.pool.on('error', err => this.error(err)); @@ -126,8 +145,141 @@ class Neutrino extends SPVNode { this.emit('reset', tip); }); + this.chain.on('full', () => { + this.logger.info('Block Headers are fully synced'); + this.pool.startFilterSync(); + }); + this.loadPlugins(); } + + /** + * Open the node and all its child objects, + * wait for the database to load. + * @returns {Promise} + */ + + async open() { + assert(!this.opened, 'SPVNode is already open.'); + this.opened = true; + + await this.handlePreopen(); + await this.chain.open(); + await this.pool.open(); + + await this.openPlugins(); + + await this.http.open(); + await this.handleOpen(); + + this.logger.info('Node is loaded.'); + } + + /** + * Close the node, wait for the database to close. + * @returns {Promise} + */ + + async close() { + assert(this.opened, 'SPVNode is not open.'); + this.opened = false; + + await this.handlePreclose(); + await this.http.close(); + + await this.closePlugins(); + + await this.pool.close(); + await this.chain.close(); + await this.handleClose(); + } + + /** + * Scan for any missed transactions. + * Note that this will replay the blockchain sync. + * @param {Number|Hash} start - Start block. + * @returns {Promise} + */ + + async scan(start) { + throw new Error('Not implemented.'); + } + + /** + * Broadcast a transaction (note that this will _not_ be verified + * by the mempool - use with care, lest you get banned from + * bitcoind nodes). + * @param {TX|Block} item + * @returns {Promise} + */ + + async broadcast(item) { + try { + await this.pool.broadcast(item); + } catch (e) { + this.emit('error', e); + } + } + + /** + * Broadcast a transaction (note that this will _not_ be verified + * by the mempool - use with care, lest you get banned from + * bitcoind nodes). + * @param {TX} tx + * @returns {Promise} + */ + + sendTX(tx) { + return this.broadcast(tx); + } + + /** + * Broadcast a transaction. Silence errors. + * @param {TX} tx + * @returns {Promise} + */ + + relay(tx) { + return this.broadcast(tx); + } + + /** + * Connect to the network. + * @returns {Promise} + */ + + connect() { + return this.pool.connect(); + } + + /** + * Disconnect from the network. + * @returns {Promise} + */ + + disconnect() { + return this.pool.disconnect(); + } + + /** + * Start the blockchain sync. + */ + + startSync() { + return this.pool.startSync(); + } + + /** + * Stop syncing the blockchain. + */ + + stopSync() { + return this.pool.stopSync(); + } } +/* + * Expose + */ + module.exports = Neutrino; From 3d2951c33063fd981421818946d0b42445449d0a Mon Sep 17 00:00:00 2001 From: masterchief164 <63920595+masterchief164@users.noreply.github.com> Date: Wed, 31 May 2023 17:27:15 +0530 Subject: [PATCH 04/21] feat: added a new column in chaindb to store neutrino state --- lib/blockchain/chaindb.js | 62 +++++++++++++++++++++++++++++++++++---- lib/blockchain/layout.js | 2 ++ lib/client/node.js | 4 +++ lib/net/peer.js | 10 +++---- lib/net/pool.js | 5 +++- lib/node/http.js | 7 +++++ lib/wallet/client.js | 13 ++++++++ lib/wallet/index.js | 1 - lib/wallet/nodeclient.js | 13 ++++++++ lib/wallet/nullclient.js | 13 ++++++++ 10 files changed, 117 insertions(+), 13 deletions(-) diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index cb91accaa..cf4d95a42 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -46,6 +46,7 @@ class ChainDB { this.state = new ChainState(); this.pending = null; this.current = null; + this.neutrinoState = null; this.cacheHash = new LRU(this.options.entryCache, null, BufferMap); this.cacheHeight = new LRU(this.options.entryCache); @@ -90,6 +91,11 @@ class ChainDB { this.logger.info('ChainDB successfully initialized.'); } + if (this.options.neutrino) { + if (!this.neutrinoState) + this.neutrinoState = await this.getNeutrinoState(); + } + this.logger.info( 'Chain State: hash=%h tx=%d coin=%d value=%s.', this.state.tip, @@ -663,10 +669,10 @@ class ChainDB { const deployment = this.network.byBit(bit); if (deployment - && start === deployment.startTime - && timeout === deployment.timeout - && threshold === deployment.threshold - && window === deployment.window) { + && start === deployment.startTime + && timeout === deployment.timeout + && threshold === deployment.threshold + && window === deployment.window) { continue; } @@ -1361,7 +1367,7 @@ class ChainDB { this.logger.debug('Resetting main chain to: %h', entry.hash); - for (;;) { + for (; ;) { this.start(); // Stop once we hit our target tip. @@ -1446,7 +1452,7 @@ class ChainDB { this.logger.debug('Removing alternate chain: %h.', tip.hash); - for (;;) { + for (; ;) { if (await this.isMainChain(tip)) break; @@ -1670,6 +1676,29 @@ class ChainDB { b.put(layout.O.encode(), flags.toRaw()); return b.write(); } + + /** + * Get Neutrino State + * @returns {Promise} - Returns neutrino state + */ + + async getNeutrinoState() { + const data = await this.db.get(layout.N.encode()); + if (!data) + return new NeutrinoState(); + return NeutrinoState.fromRaw(data); + } + + /** + * Save Neutrino State + * @returns {void} + */ + async saveNeutrinoState() { + const state = this.neutrinoState.toRaw(); + const b = this.db.batch(); + b.put(layout.N.encode(), state); + return b.write(); + } } /** @@ -1952,6 +1981,27 @@ function fromU32(num) { return data; } +class NeutrinoState { + constructor() { // TODO: do we add support for multiple filters? + this.headerHeight = 0; + this.filterHeight = 0; + } + + toRaw() { + const bw = bio.write(8); + bw.writeU32(this.headerHeight); + bw.writeU32(this.filterHeight); + return bw.render(); + } + + static fromRaw(data) { + const br = bio.read(data); + this.headersHeight = br.readU32(); + this.filterHeight = br.readU32(); + return this; + } +} + /* * Expose */ diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js index 337f95900..1443e3170 100644 --- a/lib/blockchain/layout.js +++ b/lib/blockchain/layout.js @@ -14,6 +14,7 @@ const bdb = require('bdb'); * O -> chain options * R -> tip hash * D -> versionbits deployments + * N -> neutrino state * e[hash] -> entry * h[hash] -> height * H[height] -> hash @@ -33,6 +34,7 @@ const layout = { O: bdb.key('O'), R: bdb.key('R'), D: bdb.key('D'), + N: bdb.key('N'), e: bdb.key('e', ['hash256']), h: bdb.key('h', ['hash256']), H: bdb.key('H', ['uint32']), diff --git a/lib/client/node.js b/lib/client/node.js index 50800cac1..7822408d1 100644 --- a/lib/client/node.js +++ b/lib/client/node.js @@ -169,6 +169,10 @@ class NodeClient extends Client { return this.get(`/filter/${filter}`); } + checkKeyFilter(key, filter) { + return this.call('check key filter', key, filter); + } + /** * Add a transaction to the mempool and broadcast it. * @param {TX} tx diff --git a/lib/net/peer.js b/lib/net/peer.js index 8f6c7f3c6..b9f1e4090 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -1457,7 +1457,7 @@ class Peer extends EventEmitter { if (this.options.neutrino) { if (!(this.services & services.NODE_COMPACT_FILTERS)) { - throw new Error('Peer does not support Neutrino.'); + throw new Error('Peer does not support Compact Filters.'); } } @@ -1801,21 +1801,21 @@ class Peer extends EventEmitter { /** * @param {Number} filterType - * @param {Hash} startHash + * @param {Number} startHeight * @param {Hash} stopHash * @returns {void} * @description Send `getcfheaders` to peer. */ - sendGetCFHeaders(filterType, startHash, stopHash) { + sendGetCFHeaders(filterType, startHeight, stopHash) { const packet = new packets.GetCFHeadersPacket( filterType, - startHash, + startHeight, stopHash); this.logger.debug( 'Sending getcfheaders (type=%d, start=%h, stop=%h).', - filterType, startHash, stopHash); + filterType, startHeight, stopHash); this.send(packet); } diff --git a/lib/net/pool.js b/lib/net/pool.js index 323da2e5c..ec0b6b8d7 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -4003,6 +4003,9 @@ class PoolOptions { this.listen = false; } + if (this.neutrino) + this.requiredServices |= common.services.NODE_COMPACT_FILTERS; + if (this.selfish) { this.services &= ~common.services.NETWORK; this.bip37 = false; @@ -4011,7 +4014,7 @@ class PoolOptions { if (this.bip37) this.services |= common.services.BLOOM; - if (this.bip157 || this.neutrino) + if (this.bip157) this.services |= common.services.NODE_COMPACT_FILTERS; if (this.proxy) diff --git a/lib/node/http.js b/lib/node/http.js index 8448ec015..838aa9be7 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -498,6 +498,13 @@ class HTTP extends Server { return null; }); + socket.hook('check key filter', (...args) => { + const valid = new Validator(args); + const key = valid.buf(0); + const filter = valid.buf(1); + return this.pool.checkKeyFilter(key, filter); + }); + socket.hook('estimate fee', (...args) => { const valid = new Validator(args); const blocks = valid.u32(0); diff --git a/lib/wallet/client.js b/lib/wallet/client.js index 768f38e50..93c521581 100644 --- a/lib/wallet/client.js +++ b/lib/wallet/client.js @@ -12,6 +12,8 @@ const NodeClient = require('../client/node'); const util = require('../utils/util'); const TX = require('../primitives/tx'); const hash256 = require('bcrypto/lib/hash256'); +const WalletKey = require('./walletkey'); +const Filter = require('../primitives/filter'); const parsers = { 'block connect': (entry, txs) => parseBlock(entry, txs), @@ -71,6 +73,17 @@ class WalletClient extends NodeClient { return super.setFilter(filter.toRaw()); } + /** + * Check filter against wallet key ring + * @param {WalletKey} ring + * @param {Filter} filter + * @returns {Promise} + */ + + async checkKeyFilter(ring, filter) { + return super.checkKeyFilter(ring, filter); + } + async rescan(start) { if (Buffer.isBuffer(start)) start = util.revHex(start); diff --git a/lib/wallet/index.js b/lib/wallet/index.js index 3e49f823a..304662ab7 100644 --- a/lib/wallet/index.js +++ b/lib/wallet/index.js @@ -12,7 +12,6 @@ exports.Account = require('./account'); exports.Client = require('./client'); -exports.NeutrinoClient = require('./neutrinoclient'); exports.common = require('./common'); exports.HTTP = require('./http'); exports.layout = require('./layout'); diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 9f6c43600..8372b73c0 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -8,6 +8,8 @@ const assert = require('bsert'); const AsyncEmitter = require('bevent'); +const WalletKey = require('./walletkey'); +const Filter = require('../primitives/filter'); /** * Node Client @@ -174,6 +176,17 @@ class NodeClient extends AsyncEmitter { this.node.pool.queueFilterLoad(); } + /** + * Check filter against wallet key ring + * @param {WalletKey} ring + * @param {Filter} filter + * @returns {Promise} + */ + + async checkKeyFilter(ring, filter) { + this.node.pool.checkKeyFilter(ring, filter); + } + /** * Estimate smart fee. * @param {Number?} blocks diff --git a/lib/wallet/nullclient.js b/lib/wallet/nullclient.js index 744629d4b..68221a397 100644 --- a/lib/wallet/nullclient.js +++ b/lib/wallet/nullclient.js @@ -8,6 +8,8 @@ const assert = require('bsert'); const EventEmitter = require('events'); +const WalletKey = require('./walletkey'); +const Filter = require('../primitives/filter'); /** * Null Client @@ -130,6 +132,17 @@ class NullClient extends EventEmitter { this.wdb.emit('reset filter'); } + /** + * Check filter against wallet key ring + * @param {WalletKey} ring + * @param {Filter} filter + * @returns {Promise} + */ + + async checkKeyFilter(ring, filter) { + ; + } + /** * Esimate smart fee. * @param {Number?} blocks From e9aa064403e0eb6933a02e4f8305edbf2f00d7a5 Mon Sep 17 00:00:00 2001 From: masterchief164 <63920595+masterchief164@users.noreply.github.com> Date: Tue, 6 Jun 2023 02:54:47 +0530 Subject: [PATCH 05/21] feat: added headers first sync --- lib/blockchain/chain.js | 5 ++ lib/net/pool.js | 177 +++++++++++++++++++++++++++++++++------- lib/node/neutrino.js | 41 ++++++++-- 3 files changed, 186 insertions(+), 37 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 8c9ed2953..907f2eb3b 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2663,6 +2663,11 @@ class ChainOptions { this.spv = options.spv; } + if (options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + if (options.prefix != null) { assert(typeof options.prefix === 'string'); this.prefix = options.prefix; diff --git a/lib/net/pool.js b/lib/net/pool.js index ec0b6b8d7..bf9b05f24 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -35,6 +35,7 @@ const packetTypes = packets.types; const scores = HostList.scores; const {inspectSymbol} = require('../utils'); const {consensus} = require('../protocol'); +const BasicFilter = require('../golomb/basicFilter'); /** * Pool @@ -206,7 +207,7 @@ class Pool extends EventEmitter { */ resetChain() { - if (!this.options.checkpoints) + if (!this.options.checkpoints && !this.options.neutrino) return; this.checkpoints = false; @@ -215,7 +216,11 @@ class Pool extends EventEmitter { this.headerNext = null; const tip = this.chain.tip; - + if (this.options.neutrino) { + // this.headerTip = tip; + this.headerChain.push(new HeaderEntry(tip.hash, tip.height)); + return; + } if (tip.height < this.network.lastCheckpoint) { this.checkpoints = true; this.headerTip = this.getNextTip(tip.height); @@ -637,8 +642,7 @@ class Pool extends EventEmitter { peer.loader = true; this.peers.load = peer; - - this.sendSync(peer); + this.sendSync(peer); this.emit('loader', peer); } @@ -652,7 +656,10 @@ class Pool extends EventEmitter { return; this.syncing = true; - this.resync(false); + if (this.options.neutrino) { + this.startHeadersSync(); + } else + this.resync(false); } /** @@ -710,12 +717,45 @@ class Pool extends EventEmitter { * Start the filters sync. */ - startFilterSync() { + startFilterHeadersSync() { + this.logger.info('Starting filter headers sync (%s).', + this.chain.options.network); if (!this.opened || !this.connected) return; this.filterSyncing = true; - this.refill(); + const startHeight = 0; + const stopHash = this.chain.getHash(1000); + this.peers.load.sendGetCFHeaders( + common.FILTERS.BASIC, + startHeight, + stopHash); + } + + /** + * Start the headers sync using getHeaders messages. + * @private + * @return {Promise} + */ + + async startHeadersSync() { + if (!this.syncing) + return; + + let locator; + try { + locator = await this.chain.getLocator(); + } catch (e) { + this.emit('error', e); + return; + } + + const peer = this.peers.load; + if (!peer) { + this.logger.info('No loader peer.'); + return; + } + peer.sendGetHeaders(locator); } /** @@ -808,16 +848,16 @@ class Pool extends EventEmitter { return this.sendLocator(locator, peer); } - /** - * Sync the filter headers from peer. - * @method - * @param {Peer} peer - * @returns {Promise} - */ - - async syncCompactFilters(peer) { - peer.sendGetCFCheckpt(common.FILTERS.BASIC, this.chain.tip.hash); - } + // /** + // * Sync the filter headers from peer. + // * @method + // * @param {Peer} peer + // * @returns {void} + // */ + // + // syncCompactFiltersCheckPt(peer) { + // peer.sendGetCFCheckpt(common.FILTERS.BASIC, this.chain.tip.hash); + // } /** * Send a chain locator and start syncing from peer. @@ -839,7 +879,10 @@ class Pool extends EventEmitter { peer.syncing = true; peer.blockTime = Date.now(); - + if (this.options.neutrino) { + peer.sendGetHeaders(locator); + return true; + } if (this.checkpoints) { peer.sendGetHeaders(locator, this.headerTip.hash); return true; @@ -1219,6 +1262,12 @@ class Pool extends EventEmitter { case packetTypes.GETCFCHECKPT: await this.handleGetCFCheckpt(peer, packet); break; + case packetTypes.CFCHECKPT: + await this.handleCFCheckpt(peer, packet); + break; + case packetTypes.CFHEADERS: + await this.handleCFHeaders(peer, packet); + break; case packetTypes.GETBLOCKS: await this.handleGetBlocks(peer, packet); break; @@ -1271,8 +1320,6 @@ class Pool extends EventEmitter { await this.handleBlockTxn(peer, packet); break; case packetTypes.CFILTER: - case packetTypes.CFHEADERS: - case packetTypes.CFCHECKPT: case packetTypes.UNKNOWN: await this.handleUnknown(peer, packet); break; @@ -1316,7 +1363,7 @@ class Pool extends EventEmitter { } // We want compact blocks! - if (this.options.compact) + if (this.options.compact && !this.options.neutrino) peer.sendCompact(this.options.blockMode); // Find some more peers. @@ -2037,6 +2084,58 @@ class Pool extends EventEmitter { peer.sendCFCheckpt(packet.filterType, packet.stopHash, filterHeaders); } + /** + * Handle peer `CFCheckpt` packet. + * @method + * @private + * @param {Peer} peer + * @param {CFCheckptPacket} packet + */ + + async handleCFCheckpt(peer, packet) { + if (!this.options.neutrino) { + peer.ban(); + peer.destroy(); + } + } + + /** + * Handle peer `CFHeaders` packet. + * @method + * @private + * @param {Peer} peer - Sender. + * @param {CFHeadersPacket} packet - Packet to handle. + * @returns {void} + */ + async handleCFHeaders(peer, packet) { + this.logger.info('Received CFHeaders packet from %s', peer.hostname()); + if (!this.options.neutrino) { + peer.ban(); + peer.destroy(); + return; + } + + // const filterType = packet.filterType; + const stopHash = packet.stopHash; + let previousFilterHeader = packet.previousFilterHeader; + const filterHashes = packet.filterHashes; + let blockHeight = await this.chain.getHeight(stopHash) + - filterHashes.length; + const stopHeight = await this.chain.getHeight(stopHash); + for (const filterHash of filterHashes) { + assert(blockHeight < stopHeight); + const basicFilter = new BasicFilter(); + basicFilter._hash = filterHash; + const filterHeader = basicFilter.header(previousFilterHeader); + // todo: save the filterHeader + previousFilterHeader = filterHeader; + this.chain.neutrinoState.headerHeight = blockHeight; + blockHeight++; + } + await this.chain.saveNeutrinoState(); + this.emit('cfheaders'); + } + /** * Handle `getblocks` packet. * @method @@ -2052,6 +2151,9 @@ class Pool extends EventEmitter { if (this.options.selfish) return; + if (this.options.neutrino) + return; + if (this.chain.options.spv) return; @@ -2164,7 +2266,8 @@ class Pool extends EventEmitter { async _handleHeaders(peer, packet) { const headers = packet.items; - if (!this.checkpoints) + if (!this.checkpoints && !this.options.neutrino) + // todo add support for checkpoints return; if (!this.syncing) @@ -2190,6 +2293,7 @@ class Pool extends EventEmitter { const last = this.headerChain.tail; const hash = header.hash(); const height = last.height + 1; + console.log('header'); if (!header.verify()) { this.logger.warning( @@ -2210,7 +2314,8 @@ class Pool extends EventEmitter { node = new HeaderEntry(hash, height); - if (node.height === this.headerTip.height) { + if (!this.options.neutrino && node.height === this.headerTip.height) { + // todo add support for checkpoints if (!node.hash.equals(this.headerTip.hash)) { this.logger.warning( 'Peer sent an invalid checkpoint (%s).', @@ -2225,6 +2330,9 @@ class Pool extends EventEmitter { this.headerNext = node; this.headerChain.push(node); + console.log('adding block'); + if (this.options.neutrino) + await this._addBlock(peer, header, chainCommon.flags.VERIFY_NONE); } this.logger.debug( @@ -2244,7 +2352,10 @@ class Pool extends EventEmitter { } // Request more headers. - peer.sendGetHeaders([node.hash], this.headerTip.hash); + if (this.options.neutrino) + peer.sendGetHeaders([node.hash]); + else + peer.sendGetHeaders([node.hash], this.headerTip.hash); } /** @@ -2299,7 +2410,10 @@ class Pool extends EventEmitter { async addBlock(peer, block, flags) { const hash = block.hash(); + console.log(hash); + console.log('addBlock'); const unlock = await this.locker.lock(hash); + console.log('lockedHash'); try { return await this._addBlock(peer, block, flags); } finally { @@ -2323,7 +2437,7 @@ class Pool extends EventEmitter { const hash = block.hash(); - if (!this.resolveBlock(peer, hash)) { + if (!this.options.neutrino && !this.resolveBlock(peer, hash)) { this.logger.warning( 'Received unrequested block: %h (%s).', block.hash(), peer.hostname()); @@ -2394,7 +2508,7 @@ class Pool extends EventEmitter { } async _addFilter(peer, filter, flags) { - + } /** @@ -3817,13 +3931,13 @@ class PoolOptions { if (options.spv != null) { assert(typeof options.spv === 'boolean'); - assert(options.spv === this.chain.options.spv); + // assert(options.spv === this.chain.options.spv); this.spv = options.spv; } else { this.spv = this.chain.options.spv; } - if(options.neutrino != null) { + if (options.neutrino != null) { assert(typeof options.neutrino === 'boolean'); this.neutrino = options.neutrino; } @@ -4003,8 +4117,11 @@ class PoolOptions { this.listen = false; } - if (this.neutrino) - this.requiredServices |= common.services.NODE_COMPACT_FILTERS; + if (this.neutrino) { + this.requiredServices |= common.services.NODE_COMPACT_FILTERS; + this.checkpoints = true; + this.compact = false; + } if (this.selfish) { this.services &= ~common.services.NETWORK; diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 6bf51b532..58ffc861c 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -14,6 +14,7 @@ const Node = require('./node'); const HTTP = require('./http'); const RPC = require('./rpc'); const blockstore = require('../blockstore'); +const FilterIndexer = require('../indexer/filterindexer'); /** * Neutrino Node @@ -49,10 +50,13 @@ class Neutrino extends Node { logger: this.logger, prefix: this.config.prefix, cacheSize: this.config.mb('block-cache-size'), - memory: this.memory + memory: this.memory, + spv: this.spv, + neutrino: this.neutrino }); this.chain = new Chain({ + blocks: this.blocks, network: this.network, logger: this.logger, prefix: this.config.prefix, @@ -65,14 +69,28 @@ class Neutrino extends Node { bip91: this.config.bool('bip91'), bip148: this.config.bool('bip148'), spv: true, - neutrino: true, + neutrino: this.neutrino }); + this.filterIndexers.set( + 'BASIC', + new FilterIndexer({ + network: this.network, + logger: this.logger, + blocks: this.blocks, + chain: this.chain, + memory: this.config.bool('memory'), + prefix: this.config.str('index-prefix', this.config.prefix), + filterType: 'BASIC' + }) + ); + this.pool = new Pool({ network: this.network, logger: this.logger, chain: this.chain, prefix: this.config.prefix, + checkpoints: true, proxy: this.config.str('proxy'), onion: this.config.bool('onion'), upnp: this.config.bool('upnp'), @@ -84,7 +102,8 @@ class Neutrino extends Node { memory: this.memory, selfish: true, listen: false, - neutrino: true + neutrino: this.neutrino, + spv: this.spv }); this.rpc = new RPC(this); @@ -113,7 +132,7 @@ class Neutrino extends Node { */ init() { - console.log('Initializing Neutrino Node (spv).'); + console.log('Initializing Neutrino Node.'); // Bind to errors this.chain.on('error', err => this.error(err)); this.pool.on('error', err => this.error(err)); @@ -145,8 +164,15 @@ class Neutrino extends Node { this.emit('reset', tip); }); - this.chain.on('full', () => { + this.chain.on('headersFull', () => { this.logger.info('Block Headers are fully synced'); + console.log('Block Headers are fully synced \n\n\n\n\n'); + // this.pool.startFilterCheckPtSync(); // TODO: Maybe implement this later + this.pool.startFilterHeadersSync(); + }); + + this.pool.on('cfheaders', () => { + this.logger.info('CF Headers Synced'); this.pool.startFilterSync(); }); @@ -160,10 +186,11 @@ class Neutrino extends Node { */ async open() { - assert(!this.opened, 'SPVNode is already open.'); + assert(!this.opened, 'Neutrino Node is already open.'); this.opened = true; await this.handlePreopen(); + await this.blocks.open(); await this.chain.open(); await this.pool.open(); @@ -181,7 +208,7 @@ class Neutrino extends Node { */ async close() { - assert(this.opened, 'SPVNode is not open.'); + assert(this.opened, 'Neutrino Node is not open.'); this.opened = false; await this.handlePreclose(); From 5e2bc49a3880c88354955f4739c5321314b9e278 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Wed, 7 Jun 2023 23:11:32 -0500 Subject: [PATCH 06/21] feat: implement filters flow --- lib/blockchain/chaindb.js | 7 ++-- lib/net/pool.js | 75 +++++++++++++++++++++++++++++++++++---- lib/node/neutrino.js | 6 +++- 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index cf4d95a42..9698abff1 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -1995,10 +1995,11 @@ class NeutrinoState { } static fromRaw(data) { + const state = new NeutrinoState(); const br = bio.read(data); - this.headersHeight = br.readU32(); - this.filterHeight = br.readU32(); - return this; + state.headersHeight = br.readU32(); + state.filterHeight = br.readU32(); + return state; } } diff --git a/lib/net/pool.js b/lib/net/pool.js index bf9b05f24..0140f2556 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -90,6 +90,12 @@ class Pool extends EventEmitter { this.hosts = new HostList(this.options); this.id = 0; + this.getcfheadersFilterType = null; + this.getcfheadersStopHash = null; + this.getcfiltersFilterType = null; + this.getcfiltersStartHeight = null; + this.getcfiltersStopHash = null; + if (this.options.spv) { this.spvFilter = BloomFilter.fromRate( 20000, 0.001, BloomFilter.flags.ALL); @@ -714,10 +720,10 @@ class Pool extends EventEmitter { } /** - * Start the filters sync. + * Start the filters headers sync. */ - startFilterHeadersSync() { + async startFilterHeadersSync() { this.logger.info('Starting filter headers sync (%s).', this.chain.options.network); if (!this.opened || !this.connected) @@ -725,11 +731,36 @@ class Pool extends EventEmitter { this.filterSyncing = true; const startHeight = 0; - const stopHash = this.chain.getHash(1000); + const stopHash = await this.chain.getHash(29); this.peers.load.sendGetCFHeaders( common.FILTERS.BASIC, startHeight, stopHash); + + this.getcfheadersFilterType = common.FILTERS.BASIC; + this.getcfheadersStopHash = stopHash; + } + + /** + * Start the filters sync. + */ + + async startFilterSync() { + this.logger.info('Starting filter sync (%s).', + this.chain.options.network); + if (!this.opened || !this.connected) + return; + + this.filterSyncing = true; + const startHeight = 0; + const stopHash = await this.chain.getHash(29); + this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash); + this.getcfiltersFilterType = common.FILTERS.BASIC; + this.getcfiltersStartHeight = startHeight; + this.getcfiltersStopHash = stopHash; } /** @@ -756,6 +787,7 @@ class Pool extends EventEmitter { return; } peer.sendGetHeaders(locator); + // this.emit('headersFull'); } /** @@ -1320,6 +1352,8 @@ class Pool extends EventEmitter { await this.handleBlockTxn(peer, packet); break; case packetTypes.CFILTER: + await this.handleCFilters(peer, packet); + break; case packetTypes.UNKNOWN: await this.handleUnknown(peer, packet); break; @@ -2115,10 +2149,13 @@ class Pool extends EventEmitter { return; } - // const filterType = packet.filterType; + const filterType = packet.filterType; + assert(filterType === this.getcfheadersFilterType); const stopHash = packet.stopHash; + assert(stopHash.equals(this.getcfheadersStopHash)); let previousFilterHeader = packet.previousFilterHeader; const filterHashes = packet.filterHashes; + assert(filterHashes.length <= 2000) let blockHeight = await this.chain.getHeight(stopHash) - filterHashes.length; const stopHeight = await this.chain.getHeight(stopHash); @@ -2127,15 +2164,39 @@ class Pool extends EventEmitter { const basicFilter = new BasicFilter(); basicFilter._hash = filterHash; const filterHeader = basicFilter.header(previousFilterHeader); + // todo: verify the filterHeader // todo: save the filterHeader previousFilterHeader = filterHeader; - this.chain.neutrinoState.headerHeight = blockHeight; + this.chain.db.neutrinoState.headerHeight = blockHeight; blockHeight++; } - await this.chain.saveNeutrinoState(); + await this.chain.db.saveNeutrinoState(); this.emit('cfheaders'); } + async handleCFilters(peer, packet) { + this.logger.info('Received CFilter packet from %s', peer.hostname()); + if (!this.options.neutrino) { + peer.ban(); + peer.destroy(); + return; + } + + const blockHash = packet.blockHash; + const filterType = packet.filterType; + const filter = packet.filterBytes; + + // todo: verify the filter + assert(filterType === this.getcfheadersFilterType); + const blockHeight = await this.chain.getHeight(blockHash); + const stopHeight = await this.chain.getHeight(this.getcfiltersStopHash); + assert(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight); + + // todo: save the filter + // const basicFilter = new BasicFilter(); + + } + /** * Handle `getblocks` packet. * @method @@ -2250,6 +2311,7 @@ class Pool extends EventEmitter { return await this._handleHeaders(peer, packet); } finally { unlock(); + this.emit('headersFull'); } } @@ -2330,7 +2392,6 @@ class Pool extends EventEmitter { this.headerNext = node; this.headerChain.push(node); - console.log('adding block'); if (this.options.neutrino) await this._addBlock(peer, header, chainCommon.flags.VERIFY_NONE); } diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 58ffc861c..89eff9ae7 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -164,7 +164,7 @@ class Neutrino extends Node { this.emit('reset', tip); }); - this.chain.on('headersFull', () => { + this.pool.on('headersFull', () => { this.logger.info('Block Headers are fully synced'); console.log('Block Headers are fully synced \n\n\n\n\n'); // this.pool.startFilterCheckPtSync(); // TODO: Maybe implement this later @@ -176,6 +176,10 @@ class Neutrino extends Node { this.pool.startFilterSync(); }); + this.pool.on('filtersFull', () => { + this.logger.info('Filters are fully synced'); + }) + this.loadPlugins(); } From 3e8f06114ae7483dcf242b27f4c82ae176e066dd Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Thu, 8 Jun 2023 22:18:32 -0500 Subject: [PATCH 07/21] feat: connect filters to wallet --- bin/neutrino | 5 +++++ lib/net/pool.js | 5 +++-- lib/node/neutrino.js | 3 --- lib/wallet/nodeclient.js | 8 ++++++- lib/wallet/walletdb.js | 48 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 6 deletions(-) diff --git a/bin/neutrino b/bin/neutrino index 16e1d48ee..890e88bb6 100755 --- a/bin/neutrino +++ b/bin/neutrino @@ -24,6 +24,11 @@ const node = new Neutrino({ loader: require }); +if (!node.config.bool('no-wallet') && !node.has('walletdb')) { + const plugin = require('../lib/wallet/plugin'); + node.use(plugin); +} + (async () => { await node.ensure(); await node.open(); diff --git a/lib/net/pool.js b/lib/net/pool.js index 0140f2556..0919112eb 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -2193,8 +2193,9 @@ class Pool extends EventEmitter { assert(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight); // todo: save the filter - // const basicFilter = new BasicFilter(); - + const basicFilter = new BasicFilter(); + const gcsFilter = basicFilter.fromNBytes(filter); + this.emit('cfilter', blockHeight, gcsFilter); } /** diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 89eff9ae7..b98db92ed 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -176,9 +176,6 @@ class Neutrino extends Node { this.pool.startFilterSync(); }); - this.pool.on('filtersFull', () => { - this.logger.info('Filters are fully synced'); - }) this.loadPlugins(); } diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 8372b73c0..0a3215451 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -39,7 +39,7 @@ class NodeClient extends AsyncEmitter { init() { this.node.chain.on('connect', async (entry, block) => { - if (!this.opened) + if (!this.opened || this.node.neutrino) return; await this.emitAsync('block connect', entry, block.txs); @@ -52,6 +52,12 @@ class NodeClient extends AsyncEmitter { await this.emitAsync('block disconnect', entry); }); + this.node.pool.on('cfilter', async (blockHeight, filter) => { + if(!this.opened) return; + + await this.emitAsync('cfilter', blockHeight, filter); + }) + this.node.on('tx', (tx) => { if (!this.opened) return; diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index b1f57ebf0..ae237a038 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -23,6 +23,7 @@ const common = require('./common'); const Wallet = require('./wallet'); const Account = require('./account'); const Outpoint = require('../primitives/outpoint'); +const random = require('bcrypto/lib/random'); const layouts = require('./layout'); const records = require('./records'); const NullClient = require('./nullclient'); @@ -169,6 +170,14 @@ class WalletDB extends EventEmitter { this.emit('error', e); } }); + + this.client.bind('cfilter', async (blockHeight, filter) => { + try { + await this.checkFilter(blockHeight, filter); + } catch (e) { + this.emit('error', e); + } + }) } /** @@ -568,6 +577,45 @@ class WalletDB extends EventEmitter { return this.client.resetFilter(); } + async checkFilter (blockHeight, filter) { + const gcsKey = random.randomBytes(16); + + const piter = this.db.iterator({ + gte: layout.p.min(), + lte: layout.p.max() + }); + + await piter.each((key) => { + const [data] = layout.p.decode(key); + // todo: check filter + let match = filter.match(gcsKey, data); + console.log(match); + // this.filter.add(data); + }); + + const oiter = this.db.iterator({ + gte: layout.o.min(), + lte: layout.o.max() + }); + + await oiter.each((key) => { + const [hash, index] = layout.o.decode(key); + const outpoint = new Outpoint(hash, index); + const data = outpoint.toRaw(); + // console.log(data, "\n\n\n"); + + // todo: check filter + // this.filter.add(data); + }); + + // const match = filter.match(key, data); + + // if(match) { + // // todo: check if we already have this block + // // todo: get block + // } + } + /** * Backup the wallet db. * @param {String} path From a6a3a86203d4ab2e09c40fb36f3ba16948ee39e3 Mon Sep 17 00:00:00 2001 From: masterchief164 <63920595+masterchief164@users.noreply.github.com> Date: Wed, 14 Jun 2023 19:46:22 +0530 Subject: [PATCH 08/21] feat: headers first sync complete --- lib/blockchain/chain.js | 22 +++++++++++++--------- lib/net/pool.js | 4 +--- lib/node/neutrino.js | 4 +++- lib/protocol/networks.js | 2 +- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 907f2eb3b..ed27f2aca 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -583,7 +583,7 @@ class Chain extends AsyncEmitter { // UASF is now enforced (bip148) (mainnet-only). if (this.options.bip148 && this.network === Network.main) { if (witness !== thresholdStates.LOCKED_IN - && witness !== thresholdStates.ACTIVE) { + && witness !== thresholdStates.ACTIVE) { // The BIP148 MTP check is nonsensical in // that it includes the _current_ entry's // timestamp. This requires some hackery, @@ -2000,6 +2000,8 @@ class Chain extends AsyncEmitter { */ maybeSync() { + // console.log('maybeSync'); + // console.log(this.synced); if (this.synced) return; @@ -2008,12 +2010,14 @@ class Chain extends AsyncEmitter { return; } - if (this.tip.time < util.now() - this.network.block.maxTipAge) + if (this.options.neutrino && this.tip.time < util.now() - 24 * 60 * 60) return; - - if (!this.hasChainwork()) + else if (!this.options.neutrino && + this.tip.time < util.now() - this.network.block.maxTipAge) return; + if (!this.options.neutrino && !this.hasChainwork()) + return; this.synced = true; this.emit('full'); } @@ -2144,7 +2148,7 @@ class Chain extends AsyncEmitter { assert(hash); - for (;;) { + for (; ;) { const orphan = this.orphanMap.get(hash); if (!orphan) @@ -2221,8 +2225,8 @@ class Chain extends AsyncEmitter { return pow.bits; while (prev.height !== 0 - && prev.height % pow.retargetInterval !== 0 - && prev.bits === pow.bits) { + && prev.height % pow.retargetInterval !== 0 + && prev.bits === pow.bits) { const cache = this.getPrevCache(prev); if (cache) @@ -2455,7 +2459,7 @@ class Chain extends AsyncEmitter { const state = await this.getState(prev, deployment); if (state === thresholdStates.LOCKED_IN - || state === thresholdStates.STARTED) { + || state === thresholdStates.STARTED) { version |= 1 << deployment.bit; } } @@ -2640,7 +2644,7 @@ class ChainOptions { fromOptions(options) { if (!options.spv) { assert(options.blocks && typeof options.blocks === 'object', - 'Chain requires a blockstore.'); + 'Chain requires a blockstore.'); } this.blocks = options.blocks; diff --git a/lib/net/pool.js b/lib/net/pool.js index 0919112eb..a54b80ff7 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -786,6 +786,7 @@ class Pool extends EventEmitter { this.logger.info('No loader peer.'); return; } + this.chain.synced = false; peer.sendGetHeaders(locator); // this.emit('headersFull'); } @@ -2472,10 +2473,7 @@ class Pool extends EventEmitter { async addBlock(peer, block, flags) { const hash = block.hash(); - console.log(hash); - console.log('addBlock'); const unlock = await this.locker.lock(hash); - console.log('lockedHash'); try { return await this._addBlock(peer, block, flags); } finally { diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index b98db92ed..7b9519e56 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -164,7 +164,9 @@ class Neutrino extends Node { this.emit('reset', tip); }); - this.pool.on('headersFull', () => { + this.chain.on('full', () => { + if (this.chain.height === 0) + return; this.logger.info('Block Headers are fully synced'); console.log('Block Headers are fully synced \n\n\n\n\n'); // this.pool.startFilterCheckPtSync(); // TODO: Maybe implement this later diff --git a/lib/protocol/networks.js b/lib/protocol/networks.js index 16e6bedf7..8c2db9e8e 100644 --- a/lib/protocol/networks.js +++ b/lib/protocol/networks.js @@ -792,7 +792,7 @@ regtest.block = { bip66hash: null, pruneAfterHeight: 1000, keepBlocks: 10000, - maxTipAge: 0xffffffff, + maxTipAge: 24 * 60 * 60, slowHeight: 0 }; From 9739d205a5011368b6ef95d851ec8a4bb0e63bb0 Mon Sep 17 00:00:00 2001 From: masterchief164 <63920595+masterchief164@users.noreply.github.com> Date: Fri, 16 Jun 2023 00:43:44 +0530 Subject: [PATCH 09/21] feat: CFHeaders sync complete --- bin/neutrino | 10 ++++++-- lib/blockchain/chain.js | 13 +++++----- lib/net/pool.js | 54 ++++++++++++++++++++++++++++++++--------- lib/node/neutrino.js | 3 +-- 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/bin/neutrino b/bin/neutrino index 890e88bb6..df3b5fdb7 100755 --- a/bin/neutrino +++ b/bin/neutrino @@ -18,9 +18,7 @@ const node = new Neutrino({ logLevel: 'debug', db: 'leveldb', memory: false, - persistent: true, workers: true, - listen: true, loader: require }); @@ -42,3 +40,11 @@ if (!node.config.bool('no-wallet') && !node.has('walletdb')) { console.error(err.stack); process.exit(1); }); + +process.on('unhandledRejection', (err, promise) => { + throw err; +}); + +process.on('SIGINT', async () => { + await node.close(); +}); diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index ed27f2aca..050134719 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2000,8 +2000,6 @@ class Chain extends AsyncEmitter { */ maybeSync() { - // console.log('maybeSync'); - // console.log(this.synced); if (this.synced) return; @@ -2009,17 +2007,20 @@ class Chain extends AsyncEmitter { if (this.height < this.network.lastCheckpoint) return; } - - if (this.options.neutrino && this.tip.time < util.now() - 24 * 60 * 60) + if (this.options.neutrino && this.tip.time < 1686851917) + // TODO change this later return; else if (!this.options.neutrino && this.tip.time < util.now() - this.network.block.maxTipAge) return; - if (!this.options.neutrino && !this.hasChainwork()) + if (!this.hasChainwork()) return; this.synced = true; - this.emit('full'); + if (this.options.neutrino) + this.emit('headersFull'); + else + this.emit('full'); } /** diff --git a/lib/net/pool.js b/lib/net/pool.js index a54b80ff7..46b3f2153 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -223,9 +223,10 @@ class Pool extends EventEmitter { const tip = this.chain.tip; if (this.options.neutrino) { - // this.headerTip = tip; - this.headerChain.push(new HeaderEntry(tip.hash, tip.height)); - return; + this.headerChain.push(new HeaderEntry(tip.hash, tip.height)); + this.cfHeaderChain = new List(); + this.cfHeaderChain.push(new CFHeaderEntry(consensus.ZERO_HASH, 0)); + return; } if (tip.height < this.network.lastCheckpoint) { this.checkpoints = true; @@ -648,7 +649,7 @@ class Pool extends EventEmitter { peer.loader = true; this.peers.load = peer; - this.sendSync(peer); + this.sendSync(peer); this.emit('loader', peer); } @@ -730,8 +731,11 @@ class Pool extends EventEmitter { return; this.filterSyncing = true; - const startHeight = 0; - const stopHash = await this.chain.getHash(29); + const startHeight = 1; + const chainHeight = await this.chain.tip.height; + console.log('chainHeight', chainHeight); + const stopHeight = chainHeight > 2000 ? 2000 : chainHeight; + const stopHash = await this.chain.getHash(stopHeight); this.peers.load.sendGetCFHeaders( common.FILTERS.BASIC, startHeight, @@ -772,7 +776,7 @@ class Pool extends EventEmitter { async startHeadersSync() { if (!this.syncing) return; - + console.log('startHeadersSync'); let locator; try { locator = await this.chain.getLocator(); @@ -2156,7 +2160,7 @@ class Pool extends EventEmitter { assert(stopHash.equals(this.getcfheadersStopHash)); let previousFilterHeader = packet.previousFilterHeader; const filterHashes = packet.filterHashes; - assert(filterHashes.length <= 2000) + assert(filterHashes.length <= 2000); let blockHeight = await this.chain.getHeight(stopHash) - filterHashes.length; const stopHeight = await this.chain.getHeight(stopHash); @@ -2165,6 +2169,10 @@ class Pool extends EventEmitter { const basicFilter = new BasicFilter(); basicFilter._hash = filterHash; const filterHeader = basicFilter.header(previousFilterHeader); + const lastFilterHeader = this.cfHeaderChain.tail; + const cfHeaderEntry = new CFHeaderEntry( + filterHash, lastFilterHeader.height + 1); + this.cfHeaderChain.push(cfHeaderEntry); // todo: verify the filterHeader // todo: save the filterHeader previousFilterHeader = filterHeader; @@ -2172,7 +2180,15 @@ class Pool extends EventEmitter { blockHeight++; } await this.chain.db.saveNeutrinoState(); - this.emit('cfheaders'); + if (this.headerChain.tail.height <= stopHeight) + this.emit('cfheaders'); + else { + const nextStopHeight = stopHeight + 2000 < this.chain.height + ? stopHeight + 2000 : this.chain.height; + const nextStopHash = await this.chain.getHash(nextStopHeight); + this.getcfheadersStopHash = nextStopHash; + peer.sendGetCFHeaders(filterType, stopHeight + 1, nextStopHash); + } } async handleCFilters(peer, packet) { @@ -2191,7 +2207,8 @@ class Pool extends EventEmitter { assert(filterType === this.getcfheadersFilterType); const blockHeight = await this.chain.getHeight(blockHash); const stopHeight = await this.chain.getHeight(this.getcfiltersStopHash); - assert(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight); + assert(blockHeight >= this.getcfiltersStartHeight + && blockHeight <= stopHeight); // todo: save the filter const basicFilter = new BasicFilter(); @@ -2357,7 +2374,6 @@ class Pool extends EventEmitter { const last = this.headerChain.tail; const hash = header.hash(); const height = last.height + 1; - console.log('header'); if (!header.verify()) { this.logger.warning( @@ -2415,6 +2431,8 @@ class Pool extends EventEmitter { } // Request more headers. + if (this.chain.synced) + return; if (this.options.neutrino) peer.sendGetHeaders([node.hash]); else @@ -4724,6 +4742,20 @@ class HeaderEntry { } } +class CFHeaderEntry { + /** + * Create cfheader entry. + * @constructor + */ + + constructor(hash, height) { + this.hash = hash; + this.height = height; + this.prev = null; + this.next = null; + } +} + /* * Expose */ diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 7b9519e56..79e38e8e5 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -164,11 +164,10 @@ class Neutrino extends Node { this.emit('reset', tip); }); - this.chain.on('full', () => { + this.chain.on('headersFull', () => { if (this.chain.height === 0) return; this.logger.info('Block Headers are fully synced'); - console.log('Block Headers are fully synced \n\n\n\n\n'); // this.pool.startFilterCheckPtSync(); // TODO: Maybe implement this later this.pool.startFilterHeadersSync(); }); From 565d155110a49bc08ad685ede0d6dacd889bd189 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Sat, 24 Jun 2023 19:16:57 -0500 Subject: [PATCH 10/21] feat: get block from peer on request by wallet --- lib/blockchain/chain.js | 42 ++++++++++++++++++++++++++++++++++++++- lib/blockchain/chaindb.js | 13 ++++++++++-- lib/net/pool.js | 31 +++++++++++++++++++++++++---- lib/wallet/nodeclient.js | 4 ++++ lib/wallet/walletdb.js | 13 +++++------- 5 files changed, 88 insertions(+), 15 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 050134719..5f95247e0 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -64,6 +64,8 @@ class Chain extends AsyncEmitter { this.orphanMap = new BufferMap(); this.orphanPrev = new BufferMap(); + + this.getPrunedMap = new BufferMap(); } /** @@ -1368,7 +1370,18 @@ class Chain extends AsyncEmitter { } // Do we already have this block? - if (await this.hasEntry(hash)) { + const existingEntry = await this.getEntry(hash); + + // FOR EDUCATIONAL PURPOSES ONLY: save block without checking anything + if (existingEntry && this.getPrunedMap.has(hash)) { + block = block.toBlock(); + await this.db.updateNeutrinoSave(); + await this.db.save(existingEntry, block, new CoinView()); + await this.db.updateNeutrinoSave(); + return existingEntry; + } + + if (existingEntry) { this.logger.debug('Already have block: %h.', block.hash()); throw new VerifyError(block, 'duplicate', 'duplicate', 0); } @@ -1925,6 +1938,33 @@ class Chain extends AsyncEmitter { return this.db.getBlock(hash); } + async getBlockPeer(hash, filter) { + let block = await this.db.getBlock(hash); + if (block) { + let entry = await this.getEntry(hash); + assert(entry.hash.equals(hash)); + return block; + } else { + this.logger.warning('Block not found, attempting to download'); + + // Ensure hash not height + hash = await this.db.getHash(hash); + + // FOR EDUCATIONAL PURPOSES ONLY: flag block for re-downloading + const wait = new Promise((resolve, reject) => { + this.getPrunedMap.set(hash, resolve); + }); + + await this.emitAsync('getprunedblock', hash); + await wait; + block = await this.db.getBlock(hash); + let entry = await this.getEntry(hash); + assert(entry.hash.equals(hash)); + + return block; + } + } + /** * Retrieve a block from the database (not filled with coins). * @param {Hash} block diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 9698abff1..a2a53c5a9 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -47,6 +47,7 @@ class ChainDB { this.pending = null; this.current = null; this.neutrinoState = null; + this.neutrinoSave = false; this.cacheHash = new LRU(this.options.entryCache, null, BufferMap); this.cacheHeight = new LRU(this.options.entryCache); @@ -1007,7 +1008,7 @@ class ChainDB { */ async getRawBlock(block) { - if (this.options.spv) + if (this.options.spv && !this.options.neutrino) return null; const hash = await this.getHash(block); @@ -1156,6 +1157,14 @@ class ChainDB { * @returns {Promise} */ + async updateNeutrinoSave () { + if(this.neutrinoSave) { + this.neutrinoSave = false; + } else { + this.neutrinoSave = true; + } + } + async save(entry, block, view) { this.start(); try { @@ -1484,7 +1493,7 @@ class ChainDB { async saveBlock(entry, block, view) { const hash = block.hash(); - if (this.options.spv) + if (this.options.spv && !this.neutrinoSave) return; // Write actual block data. diff --git a/lib/net/pool.js b/lib/net/pool.js index 46b3f2153..c0fcc56e0 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -81,7 +81,7 @@ class Pool extends EventEmitter { this.pendingRefill = null; this.checkpoints = false; - this.neutrino = false; + this.neutrino = this.options.neutrino; this.headerChain = new List(); this.headerNext = null; this.headerTip = null; @@ -150,6 +150,16 @@ class Pool extends EventEmitter { this.handleBadOrphan('block', err, id); }); + this.chain.on('getprunedblock', async (hash) => { + // Find the first peer with a completed handshake + for (let peer = this.peers.head(); peer; peer = peer.next) { + if (!peer.handshake) + continue; + + await this.getBlock(peer, [hash]); + } + }); + if (this.mempool) { this.mempool.on('tx', (tx) => { this.emit('tx', tx); @@ -216,7 +226,8 @@ class Pool extends EventEmitter { if (!this.options.checkpoints && !this.options.neutrino) return; - this.checkpoints = false; + if (!this.options.neutrino) + this.checkpoints = false; this.headerTip = null; this.headerChain.reset(); this.headerNext = null; @@ -757,7 +768,10 @@ class Pool extends EventEmitter { this.filterSyncing = true; const startHeight = 0; - const stopHash = await this.chain.getHash(29); + const chainHeight = await this.chain.tip.height; + console.log('chainHeight', chainHeight); + const stopHeight = chainHeight > 2000 ? 2000 : chainHeight; + const stopHash = await this.chain.getHash(stopHeight); this.peers.load.sendGetCFilters( common.FILTERS.BASIC, startHeight, @@ -2213,7 +2227,8 @@ class Pool extends EventEmitter { // todo: save the filter const basicFilter = new BasicFilter(); const gcsFilter = basicFilter.fromNBytes(filter); - this.emit('cfilter', blockHeight, gcsFilter); + this.emit('cfilter', blockHash, gcsFilter); + // todo: send getcfilters } /** @@ -2538,6 +2553,14 @@ class Pool extends EventEmitter { } // Block was orphaned. + + const resolve = this.chain.getPrunedMap.get(hash); + if (resolve) { + this.logger.warning('Received pruned block by special request'); + this.chain.getPrunedMap.delete(hash); + resolve(); + } + if (!entry) { if (this.checkpoints) { this.logger.warning( diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 0a3215451..8fc9d1c0b 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -142,6 +142,10 @@ class NodeClient extends AsyncEmitter { return entry; } + async getBlockFromNode(hash, filter) { + await this.node.chain.getBlockPeer(hash, filter); + } + /** * Send a transaction. Do not wait for promise. * @param {TX} tx diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index ae237a038..31d8f91a2 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -577,20 +577,20 @@ class WalletDB extends EventEmitter { return this.client.resetFilter(); } - async checkFilter (blockHeight, filter) { - const gcsKey = random.randomBytes(16); + async checkFilter (blockHash, filter) { + const gcsKey = blockHash.slice(0, 16); const piter = this.db.iterator({ gte: layout.p.min(), lte: layout.p.max() }); - await piter.each((key) => { + await piter.each(async (key) => { const [data] = layout.p.decode(key); // todo: check filter let match = filter.match(gcsKey, data); - console.log(match); - // this.filter.add(data); + // console.log(match); + await this.client.getBlockFromNode(blockHash, filter); }); const oiter = this.db.iterator({ @@ -602,10 +602,7 @@ class WalletDB extends EventEmitter { const [hash, index] = layout.o.decode(key); const outpoint = new Outpoint(hash, index); const data = outpoint.toRaw(); - // console.log(data, "\n\n\n"); - // todo: check filter - // this.filter.add(data); }); // const match = filter.match(key, data); From dca040073346feaf323e5c00937c0b228a43e3df Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Sun, 25 Jun 2023 12:25:27 -0500 Subject: [PATCH 11/21] feat: CFiltersSync complete --- lib/net/pool.js | 46 +++++++++++++++++++++++++++++++++++------- lib/wallet/walletdb.js | 17 ++++++---------- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/lib/net/pool.js b/lib/net/pool.js index c0fcc56e0..ffc46df46 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -83,6 +83,7 @@ class Pool extends EventEmitter { this.checkpoints = false; this.neutrino = this.options.neutrino; this.headerChain = new List(); + this.filterHeight = 0; this.headerNext = null; this.headerTip = null; @@ -770,7 +771,7 @@ class Pool extends EventEmitter { const startHeight = 0; const chainHeight = await this.chain.tip.height; console.log('chainHeight', chainHeight); - const stopHeight = chainHeight > 2000 ? 2000 : chainHeight; + const stopHeight = chainHeight > 1000 ? 1000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); this.peers.load.sendGetCFilters( common.FILTERS.BASIC, @@ -806,7 +807,6 @@ class Pool extends EventEmitter { } this.chain.synced = false; peer.sendGetHeaders(locator); - // this.emit('headersFull'); } /** @@ -1759,6 +1759,10 @@ class Pool extends EventEmitter { if (this.options.hasWitness() && !peer.hasWitness()) return; + if(this.neutrino) { + this.startSync(); + } + // Request headers instead. if (this.checkpoints) return; @@ -2041,7 +2045,7 @@ class Pool extends EventEmitter { if (!stopHeight) return; - if (stopHeight - packet.startHeight >= common.MAX_CFILTERS) + if (stopHeight - packet.startHeight > common.MAX_CFILTERS) return; const indexer = this.getFilterIndexer(filtersByVal[packet.filterType]); @@ -2190,6 +2194,7 @@ class Pool extends EventEmitter { // todo: verify the filterHeader // todo: save the filterHeader previousFilterHeader = filterHeader; + // todo: add a function for this in chain.js this.chain.db.neutrinoState.headerHeight = blockHeight; blockHeight++; } @@ -2217,18 +2222,46 @@ class Pool extends EventEmitter { const filterType = packet.filterType; const filter = packet.filterBytes; + this.filterHeight += 1; + // todo: verify the filter assert(filterType === this.getcfheadersFilterType); const blockHeight = await this.chain.getHeight(blockHash); const stopHeight = await this.chain.getHeight(this.getcfiltersStopHash); - assert(blockHeight >= this.getcfiltersStartHeight - && blockHeight <= stopHeight); + + assert(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight); // todo: save the filter const basicFilter = new BasicFilter(); const gcsFilter = basicFilter.fromNBytes(filter); this.emit('cfilter', blockHash, gcsFilter); - // todo: send getcfilters + const startHeight = stopHeight + 1; + let nextStopHeight; + if(this.filterHeight == stopHeight + 1 && stopHeight < this.chain.height) { + if(startHeight + 1000 < this.chain.height) { + nextStopHeight = startHeight + 1000; + const stopHash = await this.chain.getHash(nextStopHeight); + this.getcfiltersStartHeight = startHeight; + this.getcfiltersStopHash = stopHash; + this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash + ); + } + else { + nextStopHeight = this.chain.height; + const stopHash = await this.chain.getHash(nextStopHeight); + this.getcfiltersStartHeight = startHeight; + this.getcfiltersStopHash = stopHash; + this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash + ); + return; + } + } } /** @@ -2345,7 +2378,6 @@ class Pool extends EventEmitter { return await this._handleHeaders(peer, packet); } finally { unlock(); - this.emit('headersFull'); } } diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 31d8f91a2..6e014b145 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -589,8 +589,8 @@ class WalletDB extends EventEmitter { const [data] = layout.p.decode(key); // todo: check filter let match = filter.match(gcsKey, data); - // console.log(match); - await this.client.getBlockFromNode(blockHash, filter); + if (match) + await this.client.getBlockFromNode(blockHash, filter); }); const oiter = this.db.iterator({ @@ -598,19 +598,14 @@ class WalletDB extends EventEmitter { lte: layout.o.max() }); - await oiter.each((key) => { + await oiter.each(async (key) => { const [hash, index] = layout.o.decode(key); const outpoint = new Outpoint(hash, index); const data = outpoint.toRaw(); - // todo: check filter + let match = filter.match(gcsKey, data); + if (match) + await this.client.getBlockFromNode(blockHash, filter); }); - - // const match = filter.match(key, data); - - // if(match) { - // // todo: check if we already have this block - // // todo: get block - // } } /** From d6297d4fc979b90e3f97f2ff4d41e15c3db5e8b5 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Mon, 26 Jun 2023 12:39:31 -0500 Subject: [PATCH 12/21] feat: wallet integration complete --- lib/client/node.js | 4 ++-- lib/node/http.js | 8 ++++---- lib/wallet/client.js | 4 ++-- lib/wallet/nodeclient.js | 4 ---- lib/wallet/nullclient.js | 2 +- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/client/node.js b/lib/client/node.js index 7822408d1..96b5ac81f 100644 --- a/lib/client/node.js +++ b/lib/client/node.js @@ -169,8 +169,8 @@ class NodeClient extends Client { return this.get(`/filter/${filter}`); } - checkKeyFilter(key, filter) { - return this.call('check key filter', key, filter); + getBlockPeer(hash, filter) { + return this.call('get block peer', hash, filter); } /** diff --git a/lib/node/http.js b/lib/node/http.js index 838aa9be7..1dfb2ddb2 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -498,12 +498,12 @@ class HTTP extends Server { return null; }); - socket.hook('check key filter', (...args) => { + socket.hook('get block peer', (...args) => { const valid = new Validator(args); - const key = valid.buf(0); + const hash = valid.hash(0); const filter = valid.buf(1); - return this.pool.checkKeyFilter(key, filter); - }); + return this.pool.getBlockPeer(hash, filter); + }) socket.hook('estimate fee', (...args) => { const valid = new Validator(args); diff --git a/lib/wallet/client.js b/lib/wallet/client.js index 93c521581..ae8681b6c 100644 --- a/lib/wallet/client.js +++ b/lib/wallet/client.js @@ -80,8 +80,8 @@ class WalletClient extends NodeClient { * @returns {Promise} */ - async checkKeyFilter(ring, filter) { - return super.checkKeyFilter(ring, filter); + async getBlockFromNode(hash, filter) { + return super.getBlockPeer(hash, filter); } async rescan(start) { diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 8fc9d1c0b..2b71ba61f 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -193,10 +193,6 @@ class NodeClient extends AsyncEmitter { * @returns {Promise} */ - async checkKeyFilter(ring, filter) { - this.node.pool.checkKeyFilter(ring, filter); - } - /** * Estimate smart fee. * @param {Number?} blocks diff --git a/lib/wallet/nullclient.js b/lib/wallet/nullclient.js index 68221a397..7a683b5ca 100644 --- a/lib/wallet/nullclient.js +++ b/lib/wallet/nullclient.js @@ -139,7 +139,7 @@ class NullClient extends EventEmitter { * @returns {Promise} */ - async checkKeyFilter(ring, filter) { + async getBlockFromNode(hash, filter) { ; } From 965c56c23fe1e6760dea78f37cc9cd9c0b264feb Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Tue, 27 Jun 2023 12:03:00 -0500 Subject: [PATCH 13/21] chore: added sync tests --- lib/blockchain/chain.js | 22 ++++++++++- lib/blockchain/chaindb.js | 4 +- lib/net/pool.js | 29 ++++++++------- lib/node/http.js | 2 +- lib/node/neutrino.js | 1 - lib/wallet/client.js | 2 - lib/wallet/nodeclient.js | 9 ++--- lib/wallet/nullclient.js | 2 - lib/wallet/walletdb.js | 7 ++-- test/neutrino-test.js | 78 +++++++++++++++++++++++++++++++++++++++ 10 files changed, 124 insertions(+), 32 deletions(-) create mode 100644 test/neutrino-test.js diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 5f95247e0..27d957f70 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -1804,6 +1804,24 @@ class Chain extends AsyncEmitter { return this.hasEntry(hash); } + getCFHeaderHeight() { + return this.db.neutrinoState.headerHeight; + } + + async saveCFHeaderHeight(height) { + this.db.neutrinoState.headerHeight = height; + await this.db.saveNeutrinoState(); + } + + getCFilterHeight() { + return this.db.neutrinoState.filterHeight; + } + + async saveCFilterHeight(height) { + this.db.neutrinoState.filterHeight = height; + await this.db.saveNeutrinoState(); + } + /** * Find the corresponding block entry by hash or height. * @param {Hash|Number} hash/height @@ -1941,7 +1959,7 @@ class Chain extends AsyncEmitter { async getBlockPeer(hash, filter) { let block = await this.db.getBlock(hash); if (block) { - let entry = await this.getEntry(hash); + const entry = await this.getEntry(hash); assert(entry.hash.equals(hash)); return block; } else { @@ -1958,7 +1976,7 @@ class Chain extends AsyncEmitter { await this.emitAsync('getprunedblock', hash); await wait; block = await this.db.getBlock(hash); - let entry = await this.getEntry(hash); + const entry = await this.getEntry(hash); assert(entry.hash.equals(hash)); return block; diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index a2a53c5a9..cacf1eaf8 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -1158,7 +1158,7 @@ class ChainDB { */ async updateNeutrinoSave () { - if(this.neutrinoSave) { + if (this.neutrinoSave) { this.neutrinoSave = false; } else { this.neutrinoSave = true; @@ -2006,7 +2006,7 @@ class NeutrinoState { static fromRaw(data) { const state = new NeutrinoState(); const br = bio.read(data); - state.headersHeight = br.readU32(); + state.headerHeight = br.readU32(); state.filterHeight = br.readU32(); return state; } diff --git a/lib/net/pool.js b/lib/net/pool.js index ffc46df46..58e8e031e 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -743,9 +743,9 @@ class Pool extends EventEmitter { return; this.filterSyncing = true; - const startHeight = 1; + const startHeight = this.chain.getCFHeaderHeight() + ? this.chain.getCFHeaderHeight() : 1; const chainHeight = await this.chain.tip.height; - console.log('chainHeight', chainHeight); const stopHeight = chainHeight > 2000 ? 2000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); this.peers.load.sendGetCFHeaders( @@ -768,9 +768,9 @@ class Pool extends EventEmitter { return; this.filterSyncing = true; - const startHeight = 0; + const startHeight = this.chain.getCFilterHeight() + ? this.chain.getCFilterHeight() : 0; const chainHeight = await this.chain.tip.height; - console.log('chainHeight', chainHeight); const stopHeight = chainHeight > 1000 ? 1000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); this.peers.load.sendGetCFilters( @@ -791,7 +791,6 @@ class Pool extends EventEmitter { async startHeadersSync() { if (!this.syncing) return; - console.log('startHeadersSync'); let locator; try { locator = await this.chain.getLocator(); @@ -1759,7 +1758,7 @@ class Pool extends EventEmitter { if (this.options.hasWitness() && !peer.hasWitness()) return; - if(this.neutrino) { + if (this.neutrino) { this.startSync(); } @@ -2195,10 +2194,9 @@ class Pool extends EventEmitter { // todo: save the filterHeader previousFilterHeader = filterHeader; // todo: add a function for this in chain.js - this.chain.db.neutrinoState.headerHeight = blockHeight; blockHeight++; + await this.chain.saveCFHeaderHeight(blockHeight); } - await this.chain.db.saveNeutrinoState(); if (this.headerChain.tail.height <= stopHeight) this.emit('cfheaders'); else { @@ -2227,9 +2225,11 @@ class Pool extends EventEmitter { // todo: verify the filter assert(filterType === this.getcfheadersFilterType); const blockHeight = await this.chain.getHeight(blockHash); + await this.chain.saveCFilterHeight(blockHeight); const stopHeight = await this.chain.getHeight(this.getcfiltersStopHash); - assert(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight); + assert(blockHeight >= this.getcfiltersStartHeight + && blockHeight <= stopHeight); // todo: save the filter const basicFilter = new BasicFilter(); @@ -2237,8 +2237,9 @@ class Pool extends EventEmitter { this.emit('cfilter', blockHash, gcsFilter); const startHeight = stopHeight + 1; let nextStopHeight; - if(this.filterHeight == stopHeight + 1 && stopHeight < this.chain.height) { - if(startHeight + 1000 < this.chain.height) { + if (this.filterHeight === stopHeight + 1 + && stopHeight < this.chain.height) { + if (startHeight + 1000 < this.chain.height) { nextStopHeight = startHeight + 1000; const stopHash = await this.chain.getHash(nextStopHeight); this.getcfiltersStartHeight = startHeight; @@ -2248,8 +2249,7 @@ class Pool extends EventEmitter { startHeight, stopHash ); - } - else { + } else { nextStopHeight = this.chain.height; const stopHash = await this.chain.getHash(nextStopHeight); this.getcfiltersStartHeight = startHeight; @@ -3645,6 +3645,9 @@ class Pool extends EventEmitter { */ getBlock(peer, hashes) { + if (this.options.neutrino) + return; + if (!this.opened) return; diff --git a/lib/node/http.js b/lib/node/http.js index 1dfb2ddb2..bed31bf20 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -503,7 +503,7 @@ class HTTP extends Server { const hash = valid.hash(0); const filter = valid.buf(1); return this.pool.getBlockPeer(hash, filter); - }) + }); socket.hook('estimate fee', (...args) => { const valid = new Validator(args); diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 79e38e8e5..333509850 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -177,7 +177,6 @@ class Neutrino extends Node { this.pool.startFilterSync(); }); - this.loadPlugins(); } diff --git a/lib/wallet/client.js b/lib/wallet/client.js index ae8681b6c..6a9a12a77 100644 --- a/lib/wallet/client.js +++ b/lib/wallet/client.js @@ -12,8 +12,6 @@ const NodeClient = require('../client/node'); const util = require('../utils/util'); const TX = require('../primitives/tx'); const hash256 = require('bcrypto/lib/hash256'); -const WalletKey = require('./walletkey'); -const Filter = require('../primitives/filter'); const parsers = { 'block connect': (entry, txs) => parseBlock(entry, txs), diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 2b71ba61f..8caee014e 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -8,8 +8,6 @@ const assert = require('bsert'); const AsyncEmitter = require('bevent'); -const WalletKey = require('./walletkey'); -const Filter = require('../primitives/filter'); /** * Node Client @@ -53,10 +51,11 @@ class NodeClient extends AsyncEmitter { }); this.node.pool.on('cfilter', async (blockHeight, filter) => { - if(!this.opened) return; - + if (!this.opened) + return; + await this.emitAsync('cfilter', blockHeight, filter); - }) + }); this.node.on('tx', (tx) => { if (!this.opened) diff --git a/lib/wallet/nullclient.js b/lib/wallet/nullclient.js index 7a683b5ca..a6387fc30 100644 --- a/lib/wallet/nullclient.js +++ b/lib/wallet/nullclient.js @@ -8,8 +8,6 @@ const assert = require('bsert'); const EventEmitter = require('events'); -const WalletKey = require('./walletkey'); -const Filter = require('../primitives/filter'); /** * Null Client diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 6e014b145..d37f9bdbc 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -23,7 +23,6 @@ const common = require('./common'); const Wallet = require('./wallet'); const Account = require('./account'); const Outpoint = require('../primitives/outpoint'); -const random = require('bcrypto/lib/random'); const layouts = require('./layout'); const records = require('./records'); const NullClient = require('./nullclient'); @@ -177,7 +176,7 @@ class WalletDB extends EventEmitter { } catch (e) { this.emit('error', e); } - }) + }); } /** @@ -588,7 +587,7 @@ class WalletDB extends EventEmitter { await piter.each(async (key) => { const [data] = layout.p.decode(key); // todo: check filter - let match = filter.match(gcsKey, data); + const match = filter.match(gcsKey, data); if (match) await this.client.getBlockFromNode(blockHash, filter); }); @@ -602,7 +601,7 @@ class WalletDB extends EventEmitter { const [hash, index] = layout.o.decode(key); const outpoint = new Outpoint(hash, index); const data = outpoint.toRaw(); - let match = filter.match(gcsKey, data); + const match = filter.match(gcsKey, data); if (match) await this.client.getBlockFromNode(blockHash, filter); }); diff --git a/test/neutrino-test.js b/test/neutrino-test.js new file mode 100644 index 000000000..05f160ce0 --- /dev/null +++ b/test/neutrino-test.js @@ -0,0 +1,78 @@ +'use strict'; + +const FullNode = require('../lib/node/fullnode'); +const NeutrinoNode = require('../lib/node/neutrino'); +const {forValue} = require('./util/common'); +const assert = require('bsert'); +describe('neutrino', function () { + this.timeout(10000); + + const node1 = new NeutrinoNode({ + network: 'regtest', + memory: true, + port: 10000, + httpPort: 20000, + neutrino: true, + only: '127.0.0.1' + }); + + const node2 = new FullNode({ + network: 'regtest', + memory: true, + listen: true, + indexFilter: true, + bip157: true + }); + + async function mineBlocks(n) { + while (n) { + const block = await node2.miner.mineBlock(); + await node2.chain.add(block); + n--; + } + await forValue(node1.chain, 'height', node2.chain.height); + } + + before(async function () { + const waitForConnection = new Promise((resolve, reject) => { + node1.pool.once('peer open', async (peer) => { + resolve(peer); + }); + }); + + await node1.open(); + await node2.open(); + await node1.connect(); + await node2.connect(); + node1.startSync(); + node2.startSync(); + await mineBlocks(200); + await waitForConnection; + }); + + after(async () => { + await node1.close(); + await node2.close(); + }); + + describe('getcfheaders', () => { + it('should getcfheaders', async () => { + const headerHeight = node1.chain.getCFHeaderHeight(); + assert.equal(headerHeight, node1.chain.height); + }); + }); + + describe('getcfilters', () => { + it('should getcfilters', async () => { + const filterHeight = node1.chain.getCFHeaderHeight(); + assert.equal(filterHeight, node1.chain.height); + }); + }); + + describe('getheaders', () => { + it('should getheaders', async () => { + await mineBlocks(30); + assert.equal(node1.chain.height, node2.chain.height); + }); + }); +}); From 4be302392eefd5d6c9e209c8d7e61da0fc70af42 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Wed, 28 Jun 2023 16:13:28 -0500 Subject: [PATCH 14/21] chore: Added rpc methods --- lib/blockchain/chain.js | 8 +-- lib/blockchain/chaindb.js | 10 ++++ lib/net/pool.js | 41 ++++++------- lib/node/fullnode.js | 24 +++++++- lib/node/neutrino.js | 8 +-- lib/node/rpc.js | 45 ++++++++++++++ lib/wallet/walletdb.js | 9 ++- test/neutrino-test.js | 19 +++--- test/wallet-neutrino-test.js | 113 +++++++++++++++++++++++++++++++++++ 9 files changed, 236 insertions(+), 41 deletions(-) create mode 100644 test/wallet-neutrino-test.js diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 27d957f70..07b89f138 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -1804,8 +1804,8 @@ class Chain extends AsyncEmitter { return this.hasEntry(hash); } - getCFHeaderHeight() { - return this.db.neutrinoState.headerHeight; + async getCFHeaderHeight() { + return await this.db.getCFHeaderHeight(); } async saveCFHeaderHeight(height) { @@ -1813,8 +1813,8 @@ class Chain extends AsyncEmitter { await this.db.saveNeutrinoState(); } - getCFilterHeight() { - return this.db.neutrinoState.filterHeight; + async getCFilterHeight() { + return await this.db.getCFilterHeight(); } async saveCFilterHeight(height) { diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index cacf1eaf8..d85d4e3b3 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -1698,6 +1698,16 @@ class ChainDB { return NeutrinoState.fromRaw(data); } + async getCFHeaderHeight() { + const state = await this.getNeutrinoState(); + return state.headerHeight; + } + + async getCFilterHeight() { + const state = await this.getNeutrinoState(); + return state.filterHeight; + } + /** * Save Neutrino State * @returns {void} diff --git a/lib/net/pool.js b/lib/net/pool.js index 58e8e031e..8878319c7 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -83,7 +83,6 @@ class Pool extends EventEmitter { this.checkpoints = false; this.neutrino = this.options.neutrino; this.headerChain = new List(); - this.filterHeight = 0; this.headerNext = null; this.headerTip = null; @@ -743,18 +742,18 @@ class Pool extends EventEmitter { return; this.filterSyncing = true; - const startHeight = this.chain.getCFHeaderHeight() - ? this.chain.getCFHeaderHeight() : 1; - const chainHeight = await this.chain.tip.height; + const cFHeaderHeight = await this.chain.getCFHeaderHeight(); + const startHeight = cFHeaderHeight + ? cFHeaderHeight : 1; + const chainHeight = await this.chain.height; const stopHeight = chainHeight > 2000 ? 2000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); - this.peers.load.sendGetCFHeaders( + this.getcfheadersFilterType = common.FILTERS.BASIC; + this.getcfheadersStopHash = stopHash; + await this.peers.load.sendGetCFHeaders( common.FILTERS.BASIC, startHeight, stopHash); - - this.getcfheadersFilterType = common.FILTERS.BASIC; - this.getcfheadersStopHash = stopHash; } /** @@ -768,18 +767,19 @@ class Pool extends EventEmitter { return; this.filterSyncing = true; - const startHeight = this.chain.getCFilterHeight() - ? this.chain.getCFilterHeight() : 0; - const chainHeight = await this.chain.tip.height; + const cFilterHeight = await this.chain.getCFilterHeight(); + const startHeight = cFilterHeight + ? cFilterHeight : 0; + const chainHeight = await this.chain.height; const stopHeight = chainHeight > 1000 ? 1000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); - this.peers.load.sendGetCFilters( - common.FILTERS.BASIC, - startHeight, - stopHash); this.getcfiltersFilterType = common.FILTERS.BASIC; this.getcfiltersStartHeight = startHeight; this.getcfiltersStopHash = stopHash; + await this.peers.load.sendGetCFilters( + common.FILTERS.BASIC, + startHeight, + stopHash); } /** @@ -2220,27 +2220,26 @@ class Pool extends EventEmitter { const filterType = packet.filterType; const filter = packet.filterBytes; - this.filterHeight += 1; - // todo: verify the filter assert(filterType === this.getcfheadersFilterType); const blockHeight = await this.chain.getHeight(blockHash); - await this.chain.saveCFilterHeight(blockHeight); const stopHeight = await this.chain.getHeight(this.getcfiltersStopHash); assert(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight); + await this.chain.saveCFilterHeight(blockHeight); + const cFilterHeight = await this.chain.getCFilterHeight(); // todo: save the filter const basicFilter = new BasicFilter(); const gcsFilter = basicFilter.fromNBytes(filter); this.emit('cfilter', blockHash, gcsFilter); const startHeight = stopHeight + 1; let nextStopHeight; - if (this.filterHeight === stopHeight + 1 + if (cFilterHeight === stopHeight && stopHeight < this.chain.height) { if (startHeight + 1000 < this.chain.height) { - nextStopHeight = startHeight + 1000; + nextStopHeight = stopHeight + 1000; const stopHash = await this.chain.getHash(nextStopHeight); this.getcfiltersStartHeight = startHeight; this.getcfiltersStopHash = stopHash; @@ -2261,6 +2260,8 @@ class Pool extends EventEmitter { ); return; } + } else { + this.logger.info('CFilters sync complete'); } } diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index cd373d3b9..035d62dea 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -647,7 +647,7 @@ class FullNode extends Node { } /** - * Retrieve compact filter by hash. + * Retrieve compact filter by hash/height. * @param {Hash | Number} hash * @param {Number} type * @returns {Promise} - Returns {@link Buffer}. @@ -667,6 +667,28 @@ class FullNode extends Node { return Indexer.getFilter(hash); } + + /** + * Retrieve compact filter by hash/height. + * @param {Hash | Number} hash + * @param {Number} type + * @returns {Promise} - Returns {@link Buffer}. + */ + + async getBlockFilterHeader(hash, filterType) { + const Indexer = this.filterIndexers.get(filterType); + + if (!Indexer) + return null; + + if (typeof hash === 'number') + hash = await this.chain.getHash(hash); + + if (!hash) + return null; + + return Indexer.getFilterHeader(hash); + } } /* diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 333509850..d3a2c7671 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -164,17 +164,17 @@ class Neutrino extends Node { this.emit('reset', tip); }); - this.chain.on('headersFull', () => { + this.chain.on('headersFull', async () => { if (this.chain.height === 0) return; this.logger.info('Block Headers are fully synced'); // this.pool.startFilterCheckPtSync(); // TODO: Maybe implement this later - this.pool.startFilterHeadersSync(); + await this.pool.startFilterHeadersSync(); }); - this.pool.on('cfheaders', () => { + this.pool.on('cfheaders', async () => { this.logger.info('CF Headers Synced'); - this.pool.startFilterSync(); + await this.pool.startFilterSync(); }); this.loadPlugins(); diff --git a/lib/node/rpc.js b/lib/node/rpc.js index f200f216b..1929d93bf 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -152,11 +152,14 @@ class RPC extends RPCBase { this.add('getblockchaininfo', this.getBlockchainInfo); this.add('getbestblockhash', this.getBestBlockHash); this.add('getblockcount', this.getBlockCount); + this.add('getfiltercount', this.getFilterCount); + this.add('getfilterheadercount', this.getFilterHeaderCount); this.add('getblock', this.getBlock); this.add('getblockbyheight', this.getBlockByHeight); this.add('getblockhash', this.getBlockHash); this.add('getblockheader', this.getBlockHeader); this.add('getblockfilter', this.getBlockFilter); + this.add('getblockfilterheader', this.getBlockFilterHeader); this.add('getchaintips', this.getChainTips); this.add('getdifficulty', this.getDifficulty); this.add('getmempoolancestors', this.getMempoolAncestors); @@ -625,6 +628,22 @@ class RPC extends RPCBase { return this.chain.tip.height; } + async getFilterCount(args, help) { + if (help || args.length !== 0) + throw new RPCError(errs.MISC_ERROR, 'getfiltercount'); + + const height = await this.chain.getCFilterHeight(); + return height; + } + + async getFilterHeaderCount(args, help) { + if (help || args.length !== 0) + throw new RPCError(errs.MISC_ERROR, 'getfilterheadercount'); + + const height = await this.chain.getCFHeaderHeight(); + return height; + } + async getBlock(args, help) { if (help || args.length < 1 || args.length > 3) throw new RPCError(errs.MISC_ERROR, 'getblock "hash" ( verbose )'); @@ -765,6 +784,32 @@ class RPC extends RPCBase { return filter.toJSON(); } + async getBlockFilterHeader(args, help) { + if (help || args.length < 1 || args.length > 2) { + throw new RPCError(errs.MISC_ERROR, + 'getblockfilterheader "hash" ( "type" )'); + } + + const valid = new Validator(args); + const hash = valid.brhash(0); + const filterName = valid.str(1, 'BASIC').toUpperCase(); + + const filterType = filters[filterName]; + + if (!hash) + throw new RPCError(errs.MISC_ERROR, 'Invalid block hash.'); + + if (!filterType) + throw new RPCError(errs.MISC_ERROR, 'Filter type not supported'); + + const filterHeader = await this.node.getBlockFilterHeader(hash, filterName); + + if (!filterHeader) + throw new RPCError(errs.MISC_ERROR, 'Block filter header not found.'); + + return filterHeader; + } + async getChainTips(args, help) { if (help || args.length !== 0) throw new RPCError(errs.MISC_ERROR, 'getchaintips'); diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index d37f9bdbc..53b1f11da 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -586,10 +586,11 @@ class WalletDB extends EventEmitter { await piter.each(async (key) => { const [data] = layout.p.decode(key); - // todo: check filter const match = filter.match(gcsKey, data); - if (match) + if (match) { await this.client.getBlockFromNode(blockHash, filter); + return; + } }); const oiter = this.db.iterator({ @@ -602,8 +603,10 @@ class WalletDB extends EventEmitter { const outpoint = new Outpoint(hash, index); const data = outpoint.toRaw(); const match = filter.match(gcsKey, data); - if (match) + if (match) { await this.client.getBlockFromNode(blockHash, filter); + return; + } }); } diff --git a/test/neutrino-test.js b/test/neutrino-test.js index 05f160ce0..796dbf88f 100644 --- a/test/neutrino-test.js +++ b/test/neutrino-test.js @@ -55,24 +55,25 @@ describe('neutrino', function () { await node2.close(); }); + describe('getheaders', () => { + it('should getheaders', async () => { + assert.equal(node1.chain.height, node2.chain.height); + }); + }); + describe('getcfheaders', () => { it('should getcfheaders', async () => { - const headerHeight = node1.chain.getCFHeaderHeight(); + await new Promise(resolve => setTimeout(resolve, 400)); + const headerHeight = await node1.chain.getCFHeaderHeight(); assert.equal(headerHeight, node1.chain.height); }); }); describe('getcfilters', () => { it('should getcfilters', async () => { - const filterHeight = node1.chain.getCFHeaderHeight(); + await new Promise(resolve => setTimeout(resolve, 400)); + const filterHeight = await node1.chain.getCFilterHeight(); assert.equal(filterHeight, node1.chain.height); }); }); - - describe('getheaders', () => { - it('should getheaders', async () => { - await mineBlocks(30); - assert.equal(node1.chain.height, node2.chain.height); - }); - }); }); diff --git a/test/wallet-neutrino-test.js b/test/wallet-neutrino-test.js new file mode 100644 index 000000000..2385ada5e --- /dev/null +++ b/test/wallet-neutrino-test.js @@ -0,0 +1,113 @@ +'use strict'; + +const FullNode = require('../lib/node/fullnode'); +const Neutrino = require('../lib/node/neutrino'); +const MTX = require('../lib/primitives/mtx'); +const assert = require('bsert'); +const { consensus } = require('../lib/protocol'); +const { forValue } = require('./util/common'); + +const node1 = new FullNode({ + network: 'regtest', + memory: true, + listen: true, + indexFilter: true, + plugins: [require('../lib/wallet/plugin')], + bip157: true +}); + +const node2 = new Neutrino({ + network: 'regtest', + memory: true, + port: 10000, + httpPort: 20000, + neutrino: true, + only: '127.0.0.1', + plugins: [require('../lib/wallet/plugin')], + env: { + 'BCOIN_WALLET_HTTP_PORT': '12221' + } +}); + +const chain = node1.chain; +const miner = node1.miner; +const wdb1 = node1.require('walletdb').wdb; +const wdb2 = node2.require('walletdb').wdb; + +let wallet1 = null; +let wallet2 = null; +let cb = null; + +async function mineBlock(tx) { + const job = await miner.createJob(); + + if (!tx) + return await job.mineAsync(); + + const spend = new MTX(); + spend.addTX(tx, 0); + + spend.addOutput(await wallet2.receiveAddress(), 25 * 1e8); + spend.addOutput(await wallet2.changeAddress(), 5 * 1e8); + + spend.setLocktime(chain.height); + await wallet1.sign(spend); + + job.addTX(spend.toTX(), spend.view); + job.refresh(); + + return await job.mineAsync(); +} + +describe('wallet-neutrino', function() { + it('should open chain and miner', async () => { + miner.mempool = null; + consensus.COINBASE_MATURITY = 0; + await node1.open(); + await node2.open(); + }); + + it('should open walletdb', async () => { + wallet1 = await wdb1.create(); + wallet2 = await wdb2.create(); + miner.addresses.length = 0; + miner.addAddress(await wallet1.receiveAddress()); + }); + + it('should mine 10 blocks', async () => { + let n = 10; + while (n) { + const block = await mineBlock(cb); + cb = block.txs[0]; + await node1.chain.add(block); + n--; + } + }); + + it('should connect nodes', async () => { + await node1.connect(); + await node2.connect(); + }); + + it('should start sync chain', async () => { + node1.startSync(); + node2.startSync(); + await forValue(node2.chain, 'height', node1.chain.height); + }); + + it('should getheaders', async () => { + assert.equal(node1.chain.height, node2.chain.height); + }); + + it('should getcfheaders', async () => { + await new Promise(resolve => setTimeout(resolve, 400)); + const headerHeight = await node2.chain.getCFHeaderHeight(); + assert.equal(headerHeight, node2.chain.height); + }); + + it('should getcfilters', async () => { + await new Promise(resolve => setTimeout(resolve, 400)); + const filterHeight = await node2.chain.getCFilterHeight(); + assert.equal(filterHeight, node2.chain.height); + }); +}); From 4832f54465466716cf26be1b36a0c071c327c267 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Thu, 6 Jul 2023 14:53:56 -0500 Subject: [PATCH 15/21] feat: save filters and added wallet test --- bin/neutrino | 2 - lib/blockchain/layout.js | 1 + lib/indexer/filterindexer.js | 43 +++++++++++++++++++++ lib/indexer/indexer.js | 5 +++ lib/net/pool.js | 64 ++++++++++++++++++++---------- lib/node/neutrino.js | 14 ++++++- lib/wallet/walletdb.js | 34 +++++++--------- test/wallet-neutrino-test.js | 75 ++++++++++++++++++++++++++++-------- 8 files changed, 177 insertions(+), 61 deletions(-) diff --git a/bin/neutrino b/bin/neutrino index df3b5fdb7..d42ec71c0 100755 --- a/bin/neutrino +++ b/bin/neutrino @@ -5,8 +5,6 @@ console.log('Starting bcoin'); process.title = 'bcoin'; const Neutrino = require('../lib/node/neutrino'); -const Outpoint = require('../lib/primitives/outpoint'); -const assert = require('assert'); // Doubt in db const node = new Neutrino({ diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js index 1443e3170..532ccb050 100644 --- a/lib/blockchain/layout.js +++ b/lib/blockchain/layout.js @@ -35,6 +35,7 @@ const layout = { R: bdb.key('R'), D: bdb.key('D'), N: bdb.key('N'), + F: bdb.key('H', ['hash256']), e: bdb.key('e', ['hash256']), h: bdb.key('h', ['hash256']), H: bdb.key('H', ['uint32']), diff --git a/lib/indexer/filterindexer.js b/lib/indexer/filterindexer.js index 97265253b..ae88af139 100644 --- a/lib/indexer/filterindexer.js +++ b/lib/indexer/filterindexer.js @@ -85,6 +85,49 @@ class FilterIndexer extends Indexer { this.put(layout.f.encode(hash), gcsFilter.hash()); } + /** + * save filter header + * @param {Hash} blockHash + * @param {Hash} filterHeader + * @param {Hash} filterHash + * @returns {Promise} + */ + + async saveFilterHeader(blockHash, filterHeader, filterHash) { + assert(blockHash); + assert(filterHeader); + assert(filterHash); + + const filter = new Filter(); + filter.header = filterHeader; + + await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); + // console.log(layout.f.encode(blockHash)); + this.put(layout.f.encode(blockHash), filterHash); + } + + /** + * Save filter + * @param {Hash} blockHash + * @param {BasicFilter} basicFilter + * @param {Hash} filterHeader + * @returns {Promise} + */ + + async saveFilter(blockHash, basicFilter, filterHeader) { + assert(blockHash); + assert(basicFilter); + assert(filterHeader); + + const filter = new Filter(); + filter.filter = basicFilter.toRaw(); + filter.header = filterHeader; + + await this.blocks.writeFilter(blockHash, filter.toRaw(), this.filterType); + // console.log(layout.f.encode(blockHash)); + this.put(layout.f.encode(blockHash), basicFilter.hash()); + } + /** * Prune compact filters. * @private diff --git a/lib/indexer/indexer.js b/lib/indexer/indexer.js index 97d85f76b..c095584ca 100644 --- a/lib/indexer/indexer.js +++ b/lib/indexer/indexer.js @@ -292,6 +292,11 @@ class Indexer extends EventEmitter { */ async _syncBlock(meta, block, view) { + if (this.chain.options.neutrino) { + if (!this.batch) + this.start(); + return true; + } // In the case that the next block is being // connected or the current block disconnected // use the block and view being passed directly, diff --git a/lib/net/pool.js b/lib/net/pool.js index 8878319c7..5c75a0966 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -769,7 +769,7 @@ class Pool extends EventEmitter { this.filterSyncing = true; const cFilterHeight = await this.chain.getCFilterHeight(); const startHeight = cFilterHeight - ? cFilterHeight : 0; + ? cFilterHeight : 1; const chainHeight = await this.chain.height; const stopHeight = chainHeight > 1000 ? 1000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); @@ -931,6 +931,8 @@ class Pool extends EventEmitter { peer.blockTime = Date.now(); if (this.options.neutrino) { peer.sendGetHeaders(locator); + if (!this.syncing) + this.startFilterHeadersSync(); return true; } if (this.checkpoints) { @@ -1759,7 +1761,11 @@ class Pool extends EventEmitter { return; if (this.neutrino) { - this.startSync(); + const locator = await this.chain.getLocator(); + this.sendLocator(locator, peer); + if (!this.syncing) + this.startFilterHeadersSync(); + return; } // Request headers instead. @@ -2172,17 +2178,22 @@ class Pool extends EventEmitter { } const filterType = packet.filterType; - assert(filterType === this.getcfheadersFilterType); + + if (filterType !== this.getcfheadersFilterType) { + peer.ban(); + peer.destroy(); + return; + } + const stopHash = packet.stopHash; assert(stopHash.equals(this.getcfheadersStopHash)); let previousFilterHeader = packet.previousFilterHeader; const filterHashes = packet.filterHashes; - assert(filterHashes.length <= 2000); let blockHeight = await this.chain.getHeight(stopHash) - - filterHashes.length; + - filterHashes.length + 1; const stopHeight = await this.chain.getHeight(stopHash); for (const filterHash of filterHashes) { - assert(blockHeight < stopHeight); + assert(blockHeight <= stopHeight); const basicFilter = new BasicFilter(); basicFilter._hash = filterHash; const filterHeader = basicFilter.header(previousFilterHeader); @@ -2190,12 +2201,12 @@ class Pool extends EventEmitter { const cfHeaderEntry = new CFHeaderEntry( filterHash, lastFilterHeader.height + 1); this.cfHeaderChain.push(cfHeaderEntry); - // todo: verify the filterHeader - // todo: save the filterHeader + const blockHash = await this.chain.getHash(blockHeight); + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + await indexer.saveFilterHeader(blockHash, filterHeader, filterHash); previousFilterHeader = filterHeader; - // todo: add a function for this in chain.js - blockHeight++; await this.chain.saveCFHeaderHeight(blockHeight); + blockHeight++; } if (this.headerChain.tail.height <= stopHeight) this.emit('cfheaders'); @@ -2220,20 +2231,31 @@ class Pool extends EventEmitter { const filterType = packet.filterType; const filter = packet.filterBytes; - // todo: verify the filter - assert(filterType === this.getcfheadersFilterType); + if (filterType !== this.getcfheadersFilterType) { + peer.ban(); + peer.destroy(); + return; + } + const blockHeight = await this.chain.getHeight(blockHash); const stopHeight = await this.chain.getHeight(this.getcfiltersStopHash); assert(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight); - await this.chain.saveCFilterHeight(blockHeight); - const cFilterHeight = await this.chain.getCFilterHeight(); - // todo: save the filter - const basicFilter = new BasicFilter(); - const gcsFilter = basicFilter.fromNBytes(filter); - this.emit('cfilter', blockHash, gcsFilter); + const cFilterHeight = await this.chain.getCFilterHeight(); + + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + + const filterHeader = await indexer.getFilterHeader(blockHash); + + const basicFilter = new BasicFilter(); + const gcsFilter = basicFilter.fromNBytes(filter); + + await indexer.saveFilter(blockHash, gcsFilter, filterHeader); + + this.emit('cfilter', blockHash, gcsFilter); + await this.chain.saveCFilterHeight(blockHeight); const startHeight = stopHeight + 1; let nextStopHeight; if (cFilterHeight === stopHeight @@ -2436,7 +2458,7 @@ class Pool extends EventEmitter { this.logger.warning( 'Peer sent a bad header chain (%s).', peer.hostname()); - peer.destroy(); + peer.increaseBan(10); return; } @@ -2459,7 +2481,7 @@ class Pool extends EventEmitter { this.headerChain.push(node); if (this.options.neutrino) - await this._addBlock(peer, header, chainCommon.flags.VERIFY_NONE); + await this._addBlock(peer, header, chainCommon.flags.VERIFY_POW); } this.logger.debug( @@ -2472,7 +2494,7 @@ class Pool extends EventEmitter { peer.blockTime = Date.now(); // Request the blocks we just added. - if (checkpoint) { + if (checkpoint && !this.options.neutrino) { this.headerChain.shift(); this.resolveHeaders(peer); return; diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index d3a2c7671..4c7ccf663 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -91,6 +91,7 @@ class Neutrino extends Node { chain: this.chain, prefix: this.config.prefix, checkpoints: true, + filterIndexers: this.filterIndexers, proxy: this.config.str('proxy'), onion: this.config.bool('onion'), upnp: this.config.bool('upnp'), @@ -168,12 +169,13 @@ class Neutrino extends Node { if (this.chain.height === 0) return; this.logger.info('Block Headers are fully synced'); - // this.pool.startFilterCheckPtSync(); // TODO: Maybe implement this later await this.pool.startFilterHeadersSync(); }); this.pool.on('cfheaders', async () => { - this.logger.info('CF Headers Synced'); + if (this.chain.height === 0) + return; + this.logger.info('Filter Headers are fully synced'); await this.pool.startFilterSync(); }); @@ -200,6 +202,10 @@ class Neutrino extends Node { await this.http.open(); await this.handleOpen(); + for (const filterindex of this.filterIndexers.values()) { + await filterindex.open(); + } + this.logger.info('Node is loaded.'); } @@ -220,6 +226,10 @@ class Neutrino extends Node { await this.pool.close(); await this.chain.close(); await this.handleClose(); + + for (const filterindex of this.filterIndexers.values()) { + await filterindex.close(); + } } /** diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 53b1f11da..a4134caf9 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -26,6 +26,7 @@ const Outpoint = require('../primitives/outpoint'); const layouts = require('./layout'); const records = require('./records'); const NullClient = require('./nullclient'); +const Script = require('../script/script'); const layout = layouts.wdb; const tlayout = layouts.txdb; @@ -65,6 +66,7 @@ class WalletDB extends EventEmitter { this.state = new ChainState(); this.confirming = false; this.height = 0; + this.filterHeight = 0; this.wallets = new Map(); this.depth = 0; this.rescanning = false; @@ -577,7 +579,9 @@ class WalletDB extends EventEmitter { } async checkFilter (blockHash, filter) { - const gcsKey = blockHash.slice(0, 16); + // script pub keys + this.filterHeight = this.filterHeight + 1; + const gcsKey = blockHash.reverse().slice(0, 16); const piter = this.db.iterator({ gte: layout.p.min(), @@ -586,26 +590,14 @@ class WalletDB extends EventEmitter { await piter.each(async (key) => { const [data] = layout.p.decode(key); - const match = filter.match(gcsKey, data); - if (match) { - await this.client.getBlockFromNode(blockHash, filter); - return; - } - }); - - const oiter = this.db.iterator({ - gte: layout.o.min(), - lte: layout.o.max() - }); - - await oiter.each(async (key) => { - const [hash, index] = layout.o.decode(key); - const outpoint = new Outpoint(hash, index); - const data = outpoint.toRaw(); - const match = filter.match(gcsKey, data); - if (match) { - await this.client.getBlockFromNode(blockHash, filter); - return; + // address fromHash toScript + if (data.length === 20) { + const script = Script.fromPubkeyhash(data); + const match = filter.match(gcsKey, script); + if (match) { + await this.client.getBlockFromNode(blockHash, filter); + return; + } } }); } diff --git a/test/wallet-neutrino-test.js b/test/wallet-neutrino-test.js index 2385ada5e..f0be7547c 100644 --- a/test/wallet-neutrino-test.js +++ b/test/wallet-neutrino-test.js @@ -4,8 +4,10 @@ const FullNode = require('../lib/node/fullnode'); const Neutrino = require('../lib/node/neutrino'); const MTX = require('../lib/primitives/mtx'); const assert = require('bsert'); -const { consensus } = require('../lib/protocol'); const { forValue } = require('./util/common'); +const BasicFilter = require('../lib/golomb/basicFilter'); +const Script = require('../lib/script/script'); +const Address = require('../lib/primitives/address'); const node1 = new FullNode({ network: 'regtest', @@ -36,9 +38,11 @@ const wdb2 = node2.require('walletdb').wdb; let wallet1 = null; let wallet2 = null; -let cb = null; +const fwAddresses = []; +const nwAddresses = []; -async function mineBlock(tx) { +async function mineBlock(tx, address) { + console.log('address', address); const job = await miner.createJob(); if (!tx) @@ -46,9 +50,7 @@ async function mineBlock(tx) { const spend = new MTX(); spend.addTX(tx, 0); - - spend.addOutput(await wallet2.receiveAddress(), 25 * 1e8); - spend.addOutput(await wallet2.changeAddress(), 5 * 1e8); + spend.addOutput(address, 50000); spend.setLocktime(chain.height); await wallet1.sign(spend); @@ -62,7 +64,6 @@ async function mineBlock(tx) { describe('wallet-neutrino', function() { it('should open chain and miner', async () => { miner.mempool = null; - consensus.COINBASE_MATURITY = 0; await node1.open(); await node2.open(); }); @@ -70,18 +71,42 @@ describe('wallet-neutrino', function() { it('should open walletdb', async () => { wallet1 = await wdb1.create(); wallet2 = await wdb2.create(); - miner.addresses.length = 0; - miner.addAddress(await wallet1.receiveAddress()); + }); + + it('should create accounts', async () => { + await wallet1.createAccount('fw'); + await wallet2.createAccount('nw'); + }); + + it('should generate addresses', async () => { + miner.addresses.length = 0; + for (let i = 0; i < 10; i++) { + const key = await wallet1.createReceive(0); + const address = key.getAddress().toString(node1.network.type); + // console.log(address); + fwAddresses.push(address); + miner.addAddress(address); + } + for (let i = 0; i < 10; i++) { + const key = await wallet2.createReceive(0); + const address = key.getAddress().toString(node2.network.type); + nwAddresses.push(address); + } }); it('should mine 10 blocks', async () => { - let n = 10; - while (n) { - const block = await mineBlock(cb); - cb = block.txs[0]; - await node1.chain.add(block); - n--; + for (const address of fwAddresses) { + for (let i = 0; i < 2; i++) { + const block = await mineBlock(null, address); + await chain.add(block); } + } + for (const address of nwAddresses) { + for (let i = 0; i < 2; i++) { + const block = await mineBlock(null, address); + await chain.add(block); + } + } }); it('should connect nodes', async () => { @@ -110,4 +135,24 @@ describe('wallet-neutrino', function() { const filterHeight = await node2.chain.getCFilterHeight(); assert.equal(filterHeight, node2.chain.height); }); + + it('should send filters to wallet', async () => { + assert.equal(wdb2.filterHeight, node2.chain.height); + }); + + it('should match the filters', async () => { + const filterIndexer = node2.filterIndexers.get('BASIC'); + for (let i = 0; i < fwAddresses.length; i++) { + const hash = await node2.chain.getHash(i); + const filter = await filterIndexer.getFilter(hash); + const basicFilter = new BasicFilter(); + const gcs = basicFilter.fromNBytes(filter.filter); + const key = hash.slice(0, 16); + const address = Address.fromString(fwAddresses[i], node2.network.type); + const script = Script.fromAddress(address); + // console.log(address.hash); + console.log(script.toRaw()); + // assert(gcs.match(key, script.)); + } + }); }); From fdc3f91ad6ed45b6820c20451ee006ccc98c19b4 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Fri, 7 Jul 2023 20:29:26 -0500 Subject: [PATCH 16/21] fix: checkFilter() --- lib/net/pool.js | 12 +++--------- lib/wallet/walletdb.js | 6 ++---- test/wallet-neutrino-test.js | 8 ++------ 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/lib/net/pool.js b/lib/net/pool.js index 5c75a0966..869746e2f 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -744,7 +744,7 @@ class Pool extends EventEmitter { this.filterSyncing = true; const cFHeaderHeight = await this.chain.getCFHeaderHeight(); const startHeight = cFHeaderHeight - ? cFHeaderHeight : 1; + ? cFHeaderHeight + 1 : 1; const chainHeight = await this.chain.height; const stopHeight = chainHeight > 2000 ? 2000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); @@ -769,7 +769,7 @@ class Pool extends EventEmitter { this.filterSyncing = true; const cFilterHeight = await this.chain.getCFilterHeight(); const startHeight = cFilterHeight - ? cFilterHeight : 1; + ? cFilterHeight + 1 : 1; const chainHeight = await this.chain.height; const stopHeight = chainHeight > 1000 ? 1000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); @@ -1761,10 +1761,7 @@ class Pool extends EventEmitter { return; if (this.neutrino) { - const locator = await this.chain.getLocator(); - this.sendLocator(locator, peer); - if (!this.syncing) - this.startFilterHeadersSync(); + this.startSync(); return; } @@ -3668,9 +3665,6 @@ class Pool extends EventEmitter { */ getBlock(peer, hashes) { - if (this.options.neutrino) - return; - if (!this.opened) return; diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index a4134caf9..275ee07b3 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -579,9 +579,8 @@ class WalletDB extends EventEmitter { } async checkFilter (blockHash, filter) { - // script pub keys this.filterHeight = this.filterHeight + 1; - const gcsKey = blockHash.reverse().slice(0, 16); + const gcsKey = blockHash.slice(0, 16); const piter = this.db.iterator({ gte: layout.p.min(), @@ -590,10 +589,9 @@ class WalletDB extends EventEmitter { await piter.each(async (key) => { const [data] = layout.p.decode(key); - // address fromHash toScript if (data.length === 20) { const script = Script.fromPubkeyhash(data); - const match = filter.match(gcsKey, script); + const match = filter.match(gcsKey, script.raw); if (match) { await this.client.getBlockFromNode(blockHash, filter); return; diff --git a/test/wallet-neutrino-test.js b/test/wallet-neutrino-test.js index f0be7547c..02e040267 100644 --- a/test/wallet-neutrino-test.js +++ b/test/wallet-neutrino-test.js @@ -42,7 +42,6 @@ const fwAddresses = []; const nwAddresses = []; async function mineBlock(tx, address) { - console.log('address', address); const job = await miner.createJob(); if (!tx) @@ -83,7 +82,6 @@ describe('wallet-neutrino', function() { for (let i = 0; i < 10; i++) { const key = await wallet1.createReceive(0); const address = key.getAddress().toString(node1.network.type); - // console.log(address); fwAddresses.push(address); miner.addAddress(address); } @@ -148,11 +146,9 @@ describe('wallet-neutrino', function() { const basicFilter = new BasicFilter(); const gcs = basicFilter.fromNBytes(filter.filter); const key = hash.slice(0, 16); - const address = Address.fromString(fwAddresses[i], node2.network.type); + const address = Address.fromString(fwAddresses[i], node1.network.type); const script = Script.fromAddress(address); - // console.log(address.hash); - console.log(script.toRaw()); - // assert(gcs.match(key, script.)); + assert(gcs.match(key, script.raw)); } }); }); From d374238094e6eb64585b7f94b19cda056b813f8d Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Sat, 8 Jul 2023 04:25:21 -0500 Subject: [PATCH 17/21] test: save filters --- lib/blockchain/chain.js | 1 + lib/wallet/nodeclient.js | 7 +++++++ test/neutrino-test.js | 14 ++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 07b89f138..18f357d94 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -1979,6 +1979,7 @@ class Chain extends AsyncEmitter { const entry = await this.getEntry(hash); assert(entry.hash.equals(hash)); + this.emit('getblockpeer', entry, block); return block; } } diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index 8caee014e..48b8c6e33 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -43,6 +43,13 @@ class NodeClient extends AsyncEmitter { await this.emitAsync('block connect', entry, block.txs); }); + this.node.chain.on('getblockpeer', async (entry, block) => { + if (!this.opened) + return; + + await this.emitAsync('block connect', entry, block.txs); + }); + this.node.chain.on('disconnect', async (entry, block) => { if (!this.opened) return; diff --git a/test/neutrino-test.js b/test/neutrino-test.js index 796dbf88f..32a36871a 100644 --- a/test/neutrino-test.js +++ b/test/neutrino-test.js @@ -76,4 +76,18 @@ describe('neutrino', function () { assert.equal(filterHeight, node1.chain.height); }); }); + + describe('save filters', () => { + it('should save filters correctly', async () => { + const filterIndexer = node1.filterIndexers.get('BASIC'); + for (let i = 0; i < node1.chain.height; i++) { + const hash = await node1.chain.getHash(i); + const filterHeader = await filterIndexer.getFilterHeader(hash); + assert(filterHeader); + const filter = await filterIndexer.getFilter(hash); + assert(filter); + assert(filterHeader.equals(filter.header)); + } + }); + }); }); From c73dfcf2c88852d85125f978f6c3e56d166fc60a Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Sun, 9 Jul 2023 00:06:11 -0500 Subject: [PATCH 18/21] test: match filters fix: checkFilter() match --- lib/blockchain/chain.js | 2 -- lib/net/pool.js | 30 +++++++++-------- lib/wallet/walletdb.js | 4 ++- test/neutrino-test.js | 6 ++-- test/wallet-neutrino-test.js | 64 ++++++++++++++++++++---------------- 5 files changed, 58 insertions(+), 48 deletions(-) diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 18f357d94..562ab375f 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -1372,7 +1372,6 @@ class Chain extends AsyncEmitter { // Do we already have this block? const existingEntry = await this.getEntry(hash); - // FOR EDUCATIONAL PURPOSES ONLY: save block without checking anything if (existingEntry && this.getPrunedMap.has(hash)) { block = block.toBlock(); await this.db.updateNeutrinoSave(); @@ -1968,7 +1967,6 @@ class Chain extends AsyncEmitter { // Ensure hash not height hash = await this.db.getHash(hash); - // FOR EDUCATIONAL PURPOSES ONLY: flag block for re-downloading const wait = new Promise((resolve, reject) => { this.getPrunedMap.set(hash, resolve); }); diff --git a/lib/net/pool.js b/lib/net/pool.js index 869746e2f..a400c8933 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -746,7 +746,8 @@ class Pool extends EventEmitter { const startHeight = cFHeaderHeight ? cFHeaderHeight + 1 : 1; const chainHeight = await this.chain.height; - const stopHeight = chainHeight > 2000 ? 2000 : chainHeight; + const stopHeight = chainHeight - startHeight + 1 > 2000 + ? 2000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); this.getcfheadersFilterType = common.FILTERS.BASIC; this.getcfheadersStopHash = stopHash; @@ -771,7 +772,8 @@ class Pool extends EventEmitter { const startHeight = cFilterHeight ? cFilterHeight + 1 : 1; const chainHeight = await this.chain.height; - const stopHeight = chainHeight > 1000 ? 1000 : chainHeight; + const stopHeight = chainHeight - startHeight + 1 > 1000 + ? 1000 : chainHeight; const stopHash = await this.chain.getHash(stopHeight); this.getcfiltersFilterType = common.FILTERS.BASIC; this.getcfiltersStartHeight = startHeight; @@ -2204,6 +2206,8 @@ class Pool extends EventEmitter { previousFilterHeader = filterHeader; await this.chain.saveCFHeaderHeight(blockHeight); blockHeight++; + const cFHeaderHeight = await this.chain.getCFHeaderHeight(); + this.logger.info('CFHeaderHeight: %d', cFHeaderHeight); } if (this.headerChain.tail.height <= stopHeight) this.emit('cfheaders'); @@ -2240,19 +2244,17 @@ class Pool extends EventEmitter { assert(blockHeight >= this.getcfiltersStartHeight && blockHeight <= stopHeight); - const cFilterHeight = await this.chain.getCFilterHeight(); + const basicFilter = new BasicFilter(); + const gcsFilter = basicFilter.fromNBytes(filter); - const indexer = this.getFilterIndexer(filtersByVal[filterType]); - - const filterHeader = await indexer.getFilterHeader(blockHash); - - const basicFilter = new BasicFilter(); - const gcsFilter = basicFilter.fromNBytes(filter); - - await indexer.saveFilter(blockHash, gcsFilter, filterHeader); + const indexer = this.getFilterIndexer(filtersByVal[filterType]); + const filterHeader = await indexer.getFilterHeader(blockHash); + await indexer.saveFilter(blockHash, gcsFilter, filterHeader); - this.emit('cfilter', blockHash, gcsFilter); - await this.chain.saveCFilterHeight(blockHeight); + await this.chain.saveCFilterHeight(blockHeight); + const cFilterHeight = await this.chain.getCFilterHeight(); + this.logger.info('CFilter height: %d', cFilterHeight); + this.emit('cfilter', blockHash, gcsFilter); const startHeight = stopHeight + 1; let nextStopHeight; if (cFilterHeight === stopHeight @@ -2279,7 +2281,7 @@ class Pool extends EventEmitter { ); return; } - } else { + } else if (cFilterHeight === this.chain.height) { this.logger.info('CFilters sync complete'); } } diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 275ee07b3..b1261baef 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -27,6 +27,7 @@ const layouts = require('./layout'); const records = require('./records'); const NullClient = require('./nullclient'); const Script = require('../script/script'); +const Address = require('../primitives/address'); const layout = layouts.wdb; const tlayout = layouts.txdb; @@ -590,7 +591,8 @@ class WalletDB extends EventEmitter { await piter.each(async (key) => { const [data] = layout.p.decode(key); if (data.length === 20) { - const script = Script.fromPubkeyhash(data); + const address = Address.fromWitnessPubkeyhash(data); + const script = Script.fromAddress(address); const match = filter.match(gcsKey, script.raw); if (match) { await this.client.getBlockFromNode(blockHash, filter); diff --git a/test/neutrino-test.js b/test/neutrino-test.js index 32a36871a..24a1382a2 100644 --- a/test/neutrino-test.js +++ b/test/neutrino-test.js @@ -5,7 +5,7 @@ const NeutrinoNode = require('../lib/node/neutrino'); const {forValue} = require('./util/common'); const assert = require('bsert'); describe('neutrino', function () { - this.timeout(10000); + this.timeout(100000); const node1 = new NeutrinoNode({ network: 'regtest', @@ -28,6 +28,7 @@ describe('neutrino', function () { while (n) { const block = await node2.miner.mineBlock(); await node2.chain.add(block); + await new Promise(resolve => setTimeout(resolve, 20)); n--; } await forValue(node1.chain, 'height', node2.chain.height); @@ -46,7 +47,7 @@ describe('neutrino', function () { await node2.connect(); node1.startSync(); node2.startSync(); - await mineBlocks(200); + await mineBlocks(2100); await waitForConnection; }); @@ -57,6 +58,7 @@ describe('neutrino', function () { describe('getheaders', () => { it('should getheaders', async () => { + await mineBlocks(10); assert.equal(node1.chain.height, node2.chain.height); }); }); diff --git a/test/wallet-neutrino-test.js b/test/wallet-neutrino-test.js index 02e040267..eb2a0db6b 100644 --- a/test/wallet-neutrino-test.js +++ b/test/wallet-neutrino-test.js @@ -2,7 +2,6 @@ const FullNode = require('../lib/node/fullnode'); const Neutrino = require('../lib/node/neutrino'); -const MTX = require('../lib/primitives/mtx'); const assert = require('bsert'); const { forValue } = require('./util/common'); const BasicFilter = require('../lib/golomb/basicFilter'); @@ -41,23 +40,16 @@ let wallet2 = null; const fwAddresses = []; const nwAddresses = []; -async function mineBlock(tx, address) { - const job = await miner.createJob(); - - if (!tx) - return await job.mineAsync(); - - const spend = new MTX(); - spend.addTX(tx, 0); - spend.addOutput(address, 50000); - - spend.setLocktime(chain.height); - await wallet1.sign(spend); - - job.addTX(spend.toTX(), spend.view); - job.refresh(); +async function mineBlocks(n, address) { + for (let i = 0; i < n; i++) { + const block = await miner.mineBlock(null, address); + const entry = await chain.add(block); + assert(entry); + } +} - return await job.mineAsync(); +function parseAddress(raw, network) { + return Address.fromString(raw, network); } describe('wallet-neutrino', function() { @@ -83,8 +75,8 @@ describe('wallet-neutrino', function() { const key = await wallet1.createReceive(0); const address = key.getAddress().toString(node1.network.type); fwAddresses.push(address); - miner.addAddress(address); } + miner.addAddress(fwAddresses[0]); for (let i = 0; i < 10; i++) { const key = await wallet2.createReceive(0); const address = key.getAddress().toString(node2.network.type); @@ -94,16 +86,12 @@ describe('wallet-neutrino', function() { it('should mine 10 blocks', async () => { for (const address of fwAddresses) { - for (let i = 0; i < 2; i++) { - const block = await mineBlock(null, address); - await chain.add(block); - } + const add = parseAddress(address, node1.network); + await mineBlocks(2, add); } for (const address of nwAddresses) { - for (let i = 0; i < 2; i++) { - const block = await mineBlock(null, address); - await chain.add(block); - } + const add = parseAddress(address, node2.network); + await mineBlocks(2, add); } }); @@ -139,14 +127,32 @@ describe('wallet-neutrino', function() { }); it('should match the filters', async () => { - const filterIndexer = node2.filterIndexers.get('BASIC'); - for (let i = 0; i < fwAddresses.length; i++) { + let j = 0; + for (let i = 1;i <= 20; i++) { + const filterIndexer = node2.filterIndexers.get('BASIC'); + const hash = await node2.chain.getHash(i); + const filter = await filterIndexer.getFilter(hash); + const basicFilter = new BasicFilter(); + const gcs = basicFilter.fromNBytes(filter.filter); + const key = hash.slice(0, 16); + const address = Address.fromString(fwAddresses[j], node1.network.type); + if (i % 2 === 0) + j++; + const script = Script.fromAddress(address); + assert(gcs.match(key, script.raw)); + } + + j = 0; + for (let i = 21;i <= node2.chain.height; i++) { + const filterIndexer = node2.filterIndexers.get('BASIC'); const hash = await node2.chain.getHash(i); const filter = await filterIndexer.getFilter(hash); const basicFilter = new BasicFilter(); const gcs = basicFilter.fromNBytes(filter.filter); const key = hash.slice(0, 16); - const address = Address.fromString(fwAddresses[i], node1.network.type); + const address = Address.fromString(nwAddresses[j], node2.network.type); + if (i % 2 === 0) + j++; const script = Script.fromAddress(address); assert(gcs.match(key, script.raw)); } From a8281f66a6a23da9f9dc225549cfd04f40b71240 Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Sun, 9 Jul 2023 01:16:25 -0500 Subject: [PATCH 19/21] test: getblockpeer --- lib/wallet/walletdb.js | 11 ++++++++++- test/wallet-neutrino-test.js | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index b1261baef..9a61a0724 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -72,6 +72,7 @@ class WalletDB extends EventEmitter { this.depth = 0; this.rescanning = false; this.filterSent = false; + this.isWitness = false; // Wallet read lock. this.readLock = new MapLock(); @@ -212,6 +213,9 @@ class WalletDB extends EventEmitter { id: 'primary' }); + const account = await wallet.getAccount(wallet.wid); + this.isWitness = account.witness; + const addr = await wallet.receiveAddress(); this.logger.info( @@ -591,7 +595,12 @@ class WalletDB extends EventEmitter { await piter.each(async (key) => { const [data] = layout.p.decode(key); if (data.length === 20) { - const address = Address.fromWitnessPubkeyhash(data); + let address = null; + if (this.isWitness) + address = Address.fromWitnessPubkeyhash(data); + else + address = Address.fromPubkeyhash(data); + const script = Script.fromAddress(address); const match = filter.match(gcsKey, script.raw); if (match) { diff --git a/test/wallet-neutrino-test.js b/test/wallet-neutrino-test.js index eb2a0db6b..3a825b06c 100644 --- a/test/wallet-neutrino-test.js +++ b/test/wallet-neutrino-test.js @@ -157,4 +157,12 @@ describe('wallet-neutrino', function() { assert(gcs.match(key, script.raw)); } }); + + it('should getblockfrompeer', async () => { + for (let i = 21; i <= node2.chain.height; i++) { + const hash = await node2.chain.getHash(i); + const block = await node2.chain.getBlock(hash); + assert(block); + } + }); }); From a2c7abade2e9b14445b2c493cd863c4ab0d620cc Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Sun, 9 Jul 2023 16:21:28 -0500 Subject: [PATCH 20/21] chore: add neutrino option --- lib/indexer/indexer.js | 11 ++++++++++- lib/node/neutrino.js | 3 ++- lib/wallet/walletdb.js | 17 +++++++++-------- test/neutrino-test.js | 2 +- test/wallet-neutrino-test.js | 7 ++++++- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/lib/indexer/indexer.js b/lib/indexer/indexer.js index c095584ca..b052d6a97 100644 --- a/lib/indexer/indexer.js +++ b/lib/indexer/indexer.js @@ -50,6 +50,8 @@ class Indexer extends EventEmitter { this.blocks = this.options.blocks; this.chain = this.options.chain; + this.neutrino = this.options.neutrino; + this.closing = false; this.db = null; this.batch = null; @@ -292,7 +294,7 @@ class Indexer extends EventEmitter { */ async _syncBlock(meta, block, view) { - if (this.chain.options.neutrino) { + if (this.neutrino) { if (!this.batch) this.start(); return true; @@ -641,6 +643,8 @@ class IndexOptions { this.cacheSize = 16 << 20; this.compression = true; + this.neutrino = false; + if (options) this.fromOptions(options); } @@ -702,6 +706,11 @@ class IndexOptions { this.compression = options.compression; } + if (options.neutrino != null) { + assert(typeof options.neutrino === 'boolean'); + this.neutrino = options.neutrino; + } + return this; } diff --git a/lib/node/neutrino.js b/lib/node/neutrino.js index 4c7ccf663..b3570f69f 100644 --- a/lib/node/neutrino.js +++ b/lib/node/neutrino.js @@ -81,7 +81,8 @@ class Neutrino extends Node { chain: this.chain, memory: this.config.bool('memory'), prefix: this.config.str('index-prefix', this.config.prefix), - filterType: 'BASIC' + filterType: 'BASIC', + neutrino: true }) ); diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 9a61a0724..b1b2c7c84 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -594,19 +594,20 @@ class WalletDB extends EventEmitter { await piter.each(async (key) => { const [data] = layout.p.decode(key); + let address = null; if (data.length === 20) { - let address = null; if (this.isWitness) address = Address.fromWitnessPubkeyhash(data); else address = Address.fromPubkeyhash(data); - - const script = Script.fromAddress(address); - const match = filter.match(gcsKey, script.raw); - if (match) { - await this.client.getBlockFromNode(blockHash, filter); - return; - } + } else if (data.length === 32) { + address = Address.fromWitnessScripthash(data); + } + const script = Script.fromAddress(address); + const match = filter.match(gcsKey, script.toRaw()); + if (match) { + await this.client.getBlockFromNode(blockHash, filter); + return; } }); } diff --git a/test/neutrino-test.js b/test/neutrino-test.js index 24a1382a2..842fe2782 100644 --- a/test/neutrino-test.js +++ b/test/neutrino-test.js @@ -47,8 +47,8 @@ describe('neutrino', function () { await node2.connect(); node1.startSync(); node2.startSync(); - await mineBlocks(2100); await waitForConnection; + await mineBlocks(1000); }); after(async () => { diff --git a/test/wallet-neutrino-test.js b/test/wallet-neutrino-test.js index 3a825b06c..2ab783c62 100644 --- a/test/wallet-neutrino-test.js +++ b/test/wallet-neutrino-test.js @@ -84,7 +84,7 @@ describe('wallet-neutrino', function() { } }); - it('should mine 10 blocks', async () => { + it('should mine 40 blocks', async () => { for (const address of fwAddresses) { const add = parseAddress(address, node1.network); await mineBlocks(2, add); @@ -165,4 +165,9 @@ describe('wallet-neutrino', function() { assert(block); } }); + + it('should cleanup', async () => { + await node1.close(); + await node2.close(); + }); }); From 7b70e3ef78563f0ee42e501715004de837f0d0ce Mon Sep 17 00:00:00 2001 From: Manav Desai Date: Mon, 10 Jul 2023 15:32:49 -0500 Subject: [PATCH 21/21] test: fix p2p-test --- test/p2p-bip157-test.js | 100 ++++++++++++++++++++++++++++++++++++++++ test/p2p-test.js | 55 +--------------------- 2 files changed, 101 insertions(+), 54 deletions(-) create mode 100644 test/p2p-bip157-test.js diff --git a/test/p2p-bip157-test.js b/test/p2p-bip157-test.js new file mode 100644 index 000000000..b72c7175a --- /dev/null +++ b/test/p2p-bip157-test.js @@ -0,0 +1,100 @@ +/* eslint-env mocha */ +/* eslint prefer-arrow-callback: "off" */ + +'use strict'; + +const assert = require('bsert'); +const FullNode = require('../lib/node/fullnode'); +const NeutrinoNode = require('../lib/node/neutrino'); +const {forValue} = require('./util/common'); +const {MAX_CFILTERS} = require('../lib/net/common'); +const packets = require('../lib/net/packets'); + +describe('P2P', function () { + this.timeout(50000); + + const node1 = new NeutrinoNode({ + network: 'regtest', + memory: true, + port: 10000, + httpPort: 20000, + only: '127.0.0.1', + neutrino: true + }); + + const node2 = new FullNode({ + network: 'regtest', + memory: true, + listen: true, + indexFilter: true, + bip157: true + }); + + let peer; + const nodePackets = {}; + + node1.pool.on('packet', (packet) => { + if (!nodePackets[packet.cmd]) + nodePackets[packet.cmd] = [packet]; + else + nodePackets[packet.cmd].push(packet); + }); + + async function mineBlocks(n) { + while (n) { + const block = await node2.miner.mineBlock(); + await node2.chain.add(block); + await new Promise(resolve => setTimeout(resolve, 20)); + n--; + } + await forValue(node1.chain, 'height', node2.chain.height); + } + + before(async () => { + const waitForConnection = new Promise((resolve, reject) => { + node1.pool.once('peer open', async (peer) => { + resolve(peer); + }); + }); + + await node1.open(); + await node2.open(); + await node1.connect(); + await node2.connect(); + node1.startSync(); + node2.startSync(); + + // `peer` is node2, from node1's perspective. + // So peer.send() sends a packet from node1 to node2, + // and `nodePackets` catches the response packets that + // node2 sends back to node1. + peer = await waitForConnection; + }); + + after(async () => { + await node1.close(); + await node2.close(); + }); + + describe('BIP157', function () { + before(async () => { + // Do not exceed limit, including genesis block + await mineBlocks(MAX_CFILTERS - node1.chain.height - 1); + }); + + it('CFCheckpt', async () => { + nodePackets.cfcheckpt = []; + + await mineBlocks(2); + + const pkt = new packets.GetCFCheckptPacket( + 0, + node1.chain.tip.hash + ); + + peer.send(pkt); + await forValue(nodePackets.cfcheckpt, 'length', 1); + assert.strictEqual(nodePackets.cfcheckpt[0].filterHeaders.length, 1); + }); + }); +}); diff --git a/test/p2p-test.js b/test/p2p-test.js index 85eae0849..b9502550d 100644 --- a/test/p2p-test.js +++ b/test/p2p-test.js @@ -6,8 +6,6 @@ const assert = require('bsert'); const FullNode = require('../lib/node/fullnode'); const {forValue} = require('./util/common'); -const {MAX_CFILTERS} = require('../lib/net/common'); -const packets = require('../lib/net/packets'); describe('P2P', function () { this.timeout(5000); @@ -60,6 +58,7 @@ describe('P2P', function () { await node2.connect(); node1.startSync(); node2.startSync(); + await mineBlocks(1); // `peer` is node2, from node1's perspective. // So peer.send() sends a packet from node1 to node2, @@ -73,58 +72,6 @@ describe('P2P', function () { await node2.close(); }); - describe('BIP157', function () { - before(async () => { - // Do not exceed limit, including genesis block - await mineBlocks(MAX_CFILTERS - node1.chain.height - 1); - }); - - it('CFilters', async () => { - nodePackets.cfilter = []; - - const pkt = new packets.GetCFiltersPacket( - 0, - 0, - node1.chain.tip.hash - ); - - peer.send(pkt); - await forValue(nodePackets.cfilter, 'length', MAX_CFILTERS); - }); - - it('CFHeaders', async () => { - nodePackets.cfheaders = []; - - const pkt = new packets.GetCFHeadersPacket( - 0, - 0, - node1.chain.tip.hash - ); - - peer.send(pkt); - await forValue(nodePackets.cfheaders, 'length', 1); - assert.strictEqual( - nodePackets.cfheaders[0].filterHashes.length, - node1.chain.height + 1 - ); - }); - - it('CFCheckpt', async () => { - nodePackets.cfcheckpt = []; - - await mineBlocks(2); - - const pkt = new packets.GetCFCheckptPacket( - 0, - node1.chain.tip.hash - ); - - peer.send(pkt); - await forValue(nodePackets.cfcheckpt, 'length', 1); - assert.strictEqual(nodePackets.cfcheckpt[0].filterHeaders.length, 1); - }); - }); - describe('Compact Blocks', function () { it('should get compact block in low bandwidth mode', async () => { nodePackets.inv = [];