Merge pull request #163 from LiskHQ/161-copy-snapshot-block-to-the-ex…

…ecutable-directory

Copy snapshot block to the executable directory

config/mainnet/config.json

@@ -8,7 +8,7 @@
 		"modes": ["ipc", "ws"],
 		"port": 7887,
 		"host": "127.0.0.1",
-		"allowedMethods": []
+		"allowedMethods": ["generator", "system", "random"]
 	},
 	"genesis": {
 		"block": {

config/testnet/config.json

@@ -8,7 +8,7 @@
 		"modes": ["ipc", "ws"],
 		"port": 7887,
 		"host": "127.0.0.1",
-		"allowedMethods": []
+		"allowedMethods": ["generator", "system", "random"]
 	},
 	"genesis": {
 		"block": {

docs/migration.md

@@ -82,11 +82,11 @@ FLAGS
   -d, --lisk-core-v3-data-path=lisk-core-v3-data-path  Path where the lisk-core v3.x instance is running. Current home directory will be considered the default if not provided.
   -h, --help                                           Shows CLI help.
   -o, --output=output                                  File path to write the genesis block json. If not provided, it will default to cwd/genesis_block.json.
+  -p, --page-size                                      Maximum number of blocks to be iterated at once for computation. Default to 100000.
   -s, --snapshot-height=snapshot-height                (Required) The height at which the re-genesis block will be generated. Can be specified with the SNAPSHOT_HEIGHT as well.
   -v, --version                                        Shows the CLI version.
   --auto-migrate-config                                Migrate user configuration automatically. Default to false.
   --auto-start-lisk-core-v4                            Start lisk-core v4 automatically. Default to false.
-  --snapshot-time-gap=snapshot-time-gap                The number of seconds elapsed between the block at height HEIGHT_SNAPSHOT and the snapshot block.
 
 EXAMPLES
   lisk-migrator --snapshot-height 20931763 --lisk-core-path /path/to/data-dir

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "lisk-migrator",
-	"version": "2.0.0-rc.0",
+	"version": "2.0.0-rc.1",
 	"description": "A command-line tool for migrating the blockchain state to the latest protocol after a hard fork",
 	"author": "Lisk Foundation <[email protected]>, lightcurve GmbH <[email protected]>",
 	"license": "Apache-2.0",
@@ -34,10 +34,10 @@
 		"test:ci": "jest --coverage=true --coverage-reporters=json --verbose",
 		"test:watch": "npm test -- --watch",
 		"copy-static-files": "copyfiles -u 1 src/**/*.sql ./dist",
-		"prebuild": "if test -d dist; then rm -r dist; fi; rm -f tsconfig.tsbuildinfo; rm -f npm-shrinkwrap.json",
-		"build": "tsc && npm run copy-static-files",
-		"prepack": "oclif-dev manifest && oclif-dev readme --multi --dir=docs/commands && git add README.md docs/commands && npm shrinkwrap",
-		"prepublishOnly": "yarn && npm run lint && npm run build"
+		"prebuild": "if test -d dist; then rm -r dist; fi; rm -f tsconfig.tsbuildinfo",
+		"build": "tsc && yarn copy-static-files",
+		"prepack": "oclif-dev manifest && oclif-dev readme --multi --dir=docs/commands",
+		"prepublishOnly": "yarn && yarn lint && yarn build"
 	},
 	"oclif": {
 		"commands": "./dist/commands",
@@ -64,6 +64,7 @@
 		"@liskhq/lisk-codec": "0.4.0-rc.1",
 		"@liskhq/lisk-cryptography": "4.0.0-rc.1",
 		"@liskhq/lisk-db": "0.3.10",
+		"@liskhq/lisk-utils": "0.4.0-rc.0",
 		"@liskhq/lisk-validator": "0.8.0-rc.1",
 		"@oclif/command": "1.8.21",
 		"@oclif/config": "1.14.0",

src/assets/pos.ts

@@ -73,6 +73,7 @@ export const formatInt = (num: number | bigint): string => {
 export const getValidatorKeys = async (
 	accounts: Account[],
 	db: Database,
+	pageSize: number,
 ): Promise<Record<string, string>> => {
 	const delegateSet = new Set();
 	for (const account of accounts) {
@@ -81,9 +82,8 @@ export const getValidatorKeys = async (
 		}
 	}
 
-	const PAGE_SIZE = 100000;
-	const blockPublicKeySet = await getBlockPublicKeySet(db, PAGE_SIZE);
-	const txPublicKeySet = await getTransactionPublicKeySet(db, PAGE_SIZE);
+	const blockPublicKeySet = await getBlockPublicKeySet(db, pageSize);
+	const txPublicKeySet = await getTransactionPublicKeySet(db, pageSize);
 
 	const keys: Record<string, string> = {};
 	for (const key of blockPublicKeySet) {

src/client.ts

@@ -15,10 +15,13 @@ import { createIPCClient, APIClient } from '@liskhq/lisk-api-client';
 
 const clientCache: Record<string, APIClient> = {};
 
-export const getAPIClient = async (liskCorePath: string): Promise<APIClient> => {
+export const getAPIClient = async (
+	liskCorePath: string,
+	isForceInstantiate?: boolean,
+): Promise<APIClient> => {
 	let _client = clientCache[liskCorePath];
 
-	if (!_client) {
+	if (!_client || isForceInstantiate) {
 		_client = await createIPCClient(liskCorePath);
 		clientCache[liskCorePath] = _client;
 	}

src/createAsset.ts

@@ -75,7 +75,11 @@ export class CreateAsset {
 		this._db = db;
 	}
 
-	public init = async (snapshotHeight: number, tokenID: string): Promise<GenesisAssetEntry[]> => {
+	public init = async (
+		snapshotHeight: number,
+		tokenID: string,
+		pageSize: number,
+	): Promise<GenesisAssetEntry[]> => {
 		const authSubstoreEntries: AuthStoreEntryBuffer[] = [];
 		const userSubstoreEntries: UserSubstoreEntryBuffer[] = [];
 		const supplySubstoreEntries: SupplySubstoreEntry[] = [];
@@ -124,7 +128,7 @@ export class CreateAsset {
 		);
 
 		// Get all validator keys for PoS module
-		const validatorKeys = await getValidatorKeys(accounts, this._db);
+		const validatorKeys = await getValidatorKeys(accounts, this._db, pageSize);
 
 		for (const account of accounts) {
 			// genesis asset for auth module

src/events.ts

@@ -30,18 +30,20 @@ export const captureForgingStatusAtSnapshotHeight = (
 		const newBlock = client.block.decode(Buffer.from(encodedBlock, 'hex')) as Block;
 		if (newBlock.header.height === snapshotHeight) {
 			const forgingStatus = await client.invoke('app:getForgingStatus');
-			const forgingStatusJsonFilepath = resolve(outputDir, 'forgingStatus.json');
-			try {
-				await write(forgingStatusJsonFilepath, JSON.stringify(forgingStatus, null, 2));
-				_this.log(`Finished exporting forging status to ${forgingStatusJsonFilepath}.`);
-			} catch (error) {
-				_this.log(
-					`Unable to save the node Forging Status information to the disk, please find it below instead:\n${JSON.stringify(
-						forgingStatus,
-						null,
-						2,
-					)}`,
-				);
+			if (forgingStatus.length) {
+				const forgingStatusJsonFilepath = resolve(outputDir, 'forgingStatus.json');
+				try {
+					await write(forgingStatusJsonFilepath, JSON.stringify(forgingStatus, null, 2));
+					_this.log(`\nFinished exporting forging status to ${forgingStatusJsonFilepath}.`);
+				} catch (error) {
+					_this.log(
+						`\nUnable to save the node Forging Status information to the disk, please find it below instead:\n${JSON.stringify(
+							forgingStatus,
+							null,
+							2,
+						)}`,
+					);
+				}
 			}
 		}
 	});

src/index.ts

@@ -13,7 +13,7 @@
  */
 import util from 'util';
 import * as fs from 'fs-extra';
-import { join } from 'path';
+import { join, resolve } from 'path';
 import { ApplicationConfig, PartialApplicationConfig } from 'lisk-framework';
 import { Database } from '@liskhq/lisk-db';
 import * as semver from 'semver';
@@ -26,7 +26,6 @@ import {
 	SNAPSHOT_DIR,
 	MIN_SUPPORTED_LISK_CORE_VERSION,
 	DEFAULT_LISK_CORE_PATH,
-	LEGACY_DB_PATH,
 } from './constants';
 import { getAPIClient } from './client';
 import {
@@ -44,11 +43,12 @@ import {
 	setPrevSnapshotBlockHeightByNetID,
 } from './utils/chain';
 import { captureForgingStatusAtSnapshotHeight } from './events';
-import { createGenesisBlock, writeGenesisAssets } from './utils/genesis_block';
+import { copyGenesisBlock, createGenesisBlock, writeGenesisAssets } from './utils/genesis_block';
 import { CreateAsset } from './createAsset';
 import { ApplicationConfigV3, NetworkConfigLocal, NodeInfo } from './types';
 import { installLiskCore, startLiskCore } from './utils/node';
-import { copyDir, resolveAbsolutePath } from './utils/fs';
+import { resolveAbsolutePath } from './utils/fs';
+import { execAsync } from './utils/process';
 
 let configCoreV4: PartialApplicationConfig;
 class LiskMigrator extends Command {
@@ -69,12 +69,12 @@ class LiskMigrator extends Command {
 			char: 'd',
 			required: false,
 			description:
-				'Path where the lisk-core v3.x instance is running. The current home directory will be considered the default directory if not otherwise provided.',
+				'Path where the Lisk Core v3.x instance is running. When not supplied, defaults to the default data directory for Lisk Core.',
 		}),
 		config: flagsParser.string({
 			char: 'c',
 			required: false,
-			description: 'Custom configuration file path.',
+			description: 'Custom configuration file path for Lisk Core v3.x.',
 		}),
 		'snapshot-height': flagsParser.integer({
 			char: 's',
@@ -95,6 +95,13 @@ class LiskMigrator extends Command {
 			description: 'Start lisk core v4 automatically. Default to false.',
 			default: false,
 		}),
+		'page-size': flagsParser.integer({
+			char: 'p',
+			required: false,
+			default: 100000,
+			description:
+				'Maximum number of blocks to be iterated at once for computation. Default to 100000.',
+		}),
 	};
 
 	public async run(): Promise<void> {
@@ -108,6 +115,7 @@ class LiskMigrator extends Command {
 			const customConfigPath = flags.config;
 			const autoMigrateUserConfig = flags['auto-migrate-config'] ?? false;
 			const autoStartLiskCoreV4 = flags['auto-start-lisk-core-v4'];
+			const pageSize = Number(flags['page-size']);
 
 			const client = await getAPIClient(liskCoreV3DataPath);
 			const nodeInfo = (await client.node.getNodeInfo()) as NodeInfo;
@@ -156,7 +164,7 @@ class LiskMigrator extends Command {
 					`Lisk Migrator utility is not compatible for lisk-core version ${liskCoreVersion.version}. The minimum compatible version is: ${MIN_SUPPORTED_LISK_CORE_VERSION}.`,
 				);
 			}
-			cli.action.stop(`${liskCoreVersion.version} detected`);
+			cli.action.stop(`${appVersion} detected`);
 
 			// User specified custom config file
 			const configV3: ApplicationConfigV3 = customConfigPath
@@ -184,7 +192,7 @@ class LiskMigrator extends Command {
 			cli.action.start('Creating genesis assets');
 			const createAsset = new CreateAsset(db);
 			const tokenID = getTokenIDLsk();
-			const genesisAssets = await createAsset.init(snapshotHeight, tokenID);
+			const genesisAssets = await createAsset.init(snapshotHeight, tokenID, pageSize);
 			cli.action.stop();
 
 			// Create an app instance for creating genesis block
@@ -244,19 +252,34 @@ class LiskMigrator extends Command {
 						configCoreV4 = defaultConfigV4;
 					}
 
-					cli.action.start(`Creating legacy.db at ${LEGACY_DB_PATH}`);
-					await copyDir(snapshotDirPath, resolveAbsolutePath(LEGACY_DB_PATH));
-					this.log(`Legacy database has been created at ${LEGACY_DB_PATH}`);
+					cli.action.start('Copying genesis block to the Lisk Core executable directory');
+					const liskCoreExecPath = await execAsync('which lisk-core');
+					const liskCoreV4ConfigPath = resolve(
+						liskCoreExecPath,
+						'../..',
+						`lib/node_modules/lisk-core/config/${networkConstant.name}`,
+					);
+
+					await copyGenesisBlock(
+						`${outputDir}/genesis_block.blob`,
+						`${liskCoreV4ConfigPath}/genesis_block.blob`,
+					);
+					this.log(`Genesis block has been copied to: ${liskCoreV4ConfigPath}.`);
 					cli.action.stop();
 
 					// Ask user to manually stop Lisk Core v3 and continue
-					const isLiskCoreV3Stopped = await cli.confirm(`
-					Please stop Lisk Core v3 to continue. Type 'yes' and press Enter when ready. [yes/no]`);
+					const isLiskCoreV3Stopped = await cli.confirm(
+						"Please stop Lisk Core v3 to continue. Type 'yes' and press Enter when ready. [yes/no]",
+					);
 
 					if (isLiskCoreV3Stopped) {
-						const isUserConfirmed = await cli.confirm(`
-						Start Lisk Core with the following configuration? [yes/no] \n
-						${util.inspect(configCoreV4, false, 3)}`);
+						const isUserConfirmed = await cli.confirm(
+							`Start Lisk Core with the following configuration? [yes/no] \n${util.inspect(
+								configCoreV4,
+								false,
+								3,
+							)}`,
+						);
 
 						if (isUserConfirmed) {
 							cli.action.start('Starting lisk-core v4');
@@ -280,8 +303,9 @@ class LiskMigrator extends Command {
 				}
 			} else {
 				this.log(
-					`Please copy ${snapshotDirPath} directory to the Lisk Core V4 data directory in order to access legacy blockchain information`,
+					`Please copy ${snapshotDirPath} directory to the Lisk Core V4 data directory in order to access legacy blockchain information.`,
 				);
+				this.log('Please copy genesis block to the Lisk Core V4 network directory.');
 			}
 		} catch (error) {
 			this.error(error as string);

src/utils/config.ts

@@ -20,6 +20,7 @@ import { join, resolve } from 'path';
 import { validator } from '@liskhq/lisk-validator';
 
 import { ApplicationConfig, applicationConfigSchema } from 'lisk-framework';
+import { objects } from '@liskhq/lisk-utils';
 import { ApplicationConfigV3, LoggerConfig } from '../types';
 import {
 	DEFAULT_VERSION,
@@ -134,17 +135,6 @@ export const migrateUserConfig = async (
 		cli.action.stop();
 	}
 
-	// TODO: Verify if needed
-	if (configV3.backup?.height) {
-		cli.action.start(
-			`Setting config property 'system.backup.height' to: ${configV3.backup.height}.`,
-		);
-		configV4.system.backup = {
-			height: Math.max(snapshotHeight + 1, configV3.backup.height + 1),
-		};
-		cli.action.stop();
-	}
-
 	if (configV3.transactionPool) {
 		if (configV3.transactionPool.maxTransactions) {
 			cli.action.start(
@@ -267,7 +257,8 @@ export const migrateUserConfig = async (
 
 export const validateConfig = async (config: ApplicationConfig): Promise<boolean> => {
 	try {
-		(await validator.validate(applicationConfigSchema, config)) as unknown;
+		const mergedConfig = objects.mergeDeep({}, applicationConfigSchema.default, config);
+		(await validator.validate(applicationConfigSchema, mergedConfig)) as unknown;
 		return true;
 	} catch (error) {
 		return false;

src/utils/fs.ts

@@ -77,3 +77,13 @@ export const write = async (filePath: string, content: string): Promise<boolean
 			return resolve(true);
 		});
 	});
+
+export const copyFile = async (src: string, dest: string): Promise<boolean | Error> =>
+	new Promise((resolve, reject) => {
+		fs.copyFile(src, dest, err => {
+			if (err) {
+				return reject(err);
+			}
+			return resolve(true);
+		});
+	});

src/utils/genesis_block.ts

@@ -18,6 +18,7 @@ import { Block as BlockVersion3 } from '@liskhq/lisk-chain';
 import { SNAPSHOT_TIME_GAP } from '../constants';
 import { GenesisAssetEntry } from '../types';
 import { execAsync } from './process';
+import { copyFile } from './fs';
 
 (BigInt.prototype as any).toJSON = function () {
 	return this.toString();
@@ -72,3 +73,8 @@ export const writeGenesisAssets = async (
 		JSON.stringify({ assets: genesisAssets }, null, '\t'),
 	);
 };
+
+export const copyGenesisBlock = async (
+	currGenesisBlockFilepath: string,
+	liskCoreV4ConfigPath: string,
+): Promise<boolean | Error> => copyFile(currGenesisBlockFilepath, liskCoreV4ConfigPath);