diff --git a/rskj-core/src/main/java/co/rsk/db/migration/MissingOrchidStorageKeysProvider.java b/rskj-core/src/main/java/co/rsk/db/migration/MissingOrchidStorageKeysProvider.java index bfb31a77fac..9617c863f24 100644 --- a/rskj-core/src/main/java/co/rsk/db/migration/MissingOrchidStorageKeysProvider.java +++ b/rskj-core/src/main/java/co/rsk/db/migration/MissingOrchidStorageKeysProvider.java @@ -28,7 +28,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -42,7 +41,7 @@ public class MissingOrchidStorageKeysProvider { - private static final String MAPDB_FILENAME = "migration-extras"; + public static final String MAPDB_FILENAME = "migration-extras"; private static final Logger logger = LoggerFactory.getLogger(MissingOrchidStorageKeysProvider.class); @@ -57,7 +56,6 @@ public MissingOrchidStorageKeysProvider(String databaseDir, URL missingStorageKe this.databaseLocalFile = databasePath.resolve(MAPDB_FILENAME).toFile(); } - @Nullable public DataWord getKeccak256PreImage(Keccak256 storageKeyHash) { if (patchDatabase == null) { if (!databaseLocalFile.exists()) { @@ -82,13 +80,24 @@ public DataWord getKeccak256PreImage(Keccak256 storageKeyHash) { } byte[] storageKey = patchDatabase.get(storageKeyHash.getBytes()); - if (!Arrays.equals(storageKeyHash.getBytes(), Keccak256Helper.keccak256(storageKey))) { + if (storageKey == null) { + throw new IllegalStateException(String.format( + "We have detected a recoverable inconsistency in your database during the migration process.\n" + + "The patching information required to do this is not available at the moment, " + + "but it's automatically updated every hour.\n" + + "Please try again in a few minutes. " + + "If you have any question please reach through our support channel at http://gitter.im/rsksmart/rskj\n" + + "Missing storage key: %s", + storageKeyHash.toHexString() + )); + } + if (!Arrays.equals(storageKeyHash.getBytes(), Keccak256Helper.keccak256(DataWord.valueOf(storageKey).getData()))) { throw new IllegalStateException( String.format("You have downloaded an inconsistent database. %s doesn't match expected keccak256 hash (%s)", Hex.toHexString(storageKey), Hex.toHexString(storageKeyHash.getBytes()) )); } - return storageKey != null? DataWord.valueOf(storageKey) : null; + return DataWord.valueOf(storageKey); } } diff --git a/rskj-core/src/main/java/co/rsk/db/migration/OrchidToUnitrieMigrator.java b/rskj-core/src/main/java/co/rsk/db/migration/OrchidToUnitrieMigrator.java index 651af8f30a4..b37bd3a8921 100644 --- a/rskj-core/src/main/java/co/rsk/db/migration/OrchidToUnitrieMigrator.java +++ b/rskj-core/src/main/java/co/rsk/db/migration/OrchidToUnitrieMigrator.java @@ -27,6 +27,7 @@ import org.bouncycastle.util.encoders.Hex; import org.ethereum.core.AccountState; import org.ethereum.core.Block; +import org.ethereum.core.ImportResult; import org.ethereum.core.Repository; import org.ethereum.crypto.Keccak256Helper; import org.ethereum.datasource.HashMapDB; @@ -117,7 +118,8 @@ public static void migrateStateToUnitrieIfNeeded(RskContext ctx) throws IOExcept // this block number has to be validated before the release to ensure the migration works fine for every user long minimumBlockNumberToMigrate = ctx.getRskSystemProperties().getDatabaseMigrationMinimumHeight(); - Block blockToMigrate = ctx.getBlockStore().getBestBlock(); + Block bestBlock = ctx.getBlockStore().getBestBlock(); + Block blockToMigrate = ctx.getBlockStore().getBlockByHash(bestBlock.getParentHash().getBytes()); if (blockToMigrate == null || blockToMigrate.getNumber() < minimumBlockNumberToMigrate) { logger.error( "The database can't be migrated because the node wasn't up to date before upgrading. " + @@ -127,7 +129,7 @@ public static void migrateStateToUnitrieIfNeeded(RskContext ctx) throws IOExcept logger.error("Reset database or continue syncing with previous version"); // just opening the db against the unitrie directory creates certain file structure // we clean that here in case of an error - Files.deleteIfExists(unitrieDatabase); + FileUtil.recursiveDelete(unitrieDatabase.toString()); System.exit(1); } @@ -143,11 +145,31 @@ public static void migrateStateToUnitrieIfNeeded(RskContext ctx) throws IOExcept ) ); - unitrieMigrationTool.migrate(); + try { + unitrieMigrationTool.migrate(); + } catch (RuntimeException e) { + logger.error("Couldn't migrate the database", e); + + FileUtil.recursiveDelete(unitrieDatabase.toString()); + Files.deleteIfExists(Paths.get(databaseDir, MissingOrchidStorageKeysProvider.MAPDB_FILENAME)); + System.exit(1); + } + + ctx.getBlockStore().removeBlock(bestBlock); + if (ctx.getBlockchain().tryToConnect(bestBlock) != ImportResult.IMPORTED_BEST) { + logger.error( + "The database can't be migrated because the block {} couldn't be connected after the migration.", + bestBlock.getNumber() + ); + // just opening the db against the unitrie directory creates certain file structure + // we clean that here in case of an error + FileUtil.recursiveDelete(unitrieDatabase.toString()); + System.exit(1); + } } public void migrate() { - logger.info("Migration started"); + logger.info("Migration started. It can take up to 15 minutes"); logger.info("Block {}", blockToMigrate.getNumber()); Trie migratedTrie = migrateState(blockToMigrate); unitrieRepository.flush(); @@ -165,19 +187,7 @@ private Trie migrateState(Block blockToMigrate) { throw new IllegalStateException(String.format("Stored account state is not consistent with the expected root (%s) for block %d", Hex.toHexString(orchidStateRoot), blockToMigrate.getNumber())); } - try { - buildPartialUnitrie(orchidAccountsTrie, unitrieRepository); - } catch (MissingContractStorageKeysException e) { - StringBuilder missingStorageKeysMessage = new StringBuilder( - "We have detected an inconsistency in your database and are unable to migrate it automatically.\n" + - "Please visit https://www.github.com/rsksmart/rskj/issues/452 for information on how to continue.\n" + - "Here is the data you'll need:\n" - ); - for (Keccak256 entry : e.getMissingStorageKeys()) { - missingStorageKeysMessage.append(entry.toHexString()).append("\n"); - } - logger.error(missingStorageKeysMessage.toString()); - } + buildPartialUnitrie(orchidAccountsTrie, unitrieRepository); byte[] lastStateRoot = unitrieRepository.getRoot(); byte[] orchidMigratedStateRoot = trieConverter.getOrchidAccountTrieRoot(unitrieRepository.getMutableTrie().getTrie()); @@ -195,12 +205,11 @@ private Trie migrateState(Block blockToMigrate) { return unitrieRepository.getMutableTrie().getTrie(); } - private void buildPartialUnitrie(Trie orchidAccountsTrie, Repository repository) throws MissingContractStorageKeysException { + private void buildPartialUnitrie(Trie orchidAccountsTrie, Repository repository) { int accountsToLog = 500; int accountsCounter = 0; logger.trace("(x = {} accounts): ", accountsToLog); Iterator orchidAccountsTrieIterator = orchidAccountsTrie.getPreOrderIterator(); - Collection missingStorageKeys = new HashSet<>(); while (orchidAccountsTrieIterator.hasNext()) { Trie.IterationElement orchidAccountsTrieElement = orchidAccountsTrieIterator.next(); TrieKeySlice currentElementExpandedPath = orchidAccountsTrieElement.getNodeKey(); @@ -216,23 +225,16 @@ private void buildPartialUnitrie(Trie orchidAccountsTrie, Repository repository) byte[] codeHash = oldAccountState.getCodeHash(); byte[] accountStateRoot = oldAccountState.getStateRoot(); if (contractData != null) { - try { - migrateContract(accountAddress, repository, contractData, codeHash, accountStateRoot); - } catch (MissingContractStorageKeysException e) { - missingStorageKeys.addAll(e.getMissingStorageKeys()); - } + migrateContract(accountAddress, repository, contractData, codeHash, accountStateRoot); } if (accountsCounter % accountsToLog == 0) { logger.trace("x"); } } } - if (!missingStorageKeys.isEmpty()) { - throw new MissingContractStorageKeysException(missingStorageKeys); - } } - private void migrateContract(RskAddress accountAddress, Repository currentRepository, byte[] contractDataRaw, byte[] accountCodeHash, byte[] stateRoot) throws MissingContractStorageKeysException { + private void migrateContract(RskAddress accountAddress, Repository currentRepository, byte[] contractDataRaw, byte[] accountCodeHash, byte[] stateRoot) { ContractData contractData = new ContractData(contractDataRaw); boolean initialized = false; @@ -260,17 +262,12 @@ private void migrateContract(RskAddress accountAddress, Repository currentReposi Keccak256 storageKeyHash = new Keccak256(Keccak256Helper.keccak256(rawKey)); keccak256Cache.put(storageKeyHash, storageKey); } - Collection missingStorageKeys = new HashSet<>(); Iterator inOrderIterator = contractStorageTrie.getInOrderIterator(); while (inOrderIterator.hasNext()) { Trie.IterationElement iterationElement = inOrderIterator.next(); if (iterationElement.getNode().getValue() != null) { Keccak256 storageKeyHash = new Keccak256(iterationElement.getNodeKey().encode()); DataWord storageKey = keccak256Cache.computeIfAbsent(storageKeyHash, missingOrchidStorageKeysProvider::getKeccak256PreImage); - if (storageKey == null) { - missingStorageKeys.add(storageKeyHash); - continue; - } byte[] value = iterationElement.getNode().getValue(); migratedKeysCounter++; @@ -284,10 +281,6 @@ private void migrateContract(RskAddress accountAddress, Repository currentReposi currentRepository.addStorageBytes(contractAddress, storageKey, value); } } - if (!missingStorageKeys.isEmpty()) { - logger.error("{} keys lost", missingStorageKeys.size()); - throw new MissingContractStorageKeysException(missingStorageKeys); - } } byte[] code = contractData.getCode(); @@ -484,17 +477,4 @@ public byte[] retrieveValue(byte[] hash) { public void flush() { } } - - private static class MissingContractStorageKeysException extends Exception { - - private final Collection missingStorageKeys; - - private MissingContractStorageKeysException(Collection missingStorageKeys) { - this.missingStorageKeys = Collections.unmodifiableCollection(missingStorageKeys); - } - - private Collection getMissingStorageKeys() { - return missingStorageKeys; - } - } }