From c8b44f428bfcd9153bd91cedb704579eb38a50c4 Mon Sep 17 00:00:00 2001 From: jamescowens Date: Wed, 7 Mar 2018 14:29:57 -0500 Subject: [PATCH 1/9] Removes unnecessary lsn_reset from CheckpointLSN to improve wallet performance. --- src/db.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/db.cpp diff --git a/src/db.cpp b/src/db.cpp old mode 100644 new mode 100755 index e5bf33df56..851760c844 --- a/src/db.cpp +++ b/src/db.cpp @@ -227,7 +227,10 @@ void CDBEnv::CheckpointLSN(std::string strFile) dbenv.txn_checkpoint(0, 0, 0); if (fMockDb) return; - dbenv.lsn_reset(strFile.c_str(), 0); + // The below line is commented out. LSN reset is not necessary and improper. Txn_checkpoint alone will flush in memory log to the database file. + // This was causing extraordinary long flush times for large wallets. + // Flush is called when the wallet is closed upon client shutdown, and the below line is included there to ensure portability of wallet.dat. + //dbenv.lsn_reset(strFile.c_str(), 0); } From 801083b9f7287b9b02b3a0cdde7545d6b2aa067b Mon Sep 17 00:00:00 2001 From: jamescowens Date: Thu, 8 Mar 2018 23:03:39 -0500 Subject: [PATCH 2/9] Comments out bitdb.mapFileUseCount.erase(mi++) --- src/walletdb.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 src/walletdb.cpp diff --git a/src/walletdb.cpp b/src/walletdb.cpp old mode 100644 new mode 100755 index 3822595443..0c93c64973 --- a/src/walletdb.cpp +++ b/src/walletdb.cpp @@ -670,8 +670,11 @@ void ThreadFlushWalletDB(void* parg) // Flush wallet.dat so it's self contained bitdb.CloseDb(strFile); bitdb.CheckpointLSN(strFile); - - bitdb.mapFileUseCount.erase(mi++); + // The below line is commented out, because the above line (CheckpointLSN) should have never had + // lsn_reset in it. lsn_reset should only be called on the final flush when the wallet is closed. + // This is handled in CDB::Flush, which has a while loop that also does in the right place what + // the intention of the below line was. + // bitdb.mapFileUseCount.erase(mi++); if (fDebug10) printf("Flushed wallet.dat %" PRId64 "ms\n", GetTimeMillis() - nStart); } } From 7bef950fbcb2ca327776b7ffe804b677c172cca1 Mon Sep 17 00:00:00 2001 From: jamescowens Date: Fri, 9 Mar 2018 13:26:29 -0500 Subject: [PATCH 3/9] modifies wallet backup to do lsn_reset --- src/backup.cpp | 2 ++ src/db.cpp | 5 +++-- src/db.h | 2 +- src/walletdb.cpp | 8 ++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/backup.cpp b/src/backup.cpp index 2041f3182e..84550cd466 100644 --- a/src/backup.cpp +++ b/src/backup.cpp @@ -76,6 +76,8 @@ bool BackupWallet(const CWallet& wallet, const std::string& strDest) // Flush log data to the dat file bitdb.CloseDb(wallet.strWalletFile); bitdb.CheckpointLSN(wallet.strWalletFile); + printf("Issuing lsn_reset for backup file portability.\n"); + bitdb.lsn_reset(wallet.strWalletFile); bitdb.mapFileUseCount.erase(wallet.strWalletFile); // Copy wallet.dat diff --git a/src/db.cpp b/src/db.cpp index 851760c844..5eff316ad1 100755 --- a/src/db.cpp +++ b/src/db.cpp @@ -228,11 +228,12 @@ void CDBEnv::CheckpointLSN(std::string strFile) if (fMockDb) return; // The below line is commented out. LSN reset is not necessary and improper. Txn_checkpoint alone will flush in memory log to the database file. - // This was causing extraordinary long flush times for large wallets. - // Flush is called when the wallet is closed upon client shutdown, and the below line is included there to ensure portability of wallet.dat. + // This was causing extraordinary long flush times for large wallets. + // Flush is called when the wallet is closed upon client shutdown, and the below line is included there to ensure portability of wallet.dat. //dbenv.lsn_reset(strFile.c_str(), 0); } +void CDBEnv::lsn_reset(std::string strFile) { dbenv.lsn_reset(strFile.c_str(),0); } CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL), activeTxn(NULL) diff --git a/src/db.h b/src/db.h index 623dbfd369..f27b13560f 100644 --- a/src/db.h +++ b/src/db.h @@ -71,7 +71,7 @@ class CDBEnv void Close(); void Flush(bool fShutdown); void CheckpointLSN(std::string strFile); - + void lsn_reset(std::string strFile); void CloseDb(const std::string& strFile); bool RemoveDb(const std::string& strFile); diff --git a/src/walletdb.cpp b/src/walletdb.cpp index 0c93c64973..d0187f4120 100755 --- a/src/walletdb.cpp +++ b/src/walletdb.cpp @@ -670,10 +670,10 @@ void ThreadFlushWalletDB(void* parg) // Flush wallet.dat so it's self contained bitdb.CloseDb(strFile); bitdb.CheckpointLSN(strFile); - // The below line is commented out, because the above line (CheckpointLSN) should have never had - // lsn_reset in it. lsn_reset should only be called on the final flush when the wallet is closed. - // This is handled in CDB::Flush, which has a while loop that also does in the right place what - // the intention of the below line was. + // The below line is commented out, because the above line (CheckpointLSN) should have never had + // lsn_reset in it. lsn_reset should only be called on the final flush when the wallet is closed. + // This is handled in CDB::Flush, which has a while loop that also does in the right place what + // the intention of the below line was. // bitdb.mapFileUseCount.erase(mi++); if (fDebug10) printf("Flushed wallet.dat %" PRId64 "ms\n", GetTimeMillis() - nStart); } From 89b55dff9ff0c494e122244b5f1844be57db552e Mon Sep 17 00:00:00 2001 From: jamescowens Date: Fri, 9 Mar 2018 13:39:37 -0500 Subject: [PATCH 4/9] fix formatting problem --- src/db.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/db.cpp b/src/db.cpp index 5eff316ad1..41108be49e 100755 --- a/src/db.cpp +++ b/src/db.cpp @@ -233,7 +233,10 @@ void CDBEnv::CheckpointLSN(std::string strFile) //dbenv.lsn_reset(strFile.c_str(), 0); } -void CDBEnv::lsn_reset(std::string strFile) { dbenv.lsn_reset(strFile.c_str(),0); } +void CDBEnv::lsn_reset(std::string strFile) +{ +dbenv.lsn_reset(strFile.c_str(),0); +} CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL), activeTxn(NULL) From fad2f49b0a1f3b00e8c7cc86b7083c2bf3c68a27 Mon Sep 17 00:00:00 2001 From: jamescowens Date: Fri, 9 Mar 2018 17:58:43 -0500 Subject: [PATCH 5/9] Formatting change and passing strFile by ref --- src/db.cpp | 4 ++-- src/db.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 src/db.h diff --git a/src/db.cpp b/src/db.cpp index 41108be49e..5bc5c1ff2e 100755 --- a/src/db.cpp +++ b/src/db.cpp @@ -233,9 +233,9 @@ void CDBEnv::CheckpointLSN(std::string strFile) //dbenv.lsn_reset(strFile.c_str(), 0); } -void CDBEnv::lsn_reset(std::string strFile) +void CDBEnv::lsn_reset(const std::string& strFile) { -dbenv.lsn_reset(strFile.c_str(),0); + dbenv.lsn_reset(strFile.c_str(),0); } CDB::CDB(const char *pszFile, const char* pszMode) : diff --git a/src/db.h b/src/db.h old mode 100644 new mode 100755 index f27b13560f..c33d4e1b83 --- a/src/db.h +++ b/src/db.h @@ -71,7 +71,7 @@ class CDBEnv void Close(); void Flush(bool fShutdown); void CheckpointLSN(std::string strFile); - void lsn_reset(std::string strFile); + void lsn_reset(const std::string& strFile); void CloseDb(const std::string& strFile); bool RemoveDb(const std::string& strFile); From aa64e2626d033f44d57b5c79a62451fbfce6196b Mon Sep 17 00:00:00 2001 From: Marco Nilsson Date: Sat, 10 Mar 2018 08:27:10 +0100 Subject: [PATCH 6/9] Setting walletbackupinterval to 0 will now disable automatic backups. --- src/main.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 6d66a4325c..bfb66cacdd 100755 --- a/src/main.cpp +++ b/src/main.cpp @@ -4497,10 +4497,7 @@ void GridcoinServices() //Backup the wallet once per 900 blocks or as specified in config: int nWBI = GetArg("-walletbackupinterval", 900); - if (nWBI == 0) - nWBI = 900; - - if (TimerMain("backupwallet", nWBI)) + if (nWBI && TimerMain("backupwallet", nWBI)) { bool bWalletBackupResults = BackupWallet(*pwalletMain, GetBackupFilename("wallet.dat")); bool bConfigBackupResults = BackupConfigFile(GetBackupFilename("gridcoinresearch.conf")); From 90b0581f4eea196925221e95b3f17c915c1d4a41 Mon Sep 17 00:00:00 2001 From: Marco Nilsson Date: Sat, 10 Mar 2018 15:18:44 +0100 Subject: [PATCH 7/9] Move FixSpentCoins to wallet startup and reorganize. --- src/init.cpp | 6 +++++- src/main.cpp | 15 ++++----------- src/wallet.cpp | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 696ed0d5ff..fc353c642f 100755 --- a/src/init.cpp +++ b/src/init.cpp @@ -1032,8 +1032,12 @@ bool AppInit2(ThreadHandlerPtr threads) if (!strErrors.str().empty()) return InitError(strErrors.str()); - // Add wallet transactions that aren't already in a block to mapTransactions + // Add wallet transactions that aren't already in a block to mapTransactions pwalletMain->ReacceptWalletTransactions(); + int nMismatchSpent; + int64_t nBalanceInQuestion; + pwalletMain->FixSpentCoins(nMismatchSpent, nBalanceInQuestion); + return true; } diff --git a/src/main.cpp b/src/main.cpp index bfb66cacdd..617b0fabd6 100755 --- a/src/main.cpp +++ b/src/main.cpp @@ -3612,6 +3612,10 @@ bool ReorganizeChain(CTxDB& txdb, unsigned &cnt_dis, unsigned &cnt_con, CBlock & "Please Reindex the chain and Restart.\n"); exit(1); //todo } + + int nMismatchSpent; + int64_t nBalanceInQuestion; + pwalletMain->FixSpentCoins(nMismatchSpent, nBalanceInQuestion); } if (fDebug && cnt_dis>0) printf("ReorganizeChain: disconnected %d blocks\n",cnt_dis); @@ -4504,13 +4508,6 @@ void GridcoinServices() printf("Daily backup results: Wallet -> %s Config -> %s\r\n", (bWalletBackupResults ? "true" : "false"), (bConfigBackupResults ? "true" : "false")); } - if (false && TimerMain("FixSpentCoins",60)) - { - int nMismatchSpent; - int64_t nBalanceInQuestion; - pwalletMain->FixSpentCoins(nMismatchSpent, nBalanceInQuestion); - } - if (TimerMain("MyNeuralMagnitudeReport",30)) { try @@ -4526,10 +4523,6 @@ void GridcoinServices() { printf("Error in MyNeuralMagnitudeReport1."); } - catch(...) - { - printf("Error in MyNeuralMagnitudeReport."); - } } // Every N blocks as a Synchronized TEAM: diff --git a/src/wallet.cpp b/src/wallet.cpp index 6d95d65db2..7784ae4b19 100644 --- a/src/wallet.cpp +++ b/src/wallet.cpp @@ -2386,7 +2386,7 @@ void CWallet::FixSpentCoins(int& nMismatchFound, int64_t& nBalanceInQuestion, bo } else if (IsMine(pcoin->vout[n]) && !pcoin->IsSpent(n) && (txindex.vSpent.size() > n && !txindex.vSpent[n].IsNull())) { - if (fDebug) printf("FixSpentCoins found spent coin %s gC %s[%d], %s\n", + printf("FixSpentCoins found spent coin %s gC %s[%d], %s\n", FormatMoney(pcoin->vout[n].nValue).c_str(), pcoin->GetHash().ToString().c_str(), n, fCheckOnly? "repair not attempted" : "repairing"); nMismatchFound++; nBalanceInQuestion += pcoin->vout[n].nValue; From 934480824b389d4088371847906c6adc2861d606 Mon Sep 17 00:00:00 2001 From: Marco Nilsson Date: Thu, 15 Mar 2018 15:31:10 +0100 Subject: [PATCH 8/9] Update changelog. --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed13aeab70..2192b13564 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [3.7.11.0] Unreleased, leisure +### Fixed + - Fix wallet being locked while flushing. It now requires a clean shutdown + or a backup to migrate the wallet.dat to a different system, #1010 (@jamescowens). + +### Changed + - Automatic backups can now be disabled by using `-walletbackupinterval=0`, + #1018 (@denravonska). + - Trigger a fix spent coins check on start and after block disconnect, #1018 (@denravonska). + ## [3.7.10.0] 2018-03-05, leisure ### Fixed - Fix sync issues due to beacon age checks, #1003 (@denravonska). From 5d571a7a74de3b861a56ca898276f371c1e4d874 Mon Sep 17 00:00:00 2001 From: Marco Nilsson Date: Thu, 15 Mar 2018 15:33:24 +0100 Subject: [PATCH 9/9] Update version. --- CHANGELOG.md | 2 +- src/clientversion.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2192b13564..36eb1457c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [3.7.11.0] Unreleased, leisure +## [3.7.11.0] 2018-03-15, leisure ### Fixed - Fix wallet being locked while flushing. It now requires a clean shutdown or a backup to migrate the wallet.dat to a different system, #1010 (@jamescowens). diff --git a/src/clientversion.h b/src/clientversion.h index c909c360f1..65b371542c 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -8,7 +8,7 @@ // These need to be macros, as version.cpp's and bitcoin-qt.rc's voodoo requires it #define CLIENT_VERSION_MAJOR 3 #define CLIENT_VERSION_MINOR 7 -#define CLIENT_VERSION_REVISION 10 +#define CLIENT_VERSION_REVISION 11 #define CLIENT_VERSION_BUILD 0 // Converts the parameter X to a string after macro replacement on X has been performed.