From 76213cec5de364f4cb353aa87c52dc6202aacf4d Mon Sep 17 00:00:00 2001 From: Alright Date: Thu, 30 Mar 2023 10:16:17 -0400 Subject: [PATCH 1/8] backport zcash PR6192 memory optimizations --- src/chain.cpp | 42 ++++++++++++++++++++++ src/chain.h | 80 ++++++++++++++++++++++++++++-------------- src/main.cpp | 8 ++++- src/rest.cpp | 8 +++-- src/rpc/blockchain.cpp | 2 +- src/txdb.cpp | 31 +++++++++++++--- src/txdb.h | 4 ++- 7 files changed, 138 insertions(+), 37 deletions(-) diff --git a/src/chain.cpp b/src/chain.cpp index 5e87d88a781..8e91e6aca6d 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -24,6 +24,9 @@ #include "notaries_staked.h" #include "komodo_hardfork.h" +#include "main.h" +#include "txdb.h" + using namespace std; /** @@ -82,6 +85,45 @@ const CBlockIndex *CChain::FindFork(const CBlockIndex *pindex) const { return pindex; } +void CBlockIndex::TrimSolution() +{ + AssertLockHeld(cs_main); + + // We can correctly trim a solution as soon as the block index entry has been added + // to leveldb. Updates to the block index entry (to update validity status) will be + // handled by re-reading the solution from the existing db entry. It does not help to + // try to avoid these reads by gating trimming on the validity status: the re-reads are + // efficient anyway because of caching in leveldb, and most of them are unavoidable. + if (HasSolution()) { + std::vector empty; + nSolution.swap(empty); + } +} + +CBlockHeader CBlockIndex::GetBlockHeader() const +{ + AssertLockHeld(cs_main); + + CBlockHeader header; + header.nVersion = nVersion; + if (pprev) { + header.hashPrevBlock = pprev->GetBlockHash(); + } + header.hashMerkleRoot = hashMerkleRoot; + header.hashFinalSaplingRoot = hashFinalSaplingRoot; + header.nTime = nTime; + header.nBits = nBits; + header.nNonce = nNonce; + if (HasSolution()) { + header.nSolution = nSolution; + } else { + CDiskBlockIndex dbindex; + assert(pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex)); + header.nSolution = dbindex.GetSolution(); + } + return header; +} + /** Turn the lowest '1' bit in the binary representation of a number into a '0'. */ int static inline InvertLowestOne(int n) { return n & (n - 1); } diff --git a/src/chain.h b/src/chain.h index 734c21ddbab..91cb7c30b4d 100644 --- a/src/chain.h +++ b/src/chain.h @@ -27,12 +27,14 @@ #include "tinyformat.h" #include "uint256.h" #include "sync.h" +#include "util/strencodings.h" #include "assetchain.h" #include #include + extern CCriticalSection cs_main; static const int SPROUT_VALUE_VERSION = 1001400; @@ -216,8 +218,14 @@ class CBlockIndex unsigned int nTime; unsigned int nBits; uint256 nNonce; + +protected: + // The Equihash solution, if it is stored. Once we know that the block index + // entry is present in leveldb, this field can be cleared via the TrimSolution + // method to save memory. std::vector nSolution; +public: //! (memory only) Sequential id assigned to distinguish order in which blocks are received. uint32_t nSequenceId; @@ -291,20 +299,11 @@ class CBlockIndex return ret; } - CBlockHeader GetBlockHeader() const - { - CBlockHeader block; - block.nVersion = nVersion; - if (pprev) - block.hashPrevBlock = pprev->GetBlockHash(); - block.hashMerkleRoot = hashMerkleRoot; - block.hashFinalSaplingRoot = hashFinalSaplingRoot; - block.nTime = nTime; - block.nBits = nBits; - block.nNonce = nNonce; - block.nSolution = nSolution; - return block; - } + //! Get the block header for this block index. Requires cs_main. + CBlockHeader GetBlockHeader() const; + + //! Clear the Equihash solution to save memory. Requires cs_main. + void TrimSolution(); uint256 GetBlockHash() const { @@ -341,10 +340,11 @@ class CBlockIndex std::string ToString() const { - return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s)", + return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s, HasSolution=%s)", pprev, nHeight, hashMerkleRoot.ToString(), - GetBlockHash().ToString()); + phashBlock ? GetBlockHash().ToString() : "(nil)", + HasSolution()); } //! Check whether this block index entry is valid up to the passed validity level. @@ -356,6 +356,12 @@ class CBlockIndex return ((nStatus & BLOCK_VALID_MASK) >= nUpTo); } + //! Is the Equihash solution stored? + bool HasSolution() const + { + return !nSolution.empty(); + } + //! Raise the validity level of this block index entry. //! Returns true if the validity was changed. bool RaiseValidity(enum BlockStatus nUpTo) @@ -388,8 +394,11 @@ class CDiskBlockIndex : public CBlockIndex hashPrev = uint256(); } - explicit CDiskBlockIndex(const CBlockIndex* pindex) : CBlockIndex(*pindex) { + explicit CDiskBlockIndex(const CBlockIndex* pindex, std::function()> getSolution) : CBlockIndex(*pindex) { hashPrev = (pprev ? pprev->GetBlockHash() : uint256()); + if (!HasSolution()) { + nSolution = getSolution(); + } } ADD_SERIALIZE_METHODS; @@ -459,21 +468,38 @@ class CDiskBlockIndex : public CBlockIndex bool isStakedAndNotaryPay() const; bool isStakedAndAfterDec2019(unsigned int nTime) const; + //! This method should not be called on a CDiskBlockIndex. + void TrimSolution() + { + assert(!"called CDiskBlockIndex::TrimSolution"); + } + public: uint256 GetBlockHash() const { - CBlockHeader block; - block.nVersion = nVersion; - block.hashPrevBlock = hashPrev; - block.hashMerkleRoot = hashMerkleRoot; - block.hashFinalSaplingRoot = hashFinalSaplingRoot; - block.nTime = nTime; - block.nBits = nBits; - block.nNonce = nNonce; - block.nSolution = nSolution; - return block.GetHash(); + return GetBlockHeader().GetHash(); + } + + //! Get the block header for this block index. + CBlockHeader GetBlockHeader() const + { + CBlockHeader header; + header.nVersion = nVersion; + header.hashPrevBlock = hashPrev; + header.hashMerkleRoot = hashMerkleRoot; + header.hashFinalSaplingRoot = hashFinalSaplingRoot; + header.nTime = nTime; + header.nBits = nBits; + header.nNonce = nNonce; + header.nSolution = nSolution; + return header; } + std::vector GetSolution() const + { + assert(HasSolution()); + return nSolution; + } std::string ToString() const { diff --git a/src/main.cpp b/src/main.cpp index abc02adedea..469c9126540 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3864,7 +3864,7 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { vFiles.push_back(make_pair(*it, &vinfoBlockFile[*it])); setDirtyFileInfo.erase(it++); } - std::vector vBlocks; + std::vector vBlocks; vBlocks.reserve(setDirtyBlockIndex.size()); for (set::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { vBlocks.push_back(*it); @@ -3873,6 +3873,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { return AbortNode(state, "Files to write to block index database"); } + // Now that we have written the block indices to the database, we do not + // need to store solutions for these CBlockIndex objects in memory. + // cs_main must be held here. + for (CBlockIndex *pblockindex : vBlocks) { + pblockindex->TrimSolution(); + } } // Finally remove any pruned files if (fFlushForPrune) diff --git a/src/rest.cpp b/src/rest.cpp index afedad606f0..754b2421951 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -149,6 +149,7 @@ static bool rest_headers(HTTPRequest* req, std::vector headers; headers.reserve(count); + CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); { LOCK(cs_main); BlockMap::const_iterator it = mapBlockIndex.find(hash); @@ -161,9 +162,10 @@ static bool rest_headers(HTTPRequest* req, } } - CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); - BOOST_FOREACH(const CBlockIndex *pindex, headers) { - ssHeader << pindex->GetBlockHeader(); + if (rf == RF_BINARY || rf == RF_HEX) { + for (const CBlockIndex *pindex : headers) { + ssHeader << pindex->GetBlockHeader(); + } } switch (rf) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index fc32eb5ec7f..9f61b0e073a 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -160,7 +160,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) result.push_back(Pair("finalsaplingroot", blockindex->hashFinalSaplingRoot.GetHex())); result.push_back(Pair("time", (int64_t)blockindex->nTime)); result.push_back(Pair("nonce", blockindex->nNonce.GetHex())); - result.push_back(Pair("solution", HexStr(blockindex->nSolution))); + result.pushKV("solution", HexStr(blockindex->GetBlockHeader().nSolution)); result.push_back(Pair("bits", strprintf("%08x", blockindex->nBits))); result.push_back(Pair("difficulty", GetDifficulty(blockindex))); result.push_back(Pair("chainwork", blockindex->nChainWork.GetHex())); diff --git a/src/txdb.cpp b/src/txdb.cpp index f38cff6cacc..c2be5e04a1b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -293,7 +293,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { * @returns true on success */ bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, - const std::vector& blockinfo) + const std::vector& blockinfo) { CDBBatch batch(*this); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { @@ -301,7 +301,22 @@ bool CBlockTreeDB::WriteBatchSync(const std::vector::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) { - batch.Write(make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it)); + std::pair key = make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()); + try { + CDiskBlockIndex dbindex {*it, [this, &key]() { + // It can happen that the index entry is written, then the Equihash solution is cleared from memory, + // then the index entry is rewritten. In that case we must read the solution from the old entry. + CDiskBlockIndex dbindex_old; + if (!Read(key, dbindex_old)) { + LogPrintf("%s: Failed to read index entry", __func__); + throw runtime_error("Failed to read index entry"); + } + return dbindex_old.GetSolution(); + }}; + batch.Write(key, dbindex); + } catch (const runtime_error&) { + return false; + } } return WriteBatch(batch, true); } @@ -314,6 +329,10 @@ bool CBlockTreeDB::EraseBatchSync(const std::vector& blockin return WriteBatch(batch, true); } +bool CBlockTreeDB::ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) { + return Read(make_pair(DB_BLOCK_INDEX, blockhash), dbindex); +} + bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { return Read(make_pair(DB_TXINDEX, txid), pos); } @@ -716,7 +735,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pindexNew->nTime = diskindex.nTime; pindexNew->nBits = diskindex.nBits; pindexNew->nNonce = diskindex.nNonce; - pindexNew->nSolution = diskindex.nSolution; + // the Equihash solution will be loaded lazily from the dbindex entry pindexNew->nStatus = diskindex.nStatus; pindexNew->nCachedBranchId = diskindex.nCachedBranchId; pindexNew->nTx = diskindex.nTx; @@ -725,7 +744,11 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pindexNew->segid = diskindex.segid; pindexNew->nNotaryPay = diskindex.nNotaryPay; // Consistency checks - auto header = pindexNew->GetBlockHeader(); + CBlockHeader header; + { + LOCK(cs_main); + header = pindexNew->GetBlockHeader(); + } if (header.GetHash() != pindexNew->GetBlockHash()) return error("LoadBlockIndex(): block header inconsistency detected: on-disk = %s, in-memory = %s", diskindex.ToString(), pindexNew->ToString()); diff --git a/src/txdb.h b/src/txdb.h index 229890e9b8e..206c752bd22 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -129,7 +129,7 @@ class CBlockTreeDB : public CDBWrapper * @returns true on success */ bool WriteBatchSync(const std::vector >& fileInfo, int nLastFile, - const std::vector& blockinfo); + const std::vector& blockinfo); /*** * Erase a batch of block index records and sync * @param blockinfo the records @@ -161,6 +161,8 @@ class CBlockTreeDB : public CDBWrapper * @returns true on success */ bool ReadReindexing(bool &fReindex); + + bool ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex); /*** * Retrieve the location of a particular transaction index value * @param txid what to look for From 9156f900cc77287115ce98281bb711150b169cd9 Mon Sep 17 00:00:00 2001 From: Alright Date: Thu, 30 Mar 2023 10:31:45 -0400 Subject: [PATCH 2/8] update copyright & fix nspv nsolution --- src/chain.cpp | 1 + src/chain.h | 2 +- src/komodo_nSPV_fullnode.h | 2 +- src/main.cpp | 4 ++-- src/rest.cpp | 1 + src/rpc/blockchain.cpp | 1 + src/txdb.cpp | 1 + src/txdb.h | 1 + 8 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/chain.cpp b/src/chain.cpp index 8e91e6aca6d..ca2d8f20be8 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2017-2022 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chain.h b/src/chain.h index 91cb7c30b4d..f85fcf1de17 100644 --- a/src/chain.h +++ b/src/chain.h @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -27,7 +28,6 @@ #include "tinyformat.h" #include "uint256.h" #include "sync.h" -#include "util/strencodings.h" #include "assetchain.h" #include diff --git a/src/komodo_nSPV_fullnode.h b/src/komodo_nSPV_fullnode.h index 5c8d157d07c..7176977870d 100644 --- a/src/komodo_nSPV_fullnode.h +++ b/src/komodo_nSPV_fullnode.h @@ -124,7 +124,7 @@ int32_t NSPV_setequihdr(struct NSPV_equihdr *hdr,int32_t height) hdr->nTime = pindex->nTime; hdr->nBits = pindex->nBits; hdr->nNonce = pindex->nNonce; - memcpy(hdr->nSolution,&pindex->nSolution[0],sizeof(hdr->nSolution)); + memcpy(hdr->nSolution,&pindex->GetBlockHeader().nSolution[0],sizeof(hdr->nSolution)); return(sizeof(*hdr)); } return(-1); diff --git a/src/main.cpp b/src/main.cpp index 469c9126540..23c4400001c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,7 +1,7 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers -// Copyright (c) 2015-2020 The Zcash developers -// Copyright (c) 2015-2020 The Komodo Platform developers +// Copyright (c) 2015-2022 The Zcash developers +// Copyright (c) 2015-2023 The Komodo Platform developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rest.cpp b/src/rest.cpp index 754b2421951..5907efd4ffd 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2017-2022 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 9f61b0e073a..f0b3366871c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,5 +1,6 @@ // Copyright (c) 2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2019-2022 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/txdb.cpp b/src/txdb.cpp index c2be5e04a1b..899413e54ab 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/txdb.h b/src/txdb.h index 206c752bd22..23e3dbeac79 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. From ad5065fe645eb351f8e7e56557ee6fde2d04e5e9 Mon Sep 17 00:00:00 2001 From: Alright Date: Thu, 30 Mar 2023 14:22:08 -0400 Subject: [PATCH 3/8] latest zcash blockindex Write/Erase --- src/txdb.cpp | 14 ++++++-------- src/txdb.h | 1 + 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index 899413e54ab..523da6de9cf 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -293,18 +293,16 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { * @param blockinfo the index records to write * @returns true on success */ -bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, - const std::vector& blockinfo) -{ +bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { CDBBatch batch(*this); - for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { - batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); + for (const auto& it : fileInfo) { + batch.Write(make_pair(DB_BLOCK_FILES, it.first), *it.second); } batch.Write(DB_LAST_BLOCK, nLastFile); - for (std::vector::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) { - std::pair key = make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()); + for (const auto& it : blockinfo) { + std::pair key = make_pair(DB_BLOCK_INDEX, it->GetBlockHash()); try { - CDiskBlockIndex dbindex {*it, [this, &key]() { + CDiskBlockIndex dbindex {it, [this, &key]() { // It can happen that the index entry is written, then the Equihash solution is cleared from memory, // then the index entry is rewritten. In that case we must read the solution from the old entry. CDiskBlockIndex dbindex_old; diff --git a/src/txdb.h b/src/txdb.h index 23e3dbeac79..4fc70e6b6a3 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -46,6 +46,7 @@ struct CTimestampBlockIndexValue; struct CSpentIndexKey; struct CSpentIndexValue; class uint256; +class CDiskBlockIndex; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; From bb595acf70c3ff9d5496e0ede866b5c53d67217b Mon Sep 17 00:00:00 2001 From: Alright Date: Thu, 30 Mar 2023 14:26:44 -0400 Subject: [PATCH 4/8] temporarily disable haraka tests --- src/test-komodo/test_haraka_removal.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test-komodo/test_haraka_removal.cpp b/src/test-komodo/test_haraka_removal.cpp index 4cfd65f9507..aa7b5c465f2 100644 --- a/src/test-komodo/test_haraka_removal.cpp +++ b/src/test-komodo/test_haraka_removal.cpp @@ -22,6 +22,8 @@ namespace TestHarakaRemoval { CChain chainActive; CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); +/* +FIXME - Alright TEST(TestHarakaRemoval, test_block_ser) { // this test should cover genesis block (de)serialization before @@ -62,7 +64,7 @@ namespace TestHarakaRemoval { static constexpr const char* stream_genesis_block_diskindex_hex = "89af1b00000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a000000000000000000000000000000000000000000000000000000000000000029ab5f490f0f0f200b00000000000000000000000000000000000000000000000000000000000000fd4005000d5ba7cda5d473947263bf194285317179d2b0d307119c2e7cc4bd8ac456f0774bd52b0cd9249be9d40718b6397a4c7bbd8f2b3272fed2823cd2af4bd1632200ba4bf796727d6347b225f670f292343274cc35099466f5fb5f0cd1c105121b28213d15db2ed7bdba490b4cedc69742a57b7c25af24485e523aadbb77a0144fc76f79ef73bd8530d42b9f3b9bed1c135ad1fe152923fafe98f95f76f1615e64c4abb1137f4c31b218ba2782bc15534788dda2cc08a0ee2987c8b27ff41bd4e31cd5fb5643dfe862c9a02ca9f90c8c51a6671d681d04ad47e4b53b1518d4befafefe8cadfb912f3d03051b1efbf1dfe37b56e93a741d8dfd80d576ca250bee55fab1311fc7b3255977558cdda6f7d6f875306e43a14413facdaed2f46093e0ef1e8f8a963e1632dcbeebd8e49fd16b57d49b08f9762de89157c65233f60c8e38a1f503a48c555f8ec45dedecd574a37601323c27be597b956343107f8bd80f3a925afaf30811df83c402116bb9c1e5231c70fff899a7c82f73c902ba54da53cc459b7bf1113db65cc8f6914d3618560ea69abd13658fa7b6af92d374d6eca9529f8bd565166e4fcbf2a8dfb3c9b69539d4d2ee2e9321b85b331925df195915f2757637c2805e1d4131e1ad9ef9bc1bb1c732d8dba4738716d351ab30c996c8657bab39567ee3b29c6d054b711495c0d52e1cd5d8e55b4f0f0325b97369280755b46a02afd54be4ddd9f77c22272b8bbb17ff5118fedbae2564524e797bd28b5f74f7079d532ccc059807989f94d267f47e724b3f1ecfe00ec9e6541c961080d8891251b84b4480bc292f6a180bea089fef5bbda56e1e41390d7c0e85ba0ef530f7177413481a226465a36ef6afe1e2bca69d2078712b3912bba1a99b1fbff0d355d6ffe726d2bb6fbc103c4ac5756e5bee6e47e17424ebcbf1b63d8cb90ce2e40198b4f4198689daea254307e52a25562f4c1455340f0ffeb10f9d8e914775e37d0edca019fb1b9c6ef81255ed86bc51c5391e0591480f66e2d88c5f4fd7277697968656a9b113ab97f874fdd5f2465e5559533e01ba13ef4a8f7a21d02c30c8ded68e8c54603ab9c8084ef6d9eb4e92c75b078539e2ae786ebab6dab73a09e0aa9ac575bcefb29e930ae656e58bcb513f7e3c17e079dce4f05b5dbc18c2a872b22509740ebe6a3903e00ad1abc55076441862643f93606e3dc35e8d9f2caef3ee6be14d513b2e062b21d0061de3bd56881713a1a5c17f5ace05e1ec09da53f99442df175a49bd154aa96e4949decd52fed79ccf7ccbce32941419c314e374e4a396ac553e17b5340336a1a25c22f9e42a243ba5404450b650acfc826a6e432971ace776e15719515e1634ceb9a4a35061b668c74998d3dfb5827f6238ec015377e6f9c94f38108768cf6e5c8b132e0303fb5a200368f845ad9d46343035a6ff94031df8d8309415bb3f6cd5ede9c135fdabcc030599858d803c0f85be7661c88984d88faa3d26fb0e9aac0056a53f1b5d0baed713c853c4a2726869a0a124a8a5bbc0fc0ef80c8ae4cb53636aa02503b86a1eb9836fcc259823e2692d921d88e1ffc1e6cb2bde43939ceb3f32a611686f539f8f7c9f0bf00381f743607d40960f06d347d1cd8ac8a51969c25e37150efdf7aa4c2037a2fd0516fb444525ab157a0ed0a7412b2fa69b217fe397263153782c0f64351fbdf2678fa0dc8569912dcd8e3ccad38f34f23bbbce14c6a26ac24911b308b82c7e43062d180baeac4ba7153858365c72c63dcf5f6a5b08070b730adb017aeae925b7d0439979e2679f45ed2f25a7edcfd2fb77a8794630285ccb0a071f5cce410b46dbf9750b0354aae8b65574501cc69efb5b6a43444074fee116641bb29da56c2b4a7f456991fc92b2"; EXPECT_EQ(HexStr(ss), stream_genesis_block_diskindex_hex); } - +*/ //#define PRG_VALUES_BEFORE_PR 1 TEST(TestHarakaRemoval, test_block_prg) { From 3aa80e684eb9689052a645c6c393792456ba40dc Mon Sep 17 00:00:00 2001 From: dimxy Date: Fri, 31 Mar 2023 18:25:43 +0500 Subject: [PATCH 5/8] fix test haraka --- src/test-komodo/test_haraka_removal.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test-komodo/test_haraka_removal.cpp b/src/test-komodo/test_haraka_removal.cpp index aa7b5c465f2..fac32e63804 100644 --- a/src/test-komodo/test_haraka_removal.cpp +++ b/src/test-komodo/test_haraka_removal.cpp @@ -22,8 +22,6 @@ namespace TestHarakaRemoval { CChain chainActive; CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); -/* -FIXME - Alright TEST(TestHarakaRemoval, test_block_ser) { // this test should cover genesis block (de)serialization before @@ -57,14 +55,14 @@ FIXME - Alright EXPECT_TRUE(chainActive.Contains(&fakeIndex)); EXPECT_EQ(0, chainActive.Height()); - CDiskBlockIndex diskindex {&fakeIndex}; + CDiskBlockIndex diskindex(&fakeIndex, [](){ return std::vector(); }); ss.clear(); ss << diskindex; // VARINT(PROTOCOL_VERSION==170010) - 89af1b static constexpr const char* stream_genesis_block_diskindex_hex = "89af1b00000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a000000000000000000000000000000000000000000000000000000000000000029ab5f490f0f0f200b00000000000000000000000000000000000000000000000000000000000000fd4005000d5ba7cda5d473947263bf194285317179d2b0d307119c2e7cc4bd8ac456f0774bd52b0cd9249be9d40718b6397a4c7bbd8f2b3272fed2823cd2af4bd1632200ba4bf796727d6347b225f670f292343274cc35099466f5fb5f0cd1c105121b28213d15db2ed7bdba490b4cedc69742a57b7c25af24485e523aadbb77a0144fc76f79ef73bd8530d42b9f3b9bed1c135ad1fe152923fafe98f95f76f1615e64c4abb1137f4c31b218ba2782bc15534788dda2cc08a0ee2987c8b27ff41bd4e31cd5fb5643dfe862c9a02ca9f90c8c51a6671d681d04ad47e4b53b1518d4befafefe8cadfb912f3d03051b1efbf1dfe37b56e93a741d8dfd80d576ca250bee55fab1311fc7b3255977558cdda6f7d6f875306e43a14413facdaed2f46093e0ef1e8f8a963e1632dcbeebd8e49fd16b57d49b08f9762de89157c65233f60c8e38a1f503a48c555f8ec45dedecd574a37601323c27be597b956343107f8bd80f3a925afaf30811df83c402116bb9c1e5231c70fff899a7c82f73c902ba54da53cc459b7bf1113db65cc8f6914d3618560ea69abd13658fa7b6af92d374d6eca9529f8bd565166e4fcbf2a8dfb3c9b69539d4d2ee2e9321b85b331925df195915f2757637c2805e1d4131e1ad9ef9bc1bb1c732d8dba4738716d351ab30c996c8657bab39567ee3b29c6d054b711495c0d52e1cd5d8e55b4f0f0325b97369280755b46a02afd54be4ddd9f77c22272b8bbb17ff5118fedbae2564524e797bd28b5f74f7079d532ccc059807989f94d267f47e724b3f1ecfe00ec9e6541c961080d8891251b84b4480bc292f6a180bea089fef5bbda56e1e41390d7c0e85ba0ef530f7177413481a226465a36ef6afe1e2bca69d2078712b3912bba1a99b1fbff0d355d6ffe726d2bb6fbc103c4ac5756e5bee6e47e17424ebcbf1b63d8cb90ce2e40198b4f4198689daea254307e52a25562f4c1455340f0ffeb10f9d8e914775e37d0edca019fb1b9c6ef81255ed86bc51c5391e0591480f66e2d88c5f4fd7277697968656a9b113ab97f874fdd5f2465e5559533e01ba13ef4a8f7a21d02c30c8ded68e8c54603ab9c8084ef6d9eb4e92c75b078539e2ae786ebab6dab73a09e0aa9ac575bcefb29e930ae656e58bcb513f7e3c17e079dce4f05b5dbc18c2a872b22509740ebe6a3903e00ad1abc55076441862643f93606e3dc35e8d9f2caef3ee6be14d513b2e062b21d0061de3bd56881713a1a5c17f5ace05e1ec09da53f99442df175a49bd154aa96e4949decd52fed79ccf7ccbce32941419c314e374e4a396ac553e17b5340336a1a25c22f9e42a243ba5404450b650acfc826a6e432971ace776e15719515e1634ceb9a4a35061b668c74998d3dfb5827f6238ec015377e6f9c94f38108768cf6e5c8b132e0303fb5a200368f845ad9d46343035a6ff94031df8d8309415bb3f6cd5ede9c135fdabcc030599858d803c0f85be7661c88984d88faa3d26fb0e9aac0056a53f1b5d0baed713c853c4a2726869a0a124a8a5bbc0fc0ef80c8ae4cb53636aa02503b86a1eb9836fcc259823e2692d921d88e1ffc1e6cb2bde43939ceb3f32a611686f539f8f7c9f0bf00381f743607d40960f06d347d1cd8ac8a51969c25e37150efdf7aa4c2037a2fd0516fb444525ab157a0ed0a7412b2fa69b217fe397263153782c0f64351fbdf2678fa0dc8569912dcd8e3ccad38f34f23bbbce14c6a26ac24911b308b82c7e43062d180baeac4ba7153858365c72c63dcf5f6a5b08070b730adb017aeae925b7d0439979e2679f45ed2f25a7edcfd2fb77a8794630285ccb0a071f5cce410b46dbf9750b0354aae8b65574501cc69efb5b6a43444074fee116641bb29da56c2b4a7f456991fc92b2"; EXPECT_EQ(HexStr(ss), stream_genesis_block_diskindex_hex); } -*/ + //#define PRG_VALUES_BEFORE_PR 1 TEST(TestHarakaRemoval, test_block_prg) { From 46c237a16f910ff562f8caa942a3c972caf2bd9e Mon Sep 17 00:00:00 2001 From: DeckerSU Date: Thu, 25 May 2023 22:11:50 +0200 Subject: [PATCH 6/8] Error handling improvements. - https://github.com/DeckerSU/KomodoOcean/commit/f7d014d546b81c5efa28c18af78d61b984037abe - https://github.com/DeckerSU/KomodoOcean/commit/52334a63e81132a5034b4c34241c8774c54b0fef --- src/chain.cpp | 5 ++++- src/chain.h | 1 + src/main.cpp | 4 ++++ src/rest.cpp | 6 +++++- src/rpc/blockchain.cpp | 11 +++++++---- src/txdb.cpp | 7 ++++++- 6 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/chain.cpp b/src/chain.cpp index ca2d8f20be8..7ab394240f5 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -119,7 +119,10 @@ CBlockHeader CBlockIndex::GetBlockHeader() const header.nSolution = nSolution; } else { CDiskBlockIndex dbindex; - assert(pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex)); + if (!pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex)) { + LogPrintf("%s: Failed to read index entry", __func__); + throw std::runtime_error("Failed to read index entry"); + } header.nSolution = dbindex.GetSolution(); } return header; diff --git a/src/chain.h b/src/chain.h index f85fcf1de17..6e2b19c47c8 100644 --- a/src/chain.h +++ b/src/chain.h @@ -307,6 +307,7 @@ class CBlockIndex uint256 GetBlockHash() const { + assert(phashBlock); return *phashBlock; } diff --git a/src/main.cpp b/src/main.cpp index 23c4400001c..a3468c3d636 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6814,7 +6814,11 @@ void static CheckBlockIndex() } } } + // try { // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow + // } catch (const runtime_error&) { + // assert(!"Failed to read index entry"); + // } // End: actual consistency checks. // Try descending into the first subnode. diff --git a/src/rest.cpp b/src/rest.cpp index 5907efd4ffd..18f4e61b45f 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -161,11 +161,15 @@ static bool rest_headers(HTTPRequest* req, break; pindex = chainActive.Next(pindex); } - } if (rf == RF_BINARY || rf == RF_HEX) { + try { for (const CBlockIndex *pindex : headers) { ssHeader << pindex->GetBlockHeader(); + } + } catch (const std::runtime_error&) { + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Failed to read index entry"); + } } } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f0b3366871c..3eb9b11422d 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -809,15 +809,18 @@ UniValue getblockheader(const UniValue& params, bool fHelp, const CPubKey& mypk) CBlockIndex* pblockindex = mapBlockIndex[hash]; - if (!fVerbose) - { + try { + if (!fVerbose) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); ssBlock << pblockindex->GetBlockHeader(); std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); return strHex; - } - + } else { return blockheaderToJSON(pblockindex); + } + } catch (const runtime_error&) { + throw JSONRPCError(RPC_DATABASE_ERROR, "Failed to read index entry"); + } } UniValue getblock(const UniValue& params, bool fHelp, const CPubKey& mypk) diff --git a/src/txdb.cpp b/src/txdb.cpp index 523da6de9cf..e8e9d485f23 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -746,7 +746,12 @@ bool CBlockTreeDB::LoadBlockIndexGuts() CBlockHeader header; { LOCK(cs_main); - header = pindexNew->GetBlockHeader(); + try { + header = pindexNew->GetBlockHeader(); + } catch (const runtime_error&) { + return error("LoadBlockIndex(): failed to read index entry: diskindex hash = %s", + diskindex.GetBlockHash().ToString()); + } } if (header.GetHash() != pindexNew->GetBlockHash()) return error("LoadBlockIndex(): block header inconsistency detected: on-disk = %s, in-memory = %s", From 3115f2000e2f2a79b4acb6ee05fb5f71a381a3c6 Mon Sep 17 00:00:00 2001 From: DeckerSU Date: Thu, 25 May 2023 22:43:45 +0200 Subject: [PATCH 7/8] Declare CBlockTreeDB::Read* methods as const when they are trivially so. https://github.com/DeckerSU/KomodoOcean/commit/1f981a8fc4865cdaca11fa0827178dcd4c611dc7 --- src/txdb.cpp | 16 ++++++++-------- src/txdb.h | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index e8e9d485f23..7c740f20750 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -220,7 +220,7 @@ CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe, bool com : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe, compression, maxOpenFiles) { } -bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { +bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) const { return Read(make_pair(DB_BLOCK_FILES, nFile), info); } @@ -231,12 +231,12 @@ bool CBlockTreeDB::WriteReindexing(bool fReindexing) { return Erase(DB_REINDEX_FLAG); } -bool CBlockTreeDB::ReadReindexing(bool &fReindexing) { +bool CBlockTreeDB::ReadReindexing(bool &fReindexing) const { fReindexing = Exists(DB_REINDEX_FLAG); return true; } -bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { +bool CBlockTreeDB::ReadLastBlockFile(int &nFile) const { return Read(DB_LAST_BLOCK, nFile); } @@ -328,11 +328,11 @@ bool CBlockTreeDB::EraseBatchSync(const std::vector& blockin return WriteBatch(batch, true); } -bool CBlockTreeDB::ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) { +bool CBlockTreeDB::ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const { return Read(make_pair(DB_BLOCK_INDEX, blockhash), dbindex); } -bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { +bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const { return Read(make_pair(DB_TXINDEX, txid), pos); } @@ -343,7 +343,7 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } -bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) { +bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) const { return Read(make_pair(DB_SPENTINDEX, key), value); } @@ -672,7 +672,7 @@ bool CBlockTreeDB::WriteTimestampBlockIndex(const CTimestampBlockIndexKey &block return WriteBatch(batch); } -bool CBlockTreeDB::ReadTimestampBlockIndex(const uint256 &hash, unsigned int <imestamp) { +bool CBlockTreeDB::ReadTimestampBlockIndex(const uint256 &hash, unsigned int <imestamp) const { CTimestampBlockIndexValue(lts); if (!Read(std::make_pair(DB_BLOCKHASHINDEX, hash), lts)) @@ -686,7 +686,7 @@ bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } -bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { +bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) const { char ch; if (!Read(std::make_pair(DB_FLAG, name), ch)) return false; diff --git a/src/txdb.h b/src/txdb.h index 4fc70e6b6a3..0e7908c7d88 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -144,13 +144,13 @@ class CBlockTreeDB : public CDBWrapper * @param fileinfo where to store the results * @returns true on success */ - bool ReadBlockFileInfo(int nFile, CBlockFileInfo &fileinfo); + bool ReadBlockFileInfo(int nFile, CBlockFileInfo &fileinfo) const; /**** * Read the value of DB_LAST_BLOCK * @param nFile where to store the results * @returns true on success */ - bool ReadLastBlockFile(int &nFile); + bool ReadLastBlockFile(int &nFile) const; /*** * Write to the DB_REINDEX_FLAG * @param fReindex true to set DB_REINDEX_FLAG, false to erase the key @@ -162,16 +162,16 @@ class CBlockTreeDB : public CDBWrapper * @param fReindex true if DB_REINDEX_FLAG exists * @returns true on success */ - bool ReadReindexing(bool &fReindex); + bool ReadReindexing(bool &fReindex) const; - bool ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex); + bool ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const; /*** * Retrieve the location of a particular transaction index value * @param txid what to look for * @param pos the results * @returns true on success */ - bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); + bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const; /**** * Write transaction index records * @param list the records to write @@ -184,7 +184,7 @@ class CBlockTreeDB : public CDBWrapper * @param value the value * @returns true on success */ - bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); + bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) const; /**** * Update a batch of spent index entries * @param vect the entries to add/update @@ -259,7 +259,7 @@ class CBlockTreeDB : public CDBWrapper * @param logicalTS the timestamp (the value) * @returns true on success */ - bool ReadTimestampBlockIndex(const uint256 &hash, unsigned int &logicalTS); + bool ReadTimestampBlockIndex(const uint256 &hash, unsigned int &logicalTS) const; /*** * Store a flag value in the DB * @param name the key @@ -273,7 +273,7 @@ class CBlockTreeDB : public CDBWrapper * @param fValue the value * @returns true on success */ - bool ReadFlag(const std::string &name, bool &fValue); + bool ReadFlag(const std::string &name, bool &fValue) const; /**** * Load the block headers from disk * NOTE: this does no consistency check beyond verifying records exist From 688de6060cd9f94269604d7ab9078d8a262ee977 Mon Sep 17 00:00:00 2001 From: DeckerSU Date: Thu, 25 May 2023 22:57:55 +0200 Subject: [PATCH 8/8] txdb: get rid of consistency checks Details: https://github.com/DeckerSU/KomodoOcean/commit/24eaf1e455e650e28dad7f76b335e4aee3bb3d82 --- src/txdb.cpp | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index 7c740f20750..b87d3487439 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -742,22 +742,24 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pindexNew->nSaplingValue = diskindex.nSaplingValue; pindexNew->segid = diskindex.segid; pindexNew->nNotaryPay = diskindex.nNotaryPay; - // Consistency checks - CBlockHeader header; - { - LOCK(cs_main); - try { - header = pindexNew->GetBlockHeader(); - } catch (const runtime_error&) { - return error("LoadBlockIndex(): failed to read index entry: diskindex hash = %s", - diskindex.GetBlockHash().ToString()); - } - } - if (header.GetHash() != pindexNew->GetBlockHash()) - return error("LoadBlockIndex(): block header inconsistency detected: on-disk = %s, in-memory = %s", - diskindex.ToString(), pindexNew->ToString()); + if ( 0 ) // POW will be checked before any block is connected { + // Consistency checks + CBlockHeader header; + { + LOCK(cs_main); + try { + header = pindexNew->GetBlockHeader(); + } catch (const runtime_error&) { + return error("LoadBlockIndex(): failed to read index entry: diskindex hash = %s", + diskindex.GetBlockHash().ToString()); + } + } + if (header.GetHash() != pindexNew->GetBlockHash()) + return error("LoadBlockIndex(): block header inconsistency detected: on-disk = %s, in-memory = %s", + diskindex.ToString(), pindexNew->ToString()); + uint8_t pubkey33[33]; komodo_index2pubkey33(pubkey33,pindexNew,pindexNew->nHeight); if (!CheckProofOfWork(header,pubkey33,pindexNew->nHeight,Params().GetConsensus()))