Merge pull request #582 from Alrighttt/zcash-6192-backport

zcash backport of memory optimizations

src/chain.cpp

@@ -1,5 +1,6 @@
 // Copyright (c) 2009-2010 Satoshi Nakamoto
 // Copyright (c) 2009-2014 The Bitcoin Core developers
+// Copyright (c) 2017-2022 The Zcash developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -24,6 +25,9 @@
 #include "notaries_staked.h"
 #include "komodo_hardfork.h"
 
+#include "main.h"
+#include "txdb.h"
+
 using namespace std;
 
 /**
@@ -82,6 +86,48 @@ const CBlockIndex *CChain::FindFork(const CBlockIndex *pindex) const {
     return pindex;
 }
 
+void CBlockIndex::TrimSolution()
+{
+    AssertLockHeld(cs_main);
+
+    // We can correctly trim a solution as soon as the block index entry has been added
+    // to leveldb. Updates to the block index entry (to update validity status) will be
+    // handled by re-reading the solution from the existing db entry. It does not help to
+    // try to avoid these reads by gating trimming on the validity status: the re-reads are
+    // efficient anyway because of caching in leveldb, and most of them are unavoidable.
+    if (HasSolution()) {
+        std::vector<unsigned char> empty;
+        nSolution.swap(empty);
+    }
+}
+
+CBlockHeader CBlockIndex::GetBlockHeader() const
+{
+    AssertLockHeld(cs_main);
+
+    CBlockHeader header;
+    header.nVersion             = nVersion;
+    if (pprev) {
+        header.hashPrevBlock    = pprev->GetBlockHash();
+    }
+    header.hashMerkleRoot       = hashMerkleRoot;
+    header.hashFinalSaplingRoot = hashFinalSaplingRoot;
+    header.nTime                = nTime;
+    header.nBits                = nBits;
+    header.nNonce               = nNonce;
+    if (HasSolution()) {
+        header.nSolution        = nSolution;
+    } else {
+        CDiskBlockIndex dbindex;
+        if (!pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex)) {
+            LogPrintf("%s: Failed to read index entry", __func__);
+            throw std::runtime_error("Failed to read index entry");
+        }
+        header.nSolution        = dbindex.GetSolution();
+    }
+    return header;
+}
+
 /** Turn the lowest '1' bit in the binary representation of a number into a '0'. */
 int static inline InvertLowestOne(int n) { return n & (n - 1); }
 

src/chain.h

@@ -1,5 +1,6 @@
 // Copyright (c) 2009-2010 Satoshi Nakamoto
 // Copyright (c) 2009-2014 The Bitcoin Core developers
+// Copyright (c) 2016-2022 The Zcash developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -33,6 +34,7 @@
 
 #include <boost/foreach.hpp>
 
+
 extern CCriticalSection cs_main;
 
 static const int SPROUT_VALUE_VERSION = 1001400;
@@ -216,8 +218,14 @@ class CBlockIndex
     unsigned int nTime;
     unsigned int nBits;
     uint256 nNonce;
+
+protected:
+    // The Equihash solution, if it is stored. Once we know that the block index
+    // entry is present in leveldb, this field can be cleared via the TrimSolution
+    // method to save memory.
     std::vector<unsigned char> nSolution;
 
+public:
     //! (memory only) Sequential id assigned to distinguish order in which blocks are received.
     uint32_t nSequenceId;
 
@@ -291,23 +299,15 @@ class CBlockIndex
         return ret;
     }
 
-    CBlockHeader GetBlockHeader() const
-    {
-        CBlockHeader block;
-        block.nVersion       = nVersion;
-        if (pprev)
-            block.hashPrevBlock = pprev->GetBlockHash();
-        block.hashMerkleRoot = hashMerkleRoot;
-        block.hashFinalSaplingRoot   = hashFinalSaplingRoot;
-        block.nTime          = nTime;
-        block.nBits          = nBits;
-        block.nNonce         = nNonce;
-        block.nSolution      = nSolution;
-        return block;
-    }
+    //! Get the block header for this block index. Requires cs_main.
+    CBlockHeader GetBlockHeader() const;
+
+    //! Clear the Equihash solution to save memory. Requires cs_main.
+    void TrimSolution();
 
     uint256 GetBlockHash() const
     {
+        assert(phashBlock);
         return *phashBlock;
     }
 
@@ -341,10 +341,11 @@ class CBlockIndex
 
     std::string ToString() const
     {
-        return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s)",
+        return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s, HasSolution=%s)",
             pprev, nHeight,
             hashMerkleRoot.ToString(),
-            GetBlockHash().ToString());
+            phashBlock ? GetBlockHash().ToString() : "(nil)",
+            HasSolution());
     }
 
     //! Check whether this block index entry is valid up to the passed validity level.
@@ -356,6 +357,12 @@ class CBlockIndex
         return ((nStatus & BLOCK_VALID_MASK) >= nUpTo);
     }
 
+    //! Is the Equihash solution stored?
+    bool HasSolution() const
+    {
+        return !nSolution.empty();
+    }
+
     //! Raise the validity level of this block index entry.
     //! Returns true if the validity was changed.
     bool RaiseValidity(enum BlockStatus nUpTo)
@@ -388,8 +395,11 @@ class CDiskBlockIndex : public CBlockIndex
         hashPrev = uint256();
     }
 
-    explicit CDiskBlockIndex(const CBlockIndex* pindex) : CBlockIndex(*pindex) {
+    explicit CDiskBlockIndex(const CBlockIndex* pindex, std::function<std::vector<unsigned char>()> getSolution) : CBlockIndex(*pindex) {
         hashPrev = (pprev ? pprev->GetBlockHash() : uint256());
+        if (!HasSolution()) {
+            nSolution = getSolution();
+        }
     }
 
     ADD_SERIALIZE_METHODS;
@@ -459,21 +469,38 @@ class CDiskBlockIndex : public CBlockIndex
     bool isStakedAndNotaryPay() const;
     bool isStakedAndAfterDec2019(unsigned int nTime) const;
 
+    //! This method should not be called on a CDiskBlockIndex.
+    void TrimSolution()
+    {
+        assert(!"called CDiskBlockIndex::TrimSolution");
+    }
+
 public:
     uint256 GetBlockHash() const
     {
-        CBlockHeader block;
-        block.nVersion        = nVersion;
-        block.hashPrevBlock   = hashPrev;
-        block.hashMerkleRoot  = hashMerkleRoot;
-        block.hashFinalSaplingRoot    = hashFinalSaplingRoot;
-        block.nTime           = nTime;
-        block.nBits           = nBits;
-        block.nNonce          = nNonce;
-        block.nSolution       = nSolution;
-        return block.GetHash();
+        return GetBlockHeader().GetHash();
+    }
+
+    //! Get the block header for this block index.
+    CBlockHeader GetBlockHeader() const
+    {
+        CBlockHeader header;
+        header.nVersion             = nVersion;
+        header.hashPrevBlock        = hashPrev;
+        header.hashMerkleRoot       = hashMerkleRoot;
+        header.hashFinalSaplingRoot = hashFinalSaplingRoot;
+        header.nTime                = nTime;
+        header.nBits                = nBits;
+        header.nNonce               = nNonce;
+        header.nSolution            = nSolution;
+        return header;
     }
 
+    std::vector<unsigned char> GetSolution() const
+    {
+        assert(HasSolution());
+        return nSolution;
+    }
 
     std::string ToString() const
     {

src/komodo_nSPV_fullnode.h

@@ -124,7 +124,7 @@ int32_t NSPV_setequihdr(struct NSPV_equihdr *hdr,int32_t height)
         hdr->nTime = pindex->nTime;
         hdr->nBits = pindex->nBits;
         hdr->nNonce = pindex->nNonce;
-        memcpy(hdr->nSolution,&pindex->nSolution[0],sizeof(hdr->nSolution));
+        memcpy(hdr->nSolution,&pindex->GetBlockHeader().nSolution[0],sizeof(hdr->nSolution));
         return(sizeof(*hdr));
     }
     return(-1);

src/main.cpp

@@ -1,7 +1,7 @@
 // Copyright (c) 2009-2010 Satoshi Nakamoto
 // Copyright (c) 2009-2014 The Bitcoin Core developers
-// Copyright (c) 2015-2020 The Zcash developers
-// Copyright (c) 2015-2020 The Komodo Platform developers
+// Copyright (c) 2015-2022 The Zcash developers
+// Copyright (c) 2015-2023 The Komodo Platform developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -3864,7 +3864,7 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
                         vFiles.push_back(make_pair(*it, &vinfoBlockFile[*it]));
                     setDirtyFileInfo.erase(it++);
                 }
-                std::vector<const CBlockIndex*> vBlocks;
+                std::vector<CBlockIndex*> vBlocks;
                 vBlocks.reserve(setDirtyBlockIndex.size());
                 for (set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) {
                     vBlocks.push_back(*it);
@@ -3873,6 +3873,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
                 if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) {
                     return AbortNode(state, "Files to write to block index database");
                 }
+                // Now that we have written the block indices to the database, we do not
+                // need to store solutions for these CBlockIndex objects in memory.
+                // cs_main must be held here.
+                for (CBlockIndex *pblockindex : vBlocks) {
+                    pblockindex->TrimSolution();
+                }
             }
             // Finally remove any pruned files
             if (fFlushForPrune)
@@ -6808,7 +6814,11 @@ void static CheckBlockIndex()
                 }
             }
         }
+        // try {
         // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow
+        // } catch (const runtime_error&) {
+        //     assert(!"Failed to read index entry");
+        // }
         // End: actual consistency checks.
 
         // Try descending into the first subnode.

src/rest.cpp

@@ -1,5 +1,6 @@
 // Copyright (c) 2009-2010 Satoshi Nakamoto
 // Copyright (c) 2009-2014 The Bitcoin Core developers
+// Copyright (c) 2017-2022 The Zcash developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -149,6 +150,7 @@ static bool rest_headers(HTTPRequest* req,
 
     std::vector<const CBlockIndex *> headers;
     headers.reserve(count);
+    CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
     {
         LOCK(cs_main);
         BlockMap::const_iterator it = mapBlockIndex.find(hash);
@@ -159,11 +161,16 @@ static bool rest_headers(HTTPRequest* req,
                 break;
             pindex = chainActive.Next(pindex);
         }
-    }
 
-    CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
-    BOOST_FOREACH(const CBlockIndex *pindex, headers) {
-        ssHeader << pindex->GetBlockHeader();
+    if (rf == RF_BINARY || rf == RF_HEX) {
+            try {
+        for (const CBlockIndex *pindex : headers) {
+            ssHeader << pindex->GetBlockHeader();
+                }
+            } catch (const std::runtime_error&) {
+                return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Failed to read index entry");
+            }
+        }
     }
 
     switch (rf) {

src/rpc/blockchain.cpp

@@ -1,5 +1,6 @@
 // Copyright (c) 2010 Satoshi Nakamoto
 // Copyright (c) 2009-2014 The Bitcoin Core developers
+// Copyright (c) 2019-2022 The Zcash developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -160,7 +161,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex)
     result.push_back(Pair("finalsaplingroot", blockindex->hashFinalSaplingRoot.GetHex()));
     result.push_back(Pair("time", (int64_t)blockindex->nTime));
     result.push_back(Pair("nonce", blockindex->nNonce.GetHex()));
-    result.push_back(Pair("solution", HexStr(blockindex->nSolution)));
+    result.pushKV("solution", HexStr(blockindex->GetBlockHeader().nSolution));
     result.push_back(Pair("bits", strprintf("%08x", blockindex->nBits)));
     result.push_back(Pair("difficulty", GetDifficulty(blockindex)));
     result.push_back(Pair("chainwork", blockindex->nChainWork.GetHex()));
@@ -808,15 +809,18 @@ UniValue getblockheader(const UniValue& params, bool fHelp, const CPubKey& mypk)
 
     CBlockIndex* pblockindex = mapBlockIndex[hash];
 
-    if (!fVerbose)
-    {
+    try {
+        if (!fVerbose) {
         CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION);
         ssBlock << pblockindex->GetBlockHeader();
         std::string strHex = HexStr(ssBlock.begin(), ssBlock.end());
         return strHex;
-    }
-
+        } else {
     return blockheaderToJSON(pblockindex);
+        }
+    } catch (const runtime_error&) {
+        throw JSONRPCError(RPC_DATABASE_ERROR, "Failed to read index entry");
+    }
 }
 
 UniValue getblock(const UniValue& params, bool fHelp, const CPubKey& mypk)

src/test-komodo/test_haraka_removal.cpp

@@ -55,7 +55,7 @@ namespace TestHarakaRemoval {
         EXPECT_TRUE(chainActive.Contains(&fakeIndex));
         EXPECT_EQ(0, chainActive.Height());
 
-        CDiskBlockIndex diskindex {&fakeIndex};
+        CDiskBlockIndex diskindex(&fakeIndex, [](){ return std::vector<unsigned char>(); });
         ss.clear();
         ss << diskindex;
         // VARINT(PROTOCOL_VERSION==170010) - 89af1b