Improve Get Balance History (#914)

* Improve Get Balance History Do not fetch keychain twice for the same address * fix clang format * try to fix clang format * Better version of utils::cached * Fix clang tidy * fix compilation * Fix clang format * fix clang format !!! * Update core/src/utils/Cached.h Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix clang format !!! * Add even more cache Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
LedgerHQ · Oct 18, 2022 · e5ea4e0 · e5ea4e0
1 parent efd87c8
commit e5ea4e0
Show file tree

Hide file tree

Showing 8 changed files with 182 additions and 52 deletions.
diff --git a/core/src/utils/Cached.h b/core/src/utils/Cached.h
@@ -0,0 +1,110 @@
+/*
+ *
+ * ledger-core
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2016 Ledger
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+#pragma once
+
+#include <functional>
+#include <tuple>
+#include <unordered_map>
+namespace hash_tuple {
+
+    template <typename TT>
+    struct hash {
+        size_t
+        operator()(TT const &tt) const {
+            return std::hash<TT>()(tt);
+        }
+    };
+
+    namespace {
+        template <class T>
+        inline void hash_combine(std::size_t &seed, T const &v) {
+            seed ^= hash_tuple::hash<T>()(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); // NOLINT(hicpp-signed-bitwise)
+        }
+
+        // Recursive template code derived from Matthieu M.
+        template <class Tuple, size_t Index = std::tuple_size<Tuple>::value - 1>
+        struct HashValueImpl {
+            static void apply(size_t &seed, Tuple const &tuple) {
+                HashValueImpl<Tuple, Index - 1>::apply(seed, tuple);
+                hash_combine(seed, std::get<Index>(tuple));
+            }
+        };
+
+        template <class Tuple>
+        struct HashValueImpl<Tuple, 0> {
+            static void apply(size_t &seed, Tuple const &tuple) {
+                hash_combine(seed, std::get<0>(tuple));
+            }
+        };
+    } // namespace
+
+    template <typename... TT>
+    struct hash<std::tuple<TT...>> {
+        size_t
+        operator()(std::tuple<TT...> const &tt) const {
+            size_t seed = 0;
+            HashValueImpl<std::tuple<TT...>>::apply(seed, tt);
+            return seed;
+        }
+    };
+
+} // namespace hash_tuple
+namespace ledger::core::utils {
+
+    template <typename Function>
+    struct function_traits
+        : public function_traits<decltype(&Function::operator())> {};
+
+    template <typename ClassType, typename ReturnType, typename... Args>
+    struct function_traits<ReturnType (ClassType::*)(Args...) const> {
+        using pointer  = ReturnType (*)(Args...);
+        using function = std::function<ReturnType(Args...)>;
+    };
+
+    template <typename Function>
+    typename function_traits<Function>::function
+    to_function(Function lambda) {
+        return typename function_traits<Function>::function(lambda);
+    }
+
+    template <typename R, typename... Args>
+    using cache_type = std::unordered_map<std::tuple<Args...>, R, hash_tuple::hash<std::tuple<Args...>>>;
+
+    template <typename R, typename... Args>
+    std::function<R(Args...)> cached(cache_type<R, Args...> &cache, std::function<R(Args...)> f) {
+        return [f, &cache](Args... args) {
+            auto key = make_tuple(args...);
+            if (cache.count(key) > 0) {
+                return cache[key];
+            }
+            R result = f(std::forward<Args>(args)...);
+            cache.insert(std::pair<decltype(key), R>(key, result));
+            return result;
+        };
+    }
+} // namespace ledger::core::utils
diff --git a/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp b/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp
@@ -47,6 +47,7 @@
 #include <memory>
 #include <numeric>
 #include <spdlog/logger.h>
+#include <utils/Cached.h>
 #include <utils/DateUtils.hpp>
 #include <wallet/bitcoin/api_impl/BitcoinLikeOutputApi.h>
 #include <wallet/bitcoin/api_impl/BitcoinLikeTransactionApi.h>
@@ -459,10 +460,13 @@ namespace ledger {
                 const auto dustAmount = BitcoinLikeTransactionApi::computeBasicTransactionDustAmount(currency, keychain->getKeychainEngine());
                 soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool());
                 std::vector<BitcoinLikeBlockchainExplorerOutput> utxo;
-                BitcoinLikeUTXODatabaseHelper::queryUTXO(sql, self->getAccountUid(), from, to - from, dustAmount,
-                                                         utxo, [&keychain](const std::string &addr) {
-                                                             return keychain->contains(addr);
-                                                         });
+
+                utils::cache_type<bool, std::string> cache{};
+                std::function<bool(const std::string &)> filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param)
+                                                                                    return keychain->contains(addr);
+                                                                                }));
+
+                BitcoinLikeUTXODatabaseHelper::queryUTXO(sql, self->getAccountUid(), from, to - from, dustAmount, utxo, filter);
                 return functional::map<BitcoinLikeBlockchainExplorerOutput, std::shared_ptr<api::BitcoinLikeOutput>>(utxo, [&currency](const BitcoinLikeBlockchainExplorerOutput &output) -> std::shared_ptr<api::BitcoinLikeOutput> {
                     return std::make_shared<BitcoinLikeOutputApi>(output, currency);
                 });
@@ -492,9 +496,12 @@ namespace ledger {
                 auto keychain = self->getKeychain();
                 soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool());
                 const auto dustAmount = BitcoinLikeTransactionApi::computeBasicTransactionDustAmount(self->getWallet()->getCurrency(), keychain->getKeychainEngine());
-                return static_cast<int32_t>(BitcoinLikeUTXODatabaseHelper::UTXOcount(sql, self->getAccountUid(), dustAmount, [keychain](const std::string &addr) -> bool {
-                    return keychain->contains(addr);
-                }));
+                utils::cache_type<bool, std::string> cache{};
+                std::function<bool(const std::string &)> filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param)
+                                                                                    return keychain->contains(addr);
+                                                                                }));
+
+                return static_cast<int32_t>(BitcoinLikeUTXODatabaseHelper::UTXOcount(sql, self->getAccountUid(), dustAmount, filter));
             });
         }
 
@@ -543,11 +550,12 @@ namespace ledger {
                 soci::session sql(self->getWallet()->getDatabase()->getPool());
                 std::vector<BitcoinLikeBlockchainExplorerOutput> utxos;
                 BigInt sum(0);
-                auto keychain                                   = self->getKeychain();
-                std::function<bool(const std::string &)> filter = [&keychain](const std::string addr) -> bool {
-                    return keychain->contains(addr);
-                };
+                auto keychain         = self->getKeychain();
                 const auto dustAmount = BitcoinLikeTransactionApi::computeBasicTransactionDustAmount(self->getWallet()->getCurrency(), keychain->getKeychainEngine());
+                utils::cache_type<bool, std::string> cache{};
+                std::function<bool(const std::string &)> filter = utils::cached(cache, utils::to_function([&keychain](std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param)
+                                                                                    return keychain->contains(addr);
+                                                                                }));
                 BitcoinLikeUTXODatabaseHelper::queryUTXO(sql, uid, 0, std::numeric_limits<int32_t>::max(), dustAmount, utxos, filter);
                 switch (strategy) {
                 case api::BitcoinLikePickingStrategy::DEEP_OUTPUTS_FIRST:
@@ -587,11 +595,12 @@ namespace ledger {
                 soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool());
                 std::vector<BitcoinLikeBlockchainExplorerOutput> utxos;
                 BigInt sum(0);
-                auto keychain                                   = self->getKeychain();
-                std::function<bool(const std::string &)> filter = [&keychain](const std::string addr) -> bool {
-                    return keychain->contains(addr);
-                };
+                auto keychain         = self->getKeychain();
                 const auto dustAmount = BitcoinLikeTransactionApi::computeBasicTransactionDustAmount(self->getWallet()->getCurrency(), keychain->getKeychainEngine());
+                utils::cache_type<bool, std::string> cache{};
+                std::function<bool(const std::string &)> filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param)
+                                                                                    return keychain->contains(addr);
+                                                                                }));
                 BitcoinLikeUTXODatabaseHelper::queryUTXO(sql, uid, 0, std::numeric_limits<int32_t>::max(), dustAmount, utxos, filter);
                 for (const auto &utxo : utxos) {
                     sum = sum + utxo.value;
@@ -619,10 +628,12 @@ namespace ledger {
                 soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool());
                 std::vector<Operation> operations;
 
-                auto keychain                                   = self->getKeychain();
-                std::function<bool(const std::string &)> filter = [&keychain](const std::string addr) -> bool {
-                    return keychain->contains(addr);
-                };
+                auto keychain = self->getKeychain();
+
+                utils::cache_type<bool, std::string> cache{};
+                std::function<bool(const std::string &)> filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param)
+                                                                                    return keychain->contains(addr);
+                                                                                }));
 
                 // Get operations related to an account
                 OperationDatabaseHelper::queryOperations(sql, uid, operations, filter);

diff --git a/core/src/wallet/bitcoin/transaction_builders/BitcoinLikeUtxoPicker.cpp b/core/src/wallet/bitcoin/transaction_builders/BitcoinLikeUtxoPicker.cpp
@@ -31,6 +31,8 @@
 
 #include "BitcoinLikeUtxoPicker.h"
 
+#include "utils/Cached.h"
+
 #include <api/BitcoinLikeScript.hpp>
 #include <api/BitcoinLikeScriptChunk.hpp>
 #include <async/Promise.hpp>
@@ -252,8 +254,13 @@ namespace ledger {
                                                           const BitcoinLikeGetUtxoFunction &getUtxo) {
             return [=]() -> Future<std::vector<BitcoinLikeUtxo>> {
                 return getUtxo().map<std::vector<BitcoinLikeUtxo>>(getContext(), [=](auto const &utxos) {
-                    auto const isNotExcluded = [&](auto const &currentUtxo) {
-                        return !(currentUtxo.address.isEmpty() || !keychain->contains(currentUtxo.address.getValue()) || request.excludedUtxos.count(BitcoinLikeTransactionUtxoDescriptor{currentUtxo.transactionHash, currentUtxo.index}) > 0);
+                    utils::cache_type<bool, std::string> cache{};
+                    std::function<bool(const std::string &)> keychainContains = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param)
+                                                                                                  return keychain->contains(addr);
+                                                                                              }));
+
+                    auto const isNotExcluded                                  = [&keychainContains, &request](auto const &currentUtxo) {
+                        return !(currentUtxo.address.isEmpty() || !keychainContains(currentUtxo.address.getValue()) || request.excludedUtxos.count(BitcoinLikeTransactionUtxoDescriptor{currentUtxo.transactionHash, currentUtxo.index}) > 0);
                     };
 
                     std::vector<BitcoinLikeUtxo> filteredUtxos;

diff --git a/core/src/wallet/cosmos/CosmosLikeAccount.cpp b/core/src/wallet/cosmos/CosmosLikeAccount.cpp
@@ -43,6 +43,7 @@
 #include <math/Base58.hpp>
 #include <numeric>
 #include <soci.h>
+#include <utils/Cached.h>
 #include <utils/DateUtils.hpp>
 #include <utils/Option.hpp>
 #include <wallet/common/Block.h>
@@ -471,10 +472,10 @@ namespace ledger {
                     std::vector<Operation> operations;
 
                     auto keychain = self->getKeychain();
-                    std::function<bool(const std::string &)> filter =
-                        [&keychain](const std::string addr) -> bool {
-                        return keychain->contains(addr);
-                    };
+                    utils::cache_type<bool, std::string> cache{};
+                    std::function<bool(const std::string &)> filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param)
+                                                                                        return keychain->contains(addr);
+                                                                                    }));
 
                     // Get operations related to an account
                     CosmosLikeOperationDatabaseHelper::queryOperations(sql, uid, operations, filter);

diff --git a/core/src/wallet/ethereum/EthereumLikeAccount.cpp b/core/src/wallet/ethereum/EthereumLikeAccount.cpp
@@ -360,8 +360,8 @@ namespace ledger {
                 std::vector<Operation> operations;
 
                 auto keychain                                   = self->getKeychain();
-                std::function<bool(const std::string &)> filter = [&keychain](const std::string &addr) -> bool {
-                    auto keychainAddr = keychain->getAddress()->toString();
+                auto keychainAddr                               = keychain->getAddress()->toString();
+                std::function<bool(const std::string &)> filter = [&keychain, &keychainAddr](const std::string &addr) -> bool {
                     return addr == keychainAddr;
                 };
 
@@ -542,9 +542,8 @@ namespace ledger {
                 // Get rid of leading zeros
                 auto skipEIP55Check = true;
                 // auto toAddress = BigInt::fromHex(hex::toString(reader.read(32))).toHexString();
-                erc20Tx.to          = EthereumLikeAddress::fromEIP55(
-                                          "0x" + BigInt::fromHex(hex::toString(reader.read(32))).toHexString(),
-                                          account->getWallet()->getCurrency(), Option<std::string>(""), skipEIP55Check)
+                erc20Tx.to          = EthereumLikeAddress::fromEIP55("0x" + BigInt::fromHex(hex::toString(reader.read(32))).toHexString(),
+                                                                     account->getWallet()->getCurrency(), Option<std::string>(""), skipEIP55Check)
                                  ->toEIP55();
                 erc20Tx.value           = BigInt::fromHex(hex::toString(reader.read(32)));
                 erc20Tx.type            = api::OperationType::SEND;

diff --git a/core/src/wallet/ripple/RippleLikeAccount.cpp b/core/src/wallet/ripple/RippleLikeAccount.cpp
@@ -40,6 +40,7 @@
 #include <database/soci-option.h>
 #include <events/Event.hpp>
 #include <math/Base58.hpp>
+#include <utils/Cached.h>
 #include <utils/DateUtils.hpp>
 #include <utils/Option.hpp>
 #include <wallet/common/database/BlockDatabaseHelper.h>
@@ -223,10 +224,11 @@ namespace ledger {
                 soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool());
                 std::vector<Operation> operations;
 
-                auto keychain                                   = self->getKeychain();
-                std::function<bool(const std::string &)> filter = [&keychain](const std::string addr) -> bool {
-                    return keychain->contains(addr);
-                };
+                auto keychain = self->getKeychain();
+                utils::cache_type<bool, std::string> cache{};
+                std::function<bool(const std::string &)> filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param)
+                                                                                    return keychain->contains(addr);
+                                                                                }));
 
                 // Get operations related to an account
                 OperationDatabaseHelper::queryOperations(sql, uid, operations, filter);
@@ -305,10 +307,8 @@ namespace ledger {
             auto eventPublisher  = std::make_shared<EventPublisher>(getContext());
 
             _currentSyncEventBus = eventPublisher->getEventBus();
-            auto future          = _synchronizer->synchronizeAccount(
-                                                    std::static_pointer_cast<RippleLikeAccount>(shared_from_this()))
-                              ->getFuture();
-            auto self = std::static_pointer_cast<RippleLikeAccount>(shared_from_this());
+            auto future          = _synchronizer->synchronizeAccount(std::static_pointer_cast<RippleLikeAccount>(shared_from_this()))->getFuture();
+            auto self            = std::static_pointer_cast<RippleLikeAccount>(shared_from_this());
 
             // Update current block height (needed to compute trust level)
             _explorer->getCurrentBlock().onComplete(getContext(),

diff --git a/core/src/wallet/stellar/StellarLikeAccount.cpp b/core/src/wallet/stellar/StellarLikeAccount.cpp
@@ -46,6 +46,7 @@
 #include <events/Event.hpp>
 #include <events/LambdaEventReceiver.hpp>
 #include <set>
+#include <utils/Cached.h>
 #include <utils/DateUtils.hpp>
 #include <wallet/common/BalanceHistory.hpp>
 #include <wallet/common/database/AccountDatabaseHelper.h>
@@ -176,10 +177,11 @@ namespace ledger {
                 soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool());
                 std::vector<Operation> operations;
 
-                auto keychain                                   = self->getKeychain();
-                std::function<bool(const std::string &)> filter = [&keychain](const std::string addr) -> bool {
-                    return keychain->contains(addr);
-                };
+                auto keychain = self->getKeychain();
+                utils::cache_type<bool, std::string> cache{};
+                std::function<bool(const std::string &)> filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param)
+                                                                                    return keychain->contains(addr);
+                                                                                }));
 
                 // Get operations related to an account
                 OperationDatabaseHelper::queryOperations(sql, uid, operations, filter);