From e5ea4e06aba8f38b9f03ee901f6bb3d3c7bc86bb Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Tue, 18 Oct 2022 11:11:41 +0200 Subject: [PATCH] Improve Get Balance History (#914) * Improve Get Balance History Do not fetch keychain twice for the same address * fix clang format * try to fix clang format * Better version of utils::cached * Fix clang tidy * fix compilation * Fix clang format * fix clang format !!! * Update core/src/utils/Cached.h Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix clang format !!! * Add even more cache Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- core/src/utils/Cached.h | 110 ++++++++++++++++++ .../src/wallet/bitcoin/BitcoinLikeAccount.cpp | 49 +++++--- .../BitcoinLikeUtxoPicker.cpp | 11 +- core/src/wallet/cosmos/CosmosLikeAccount.cpp | 9 +- .../wallet/ethereum/EthereumLikeAccount.cpp | 9 +- core/src/wallet/ripple/RippleLikeAccount.cpp | 16 +-- .../src/wallet/stellar/StellarLikeAccount.cpp | 10 +- core/src/wallet/tezos/TezosLikeAccount.cpp | 20 ++-- 8 files changed, 182 insertions(+), 52 deletions(-) create mode 100644 core/src/utils/Cached.h diff --git a/core/src/utils/Cached.h b/core/src/utils/Cached.h new file mode 100644 index 0000000000..3770b0a9f2 --- /dev/null +++ b/core/src/utils/Cached.h @@ -0,0 +1,110 @@ +/* + * + * ledger-core + * + * The MIT License (MIT) + * + * Copyright (c) 2016 Ledger + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ +#pragma once + +#include +#include +#include +namespace hash_tuple { + + template + struct hash { + size_t + operator()(TT const &tt) const { + return std::hash()(tt); + } + }; + + namespace { + template + inline void hash_combine(std::size_t &seed, T const &v) { + seed ^= hash_tuple::hash()(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); // NOLINT(hicpp-signed-bitwise) + } + + // Recursive template code derived from Matthieu M. + template ::value - 1> + struct HashValueImpl { + static void apply(size_t &seed, Tuple const &tuple) { + HashValueImpl::apply(seed, tuple); + hash_combine(seed, std::get(tuple)); + } + }; + + template + struct HashValueImpl { + static void apply(size_t &seed, Tuple const &tuple) { + hash_combine(seed, std::get<0>(tuple)); + } + }; + } // namespace + + template + struct hash> { + size_t + operator()(std::tuple const &tt) const { + size_t seed = 0; + HashValueImpl>::apply(seed, tt); + return seed; + } + }; + +} // namespace hash_tuple +namespace ledger::core::utils { + + template + struct function_traits + : public function_traits {}; + + template + struct function_traits { + using pointer = ReturnType (*)(Args...); + using function = std::function; + }; + + template + typename function_traits::function + to_function(Function lambda) { + return typename function_traits::function(lambda); + } + + template + using cache_type = std::unordered_map, R, hash_tuple::hash>>; + + template + std::function cached(cache_type &cache, std::function f) { + return [f, &cache](Args... args) { + auto key = make_tuple(args...); + if (cache.count(key) > 0) { + return cache[key]; + } + R result = f(std::forward(args)...); + cache.insert(std::pair(key, result)); + return result; + }; + } +} // namespace ledger::core::utils diff --git a/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp b/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp index db55a9ebd6..9835f96181 100644 --- a/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp +++ b/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -459,10 +460,13 @@ namespace ledger { const auto dustAmount = BitcoinLikeTransactionApi::computeBasicTransactionDustAmount(currency, keychain->getKeychainEngine()); soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool()); std::vector utxo; - BitcoinLikeUTXODatabaseHelper::queryUTXO(sql, self->getAccountUid(), from, to - from, dustAmount, - utxo, [&keychain](const std::string &addr) { - return keychain->contains(addr); - }); + + utils::cache_type cache{}; + std::function filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); + + BitcoinLikeUTXODatabaseHelper::queryUTXO(sql, self->getAccountUid(), from, to - from, dustAmount, utxo, filter); return functional::map>(utxo, [¤cy](const BitcoinLikeBlockchainExplorerOutput &output) -> std::shared_ptr { return std::make_shared(output, currency); }); @@ -492,9 +496,12 @@ namespace ledger { auto keychain = self->getKeychain(); soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool()); const auto dustAmount = BitcoinLikeTransactionApi::computeBasicTransactionDustAmount(self->getWallet()->getCurrency(), keychain->getKeychainEngine()); - return static_cast(BitcoinLikeUTXODatabaseHelper::UTXOcount(sql, self->getAccountUid(), dustAmount, [keychain](const std::string &addr) -> bool { - return keychain->contains(addr); - })); + utils::cache_type cache{}; + std::function filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); + + return static_cast(BitcoinLikeUTXODatabaseHelper::UTXOcount(sql, self->getAccountUid(), dustAmount, filter)); }); } @@ -543,11 +550,12 @@ namespace ledger { soci::session sql(self->getWallet()->getDatabase()->getPool()); std::vector utxos; BigInt sum(0); - auto keychain = self->getKeychain(); - std::function filter = [&keychain](const std::string addr) -> bool { - return keychain->contains(addr); - }; + auto keychain = self->getKeychain(); const auto dustAmount = BitcoinLikeTransactionApi::computeBasicTransactionDustAmount(self->getWallet()->getCurrency(), keychain->getKeychainEngine()); + utils::cache_type cache{}; + std::function filter = utils::cached(cache, utils::to_function([&keychain](std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); BitcoinLikeUTXODatabaseHelper::queryUTXO(sql, uid, 0, std::numeric_limits::max(), dustAmount, utxos, filter); switch (strategy) { case api::BitcoinLikePickingStrategy::DEEP_OUTPUTS_FIRST: @@ -587,11 +595,12 @@ namespace ledger { soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool()); std::vector utxos; BigInt sum(0); - auto keychain = self->getKeychain(); - std::function filter = [&keychain](const std::string addr) -> bool { - return keychain->contains(addr); - }; + auto keychain = self->getKeychain(); const auto dustAmount = BitcoinLikeTransactionApi::computeBasicTransactionDustAmount(self->getWallet()->getCurrency(), keychain->getKeychainEngine()); + utils::cache_type cache{}; + std::function filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); BitcoinLikeUTXODatabaseHelper::queryUTXO(sql, uid, 0, std::numeric_limits::max(), dustAmount, utxos, filter); for (const auto &utxo : utxos) { sum = sum + utxo.value; @@ -619,10 +628,12 @@ namespace ledger { soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool()); std::vector operations; - auto keychain = self->getKeychain(); - std::function filter = [&keychain](const std::string addr) -> bool { - return keychain->contains(addr); - }; + auto keychain = self->getKeychain(); + + utils::cache_type cache{}; + std::function filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); // Get operations related to an account OperationDatabaseHelper::queryOperations(sql, uid, operations, filter); diff --git a/core/src/wallet/bitcoin/transaction_builders/BitcoinLikeUtxoPicker.cpp b/core/src/wallet/bitcoin/transaction_builders/BitcoinLikeUtxoPicker.cpp index 71e221769f..b9fe592f36 100644 --- a/core/src/wallet/bitcoin/transaction_builders/BitcoinLikeUtxoPicker.cpp +++ b/core/src/wallet/bitcoin/transaction_builders/BitcoinLikeUtxoPicker.cpp @@ -31,6 +31,8 @@ #include "BitcoinLikeUtxoPicker.h" +#include "utils/Cached.h" + #include #include #include @@ -252,8 +254,13 @@ namespace ledger { const BitcoinLikeGetUtxoFunction &getUtxo) { return [=]() -> Future> { return getUtxo().map>(getContext(), [=](auto const &utxos) { - auto const isNotExcluded = [&](auto const ¤tUtxo) { - return !(currentUtxo.address.isEmpty() || !keychain->contains(currentUtxo.address.getValue()) || request.excludedUtxos.count(BitcoinLikeTransactionUtxoDescriptor{currentUtxo.transactionHash, currentUtxo.index}) > 0); + utils::cache_type cache{}; + std::function keychainContains = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); + + auto const isNotExcluded = [&keychainContains, &request](auto const ¤tUtxo) { + return !(currentUtxo.address.isEmpty() || !keychainContains(currentUtxo.address.getValue()) || request.excludedUtxos.count(BitcoinLikeTransactionUtxoDescriptor{currentUtxo.transactionHash, currentUtxo.index}) > 0); }; std::vector filteredUtxos; diff --git a/core/src/wallet/cosmos/CosmosLikeAccount.cpp b/core/src/wallet/cosmos/CosmosLikeAccount.cpp index 83b55fbcb2..a82738279c 100644 --- a/core/src/wallet/cosmos/CosmosLikeAccount.cpp +++ b/core/src/wallet/cosmos/CosmosLikeAccount.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -471,10 +472,10 @@ namespace ledger { std::vector operations; auto keychain = self->getKeychain(); - std::function filter = - [&keychain](const std::string addr) -> bool { - return keychain->contains(addr); - }; + utils::cache_type cache{}; + std::function filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); // Get operations related to an account CosmosLikeOperationDatabaseHelper::queryOperations(sql, uid, operations, filter); diff --git a/core/src/wallet/ethereum/EthereumLikeAccount.cpp b/core/src/wallet/ethereum/EthereumLikeAccount.cpp index 25caf3f400..3f4bab9f26 100644 --- a/core/src/wallet/ethereum/EthereumLikeAccount.cpp +++ b/core/src/wallet/ethereum/EthereumLikeAccount.cpp @@ -360,8 +360,8 @@ namespace ledger { std::vector operations; auto keychain = self->getKeychain(); - std::function filter = [&keychain](const std::string &addr) -> bool { - auto keychainAddr = keychain->getAddress()->toString(); + auto keychainAddr = keychain->getAddress()->toString(); + std::function filter = [&keychain, &keychainAddr](const std::string &addr) -> bool { return addr == keychainAddr; }; @@ -542,9 +542,8 @@ namespace ledger { // Get rid of leading zeros auto skipEIP55Check = true; // auto toAddress = BigInt::fromHex(hex::toString(reader.read(32))).toHexString(); - erc20Tx.to = EthereumLikeAddress::fromEIP55( - "0x" + BigInt::fromHex(hex::toString(reader.read(32))).toHexString(), - account->getWallet()->getCurrency(), Option(""), skipEIP55Check) + erc20Tx.to = EthereumLikeAddress::fromEIP55("0x" + BigInt::fromHex(hex::toString(reader.read(32))).toHexString(), + account->getWallet()->getCurrency(), Option(""), skipEIP55Check) ->toEIP55(); erc20Tx.value = BigInt::fromHex(hex::toString(reader.read(32))); erc20Tx.type = api::OperationType::SEND; diff --git a/core/src/wallet/ripple/RippleLikeAccount.cpp b/core/src/wallet/ripple/RippleLikeAccount.cpp index 4dbc741612..2aaf58d8ae 100644 --- a/core/src/wallet/ripple/RippleLikeAccount.cpp +++ b/core/src/wallet/ripple/RippleLikeAccount.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -223,10 +224,11 @@ namespace ledger { soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool()); std::vector operations; - auto keychain = self->getKeychain(); - std::function filter = [&keychain](const std::string addr) -> bool { - return keychain->contains(addr); - }; + auto keychain = self->getKeychain(); + utils::cache_type cache{}; + std::function filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); // Get operations related to an account OperationDatabaseHelper::queryOperations(sql, uid, operations, filter); @@ -305,10 +307,8 @@ namespace ledger { auto eventPublisher = std::make_shared(getContext()); _currentSyncEventBus = eventPublisher->getEventBus(); - auto future = _synchronizer->synchronizeAccount( - std::static_pointer_cast(shared_from_this())) - ->getFuture(); - auto self = std::static_pointer_cast(shared_from_this()); + auto future = _synchronizer->synchronizeAccount(std::static_pointer_cast(shared_from_this()))->getFuture(); + auto self = std::static_pointer_cast(shared_from_this()); // Update current block height (needed to compute trust level) _explorer->getCurrentBlock().onComplete(getContext(), diff --git a/core/src/wallet/stellar/StellarLikeAccount.cpp b/core/src/wallet/stellar/StellarLikeAccount.cpp index a3210c7b1a..0d06c56566 100644 --- a/core/src/wallet/stellar/StellarLikeAccount.cpp +++ b/core/src/wallet/stellar/StellarLikeAccount.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -176,10 +177,11 @@ namespace ledger { soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool()); std::vector operations; - auto keychain = self->getKeychain(); - std::function filter = [&keychain](const std::string addr) -> bool { - return keychain->contains(addr); - }; + auto keychain = self->getKeychain(); + utils::cache_type cache{}; + std::function filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); // Get operations related to an account OperationDatabaseHelper::queryOperations(sql, uid, operations, filter); diff --git a/core/src/wallet/tezos/TezosLikeAccount.cpp b/core/src/wallet/tezos/TezosLikeAccount.cpp index f8fdd8d9d8..6cba56ed67 100644 --- a/core/src/wallet/tezos/TezosLikeAccount.cpp +++ b/core/src/wallet/tezos/TezosLikeAccount.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -166,12 +167,10 @@ namespace ledger { auto originatedAccountUid = TezosLikeAccountDatabaseHelper::createOriginatedAccountUid(getAccountUid(), origAccount.address); - const auto found = std::find_if( - _originatedAccounts.begin(), - _originatedAccounts.end(), - [&originatedAccountUid](const std::shared_ptr &element) { - return std::dynamic_pointer_cast(element)->getAccountUid() == originatedAccountUid; - }); + const auto found = std::find_if(_originatedAccounts.begin(), _originatedAccounts.end(), + [&originatedAccountUid](const std::shared_ptr &element) { + return std::dynamic_pointer_cast(element)->getAccountUid() == originatedAccountUid; + }); if (found == _originatedAccounts.end()) { _originatedAccounts.emplace_back( @@ -285,10 +284,11 @@ namespace ledger { soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool()); std::vector operations; - auto keychain = self->getKeychain(); - std::function filter = [&keychain](const std::string addr) -> bool { - return keychain->contains(addr); - }; + auto keychain = self->getKeychain(); + utils::cache_type cache{}; + std::function filter = utils::cached(cache, utils::to_function([&keychain](const std::string addr) -> bool { // NOLINT(performance-unnecessary-value-param) + return keychain->contains(addr); + })); // Get operations related to an account TezosLikeAccountDatabaseHelper::queryOperations(sql, uid, operations, filter);