fix: reconcile tests and benchmarks for proofs

This commit represents the updating of the rest of the testing and
benchmarking code to function as-expected with the new transaction
format and protocol. From this point forward, you should expect CI to
pass.

This is an omnibus update and includes help from multiple authors, I
have attempted to capture contributors below as co-authors.

Co-authored-by: Alexander Jung <[email protected]>
Co-authored-by: Gert-Jaap Glasbergen <[email protected]>
Co-authored-by: James Lovejoy <[email protected]>
Co-authored-by: Michael Maurer <[email protected]>
Signed-off-by: Sam Stuewe <[email protected]>

benchmarks/CMakeLists.txt

@@ -3,7 +3,8 @@ project(benchmarks)
 include_directories(. ../src ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
 set(SECP256K1_LIBRARY $<TARGET_FILE:secp256k1>)
 
-add_executable(run_benchmarks   low_level.cpp
+add_executable(run_benchmarks   audits.cpp
+                                low_level.cpp
                                 transactions.cpp
                                 uhs_leveldb.cpp
                                 uhs_set.cpp

benchmarks/audits.cpp

@@ -0,0 +1,210 @@
+// Copyright (c) 2021 MIT Digital Currency Initiative,
+//                    Federal Reserve Bank of Boston
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "uhs/transaction/transaction.hpp"
+#include "uhs/twophase/locking_shard/locking_shard.hpp"
+#include "util/common/hash.hpp"
+#include "util/common/hashmap.hpp"
+#include "util/common/keys.hpp"
+#include "util/common/config.hpp"
+#include "util/common/random_source.hpp"
+#include "util/common/snapshot_map.hpp"
+
+#include <secp256k1_bppp.h>
+#include <benchmark/benchmark.h>
+#include <gtest/gtest.h>
+#include <unordered_map>
+#include <random>
+#include <unordered_set>
+
+#define SWEEP_MAX 100000
+#define EPOCH     1000
+
+using namespace cbdc;
+using secp256k1_context_destroy_type = void (*)(secp256k1_context*);
+using uhs_element = locking_shard::locking_shard::uhs_element;
+
+struct GensDeleter {
+    explicit GensDeleter(secp256k1_context* ctx) : m_ctx(ctx) {}
+
+    void operator()(secp256k1_bppp_generators* gens) const {
+        secp256k1_bppp_generators_destroy(m_ctx, gens);
+    }
+
+    secp256k1_context* m_ctx;
+};
+
+static std::default_random_engine m_shuffle;
+
+static const inline auto rnd
+    = std::make_unique<random_source>(config::random_source);
+
+static std::unique_ptr<secp256k1_context, secp256k1_context_destroy_type>
+    secp{secp256k1_context_create(SECP256K1_CONTEXT_NONE),
+         &secp256k1_context_destroy};
+
+/// should be set to exactly `floor(log_base(value)) + 1`
+///
+/// We use n_bits = 64, base = 16, so this should always be 24.
+static const inline auto generator_count = 16 + 8;
+
+static std::unique_ptr<secp256k1_bppp_generators, GensDeleter>
+    generators{
+        secp256k1_bppp_generators_create(secp.get(),
+                                         generator_count),
+        GensDeleter(secp.get())};
+
+static auto gen_map(uint64_t map_size, bool deleted = false) -> snapshot_map<hash_t, uhs_element> {
+    std::uniform_int_distribution<uint64_t> dist(EPOCH - 100, EPOCH + 100);
+    auto uhs = snapshot_map<hash_t, uhs_element>();
+
+    auto comm = commit(secp.get(), 10, hash_t{}).value();
+    auto rng
+        = transaction::prove(secp.get(),
+                             generators.get(),
+                             *rnd,
+                             {hash_t{}, 10},
+                             &comm);
+    auto commitment = serialize_commitment(secp.get(), comm);
+
+    for(uint64_t i = 1; i <= map_size; i++) {
+        transaction::compact_output out{commitment, rng, rnd->random_hash()};
+        auto del = deleted ? std::optional<uint64_t>{dist(m_shuffle)} : std::nullopt;
+        uhs_element el0{out, 0, del};
+        auto key = transaction::calculate_uhs_id(out);
+        uhs.emplace(key, el0);
+    }
+    return uhs;
+}
+
+static auto audit(snapshot_map<hash_t, uhs_element>& uhs,
+                  snapshot_map<hash_t, uhs_element>& locked,
+                  snapshot_map<hash_t, uhs_element>& spent)
+    -> std::optional<commitment_t> {
+
+    {
+        uhs.snapshot();
+        locked.snapshot();
+        spent.snapshot();
+    }
+
+    bool failed = false;
+    uint64_t epoch = EPOCH;
+
+    static constexpr auto scratch_size = 8192UL * 1024UL;
+    [[maybe_unused]] secp256k1_scratch_space* scratch
+        = secp256k1_scratch_space_create(secp.get(), scratch_size);
+
+    static constexpr size_t threshold = 100000;
+    size_t cursor = 0;
+    std::vector<commitment_t> comms{};
+    auto* range_batch = secp256k1_bppp_rangeproof_batch_create(secp.get(), 34 * (threshold + 1));
+    auto summarize
+        = [&](const snapshot_map<hash_t, uhs_element>& m) {
+        for(const auto& [id, elem] : m) {
+            if(failed) {
+                break;
+            }
+            if(elem.m_creation_epoch <= epoch
+               && (!elem.m_deletion_epoch.has_value()
+                   || (elem.m_deletion_epoch.value() > epoch))) {
+
+                auto uhs_id
+                    = transaction::calculate_uhs_id(elem.m_out);
+                if(uhs_id != id) {
+                    failed = true;
+                }
+                auto comm = elem.m_out.m_value_commitment;
+                auto c = deserialize_commitment(secp.get(), comm).value();
+                auto r = transaction::validation::range_batch_add(
+                    *range_batch,
+                    scratch,
+                    elem.m_out.m_range,
+                    c
+                );
+                if(!r.has_value()) {
+                    ++cursor;
+                }
+                comms.push_back(comm);
+            }
+            if(cursor >= threshold) {
+                failed = transaction::validation::check_range_batch(*range_batch).has_value();
+                [[maybe_unused]] auto res = secp256k1_bppp_rangeproof_batch_clear(secp.get(), range_batch);
+                cursor = 0;
+            }
+        }
+        if(cursor > 0) {
+            failed = transaction::validation::check_range_batch(*range_batch).has_value();
+            [[maybe_unused]] auto res = secp256k1_bppp_rangeproof_batch_clear(secp.get(), range_batch);
+            cursor = 0;
+        }
+    };
+
+    summarize(uhs);
+    summarize(locked);
+    summarize(spent);
+    [[maybe_unused]] auto res = secp256k1_bppp_rangeproof_batch_destroy(secp.get(), range_batch);
+    free(range_batch);
+
+    secp256k1_scratch_space_destroy(secp.get(), scratch);
+
+//    std::vector<commitment_t> comms{};
+//    comms.reserve(pool.size());
+//
+//    for(auto& f : pool) {
+//        auto c = f.get();
+//        failed = !c.has_value();
+//        if(failed) {
+//            break;
+//        }
+//        comms.emplace_back(std::move(c.value()));
+//k   }
+
+    {
+        uhs.release_snapshot();
+        locked.release_snapshot();
+        spent.release_snapshot();
+    }
+
+    if(failed) {
+        return std::nullopt;
+    }
+
+    return sum_commitments(secp.get(), comms);
+}
+
+static void audit_routine(benchmark::State& state) {
+    auto key_count = state.range(0);
+
+    auto seed = std::chrono::high_resolution_clock::now()
+                    .time_since_epoch()
+                    .count();
+    seed %= std::numeric_limits<uint32_t>::max();
+    m_shuffle.seed(static_cast<uint32_t>(seed));
+
+    uint32_t locked_sz{};
+    uint32_t spent_sz{};
+    {
+        std::uniform_int_distribution<uint32_t> locked(0, key_count);
+        locked_sz = locked(m_shuffle);
+        std::uniform_int_distribution<uint32_t> spent(0, key_count - locked_sz);
+        spent_sz = spent(m_shuffle);
+    }
+
+    snapshot_map<hash_t, uhs_element> uhs = gen_map(key_count - (locked_sz + spent_sz));
+    snapshot_map<hash_t, uhs_element> locked = gen_map(locked_sz);
+    snapshot_map<hash_t, uhs_element> spent = gen_map(spent_sz, true);
+    for(auto _ : state) {
+        auto res = audit(uhs, locked, spent);
+        ASSERT_NE(res, std::nullopt);
+    }
+}
+
+BENCHMARK(audit_routine)
+    ->RangeMultiplier(10)
+    ->Range(10, SWEEP_MAX)
+    ->Complexity(benchmark::oAuto);
+
+BENCHMARK_MAIN();

benchmarks/low_level.cpp

@@ -166,15 +166,7 @@ BENCHMARK_F(low_level, no_inputs)(benchmark::State& state) {
 BENCHMARK_F(low_level, calculate_uhs_id)(benchmark::State& state) {
     m_valid_tx = wallet1.send_to(2, wallet2.generate_key(), true).value();
     auto cp_tx = cbdc::transaction::compact_tx(m_valid_tx);
-    auto engine = std::default_random_engine();
     for(auto _ : state) {
-        state.PauseTiming();
-        uint64_t i = (uint64_t)engine();
-        state.ResumeTiming();
-        cbdc::transaction::uhs_id_from_output(cp_tx.m_id,
-                                              i,
-                                              m_valid_tx.m_outputs[0]);
+        cbdc::transaction::calculate_uhs_id(cp_tx.m_outputs[0]);
     }
 }
-
-BENCHMARK_MAIN();

benchmarks/uhs_leveldb.cpp

@@ -100,12 +100,12 @@ static void uhs_leveldb_put_new(benchmark::State& state) {
         db.wallet2.confirm_transaction(db.m_valid_tx);
 
         db.m_cp_tx = cbdc::transaction::compact_tx(db.m_valid_tx);
-        std::array<char, sizeof(db.m_cp_tx.m_uhs_outputs)> out_arr{};
+        std::array<char, sizeof(db.m_cp_tx.m_outputs)> out_arr{};
         std::memcpy(out_arr.data(),
-                    db.m_cp_tx.m_uhs_outputs.data(),
-                    db.m_cp_tx.m_uhs_outputs.size());
+                    db.m_cp_tx.m_outputs.data(),
+                    db.m_cp_tx.m_outputs.size());
         leveldb::Slice OutPointKey(out_arr.data(),
-                                   db.m_cp_tx.m_uhs_outputs.size());
+                                   db.m_cp_tx.m_outputs.size());
 
         // actual storage
         state.ResumeTiming();
@@ -121,12 +121,12 @@ static void uhs_leveldb_item_delete(benchmark::State& state) {
     auto db = db_container();
 
     db.m_cp_tx = cbdc::transaction::compact_tx(db.m_valid_tx);
-    std::array<char, sizeof(db.m_cp_tx.m_uhs_outputs)> out_arr{};
+    std::array<char, sizeof(db.m_cp_tx.m_outputs)> out_arr{};
     std::memcpy(out_arr.data(),
-                db.m_cp_tx.m_uhs_outputs.data(),
-                db.m_cp_tx.m_uhs_outputs.size());
+                db.m_cp_tx.m_outputs.data(),
+                db.m_cp_tx.m_outputs.size());
     leveldb::Slice OutPointKey(out_arr.data(),
-                               db.m_cp_tx.m_uhs_outputs.size());
+                               db.m_cp_tx.m_outputs.size());
 
     for(auto _ : state) {
         state.PauseTiming();
@@ -148,10 +148,11 @@ static void uhs_leveldb_shard_sim(benchmark::State& state) {
         leveldb::WriteBatch batch;
         state.ResumeTiming();
         for(const auto& tx : db.block) {
-            for(const auto& out : tx.m_uhs_outputs) {
-                std::array<char, sizeof(out)> out_arr{};
-                std::memcpy(out_arr.data(), out.data(), out.size());
-                leveldb::Slice OutPointKey(out_arr.data(), out.size());
+            for(const auto& out : tx.m_outputs) {
+                auto id = calculate_uhs_id(out);
+                std::array<char, sizeof(id)> out_arr{};
+                std::memcpy(out_arr.data(), id.data(), id.size());
+                leveldb::Slice OutPointKey(out_arr.data(), id.size());
                 batch.Put(OutPointKey, leveldb::Slice());
             }
             for(const auto& inp : tx.m_inputs) {
@@ -176,10 +177,11 @@ static void uhs_leveldb_shard_sim_brief(benchmark::State& state) {
         leveldb::WriteBatch batch;
         state.ResumeTiming();
         for(const auto& tx : db.block_abridged) {
-            for(const auto& out : tx.m_uhs_outputs) {
-                std::array<char, sizeof(out)> out_arr{};
-                std::memcpy(out_arr.data(), out.data(), out.size());
-                leveldb::Slice OutPointKey(out_arr.data(), out.size());
+            for(const auto& out : tx.m_outputs) {
+                auto id = calculate_uhs_id(out);
+                std::array<char, sizeof(id)> out_arr{};
+                std::memcpy(out_arr.data(), id.data(), id.size());
+                leveldb::Slice OutPointKey(out_arr.data(), id.size());
                 batch.Put(OutPointKey, leveldb::Slice());
             }
             for(const auto& inp : tx.m_inputs) {

benchmarks/uhs_set.cpp

@@ -6,59 +6,69 @@
 #include "uhs/transaction/transaction.hpp"
 #include "uhs/transaction/validation.hpp"
 #include "uhs/transaction/wallet.hpp"
+#include "uhs/twophase/locking_shard/locking_shard.hpp"
 #include "util/common/hash.hpp"
 #include "util/common/hashmap.hpp"
+#include "util/common/snapshot_map.hpp"
 
 #include <benchmark/benchmark.h>
 #include <gtest/gtest.h>
 #include <unordered_set>
 #include <variant>
 
 class uhs_set : public ::benchmark::Fixture {
+  public:
+    uhs_set() {
+        Iterations(10000);
+    }
+
   protected:
     void SetUp(const ::benchmark::State&) override {
+        m_uhs.snapshot();
+
         auto mint_tx1 = wallet1.mint_new_coins(1, 100);
         wallet1.confirm_transaction(mint_tx1);
+        auto cp_1 = cbdc::transaction::compact_tx(mint_tx1);
+        auto id_1 = cbdc::transaction::calculate_uhs_id(cp_1.m_outputs[0]);
+        uhs_element el_1 {cp_1.m_outputs[0], epoch++, std::nullopt};
+        m_uhs.emplace(id_1, el_1);
+
         auto mint_tx2 = wallet2.mint_new_coins(1, 100);
         wallet2.confirm_transaction(mint_tx2);
-        m_cp_tx = cbdc::transaction::compact_tx(m_valid_tx);
+        auto cp_2 = cbdc::transaction::compact_tx(mint_tx2);
+        auto id_2 = cbdc::transaction::calculate_uhs_id(cp_2.m_outputs[0]);
+        uhs_element el_2 {cp_2.m_outputs[0], epoch++, std::nullopt};
+        m_uhs.emplace(id_2, el_2);
     }
 
     cbdc::transaction::wallet wallet1;
     cbdc::transaction::wallet wallet2;
 
     cbdc::transaction::full_tx m_valid_tx{};
-    cbdc::transaction::compact_tx m_cp_tx;
+    cbdc::transaction::compact_tx m_cp_tx{};
 
-    std::unordered_set<cbdc::hash_t, cbdc::hashing::null> set;
+    using uhs_element = cbdc::locking_shard::locking_shard::uhs_element;
+    cbdc::snapshot_map<cbdc::hash_t, uhs_element> m_uhs{};
+    size_t epoch{0};
 };
 
-// benchmark how long it takes to emplace new values into an unordered set
-BENCHMARK_F(uhs_set, emplace_new)(benchmark::State& state) {
+// minimal tx execution
+BENCHMARK_F(uhs_set, swap)(benchmark::State& state) {
     for(auto _ : state) {
         m_valid_tx = wallet1.send_to(2, wallet1.generate_key(), true).value();
         wallet1.confirm_transaction(m_valid_tx);
         m_cp_tx = cbdc::transaction::compact_tx(m_valid_tx);
 
         state.ResumeTiming();
-        set.emplace(m_cp_tx.m_id);
-        state.PauseTiming();
-
-        m_cp_tx = cbdc::transaction::compact_tx(m_valid_tx);
-    }
-}
-
-// benchmark how long it takes to remove values from an unordered set
-BENCHMARK_F(uhs_set, erase_item)(benchmark::State& state) {
-    for(auto _ : state) {
-        m_valid_tx = wallet1.send_to(2, wallet1.generate_key(), true).value();
-        wallet1.confirm_transaction(m_valid_tx);
-        m_cp_tx = cbdc::transaction::compact_tx(m_valid_tx);
-        set.emplace(m_cp_tx.m_id);
-        state.ResumeTiming();
-        set.erase(m_cp_tx.m_id);
+        for(const auto& inp : m_cp_tx.m_inputs) {
+            m_uhs.erase(inp);
+        }
+        for(const auto& outp : m_cp_tx.m_outputs) {
+            const auto& uhs_id = cbdc::transaction::calculate_uhs_id(outp);
+            uhs_element el{outp, epoch, std::nullopt};
+            m_uhs.emplace(uhs_id, el);
+        }
         state.PauseTiming();
-
-        m_cp_tx = cbdc::transaction::compact_tx(m_valid_tx);
+        epoch++;
     }
 }