add sql ledger

CMakeLists.txt

@@ -6,6 +6,7 @@ LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
 
 OPTION(LEDGERDB "Use LedgerDB" ON)
 OPTION(AMZQLDB "Use QLDB" OFF)
+OPTION(SQLLEDGER "Use SQL Ledger" OFF)
 
 if(LEDGERDB)
   message(STATUS "Use LedgerDB")
@@ -17,6 +18,11 @@ if(AMZQLDB)
   add_definitions(-DAMZQLDB)
 endif()
 
+if(SQLLEDGER)
+  message(STATUS "Use SQL Ledger")
+  add_definitions(-DSQLLEDGER)
+endif()
+
 SET(BOOST_COMPONENT "filesystem")
 LIST(APPEND BOOST_COMPONENT "program_options")
 LIST(APPEND BOOST_COMPONENT "thread")

distributed/store/common/backend/versionstore.cc

@@ -15,6 +15,10 @@ VersionedKVStore::VersionedKVStore(const string& db_path, int timeout) {
   ledgebase::qldb::BPlusConfig::Init(45, 15);
   qldb_.reset(new ledgebase::qldb::QLDB());
 #endif
+#ifdef SQLLEDGER
+  ledgebase::qldb::BPlusConfig::Init(45, 15);
+  sqlledger_.reset(new ledgebase::sqlledger::SQLLedger(timeout, db_path));
+#endif
 }
 
 VersionedKVStore::~VersionedKVStore() { }
@@ -26,6 +30,9 @@ bool VersionedKVStore::GetDigest(strongstore::proto::Reply* reply) {
 #ifdef LEDGERDB
   ldb->GetRootDigest(&tip, &hash);
 #endif
+#ifdef SQLLEDGER
+  sqlledger_->GetDigest(&tip, &hash);
+#endif
 
   auto digest = reply->mutable_digest();
   digest->set_block(tip);
@@ -80,6 +87,19 @@ bool VersionedKVStore::GetNVersions(
     }
   }
 #endif
+#ifdef SQLLEDGER
+  for (auto& key : ver_keys) {
+    auto result = sqlledger_->GetHistory(key.first, key.second);
+    for (auto& res : result) {
+      auto docs = ledgebase::Utils::splitBy(res, '|');
+      auto kv = reply->add_values();
+      kv->set_key(docs[3]);
+      kv->set_val(docs[4]);
+      kv->set_estimate_block(std::stoul(docs[0]));
+      reply->add_timestamps(std::stoul(docs[5]));
+    }
+  }
+#endif
 
   return true;
 }
@@ -128,6 +148,21 @@ bool VersionedKVStore::BatchGet(const std::vector<std::string>& keys,
     }
   }
 #endif
+#ifdef SQLLEDGER
+  for (auto& key : keys) {
+    auto result = sqlledger_->GetCommitted(key);
+    if (result.size() == 0) {
+      continue;
+    }
+    auto docs = ledgebase::Utils::splitBy(result, '|');
+
+    auto kv = reply->add_values();
+    kv->set_key(docs[3]);
+    kv->set_val(docs[4]);
+    kv->set_estimate_block(std::stoul(docs[0]));
+    reply->add_timestamps(std::stoul(docs[5]));
+  }
+#endif
 
   return true;
 }
@@ -172,6 +207,17 @@ bool VersionedKVStore::GetRange(const std::string &start,
       p->add_pos(pos);
     }
   }
+#endif
+#ifdef SQLLEDGER
+  auto results = sqlledger_->Range(start, end);
+  for (auto& r : results) {
+    auto docs = ledgebase::Utils::splitBy(r.second, '|');
+    auto kv = reply->add_values();
+    kv->set_key(docs[3]);
+    kv->set_val(docs[4]);
+    kv->set_estimate_block(std::stoul(docs[0]));
+    reply->add_timestamps(std::stoul(docs[5]));
+  }
 #endif
   return true;
 }
@@ -219,8 +265,35 @@ bool VersionedKVStore::GetProof(
       p->add_mpt_pos(mptproof[i].GetMapPos(j));
     }
   }
-
 #endif
+#ifdef SQLLEDGER
+  GetDigest(reply);
+  for (auto& entry : keys) {
+    for (auto& key : entry.second) {
+      auto res = sqlledger_->getProof(key, entry.first);
+      auto p = reply->add_sproof();
+      auto blk_proof = p->mutable_blk_proof();
+      auto txn_proof = p->mutable_txn_proof();
+      p->set_digest(res.digest);
+      for (size_t i = 0; i < res.blks.size(); ++i) {
+        p->add_blocks(res.blks[i]);
+      }
+      blk_proof->set_digest(res.blk_proof.digest);
+      blk_proof->set_value(res.blk_proof.value);
+      for (size_t i = 0; i < res.blk_proof.proof.size(); ++i) {
+        blk_proof->add_proof(res.blk_proof.proof[i]);
+        blk_proof->add_pos(res.blk_proof.pos[i]);
+      }
+      txn_proof->set_digest(res.txn_proof.digest);
+      txn_proof->set_value(res.txn_proof.value);
+      for (size_t i = 0; i < res.txn_proof.proof.size(); ++i) {
+        txn_proof->add_proof(res.txn_proof.proof[i]);
+        txn_proof->add_pos(res.txn_proof.pos[i]);
+      }
+    }
+  }
+#endif
+
   gettimeofday(&t1, NULL);
   auto lat = (t1.tv_sec - t0.tv_sec)*1000000 + t1.tv_usec - t0.tv_usec;
   //std::cout << "getproof " << lat << " " << ks.size() << std::endl;
@@ -249,6 +322,16 @@ bool VersionedKVStore::GetProof(const uint64_t& seq,
       reply_mptproof->add_pos(mptproof.GetMapPos(i));
     }
   }
+#endif
+#ifdef SQLLEDGER
+  auto auditor = sqlledger_->getAudit(seq);
+  auto a = reply->mutable_saudit();
+  a->set_block_no(auditor.block_seq);
+  a->set_digest(auditor.digest);
+  a->set_txns(auditor.txns);
+  for (auto& b : auditor.blks) {
+    a->add_blocks(b);
+  }
 #endif
   return true;
 }
@@ -275,6 +358,17 @@ void VersionedKVStore::put(const vector<string> &keys,
     BatchGet(keys, reply);
   }
 #endif
+#ifdef SQLLEDGER
+  auto estimate_blocks = sqlledger_->Set(keys, values);
+  if (reply != nullptr) {
+    for (size_t i = 0; i < keys.size(); ++i) {
+      auto kv = reply->add_values();
+      kv->set_key(keys[i]);
+      kv->set_val(values[i]);
+      kv->set_estimate_block(estimate_blocks);
+    }
+  }
+#endif
 }
 
 bool VersionedKVStore::get(const std::string &key,

distributed/store/common/backend/versionstore.h

@@ -8,6 +8,7 @@
 #include "ledger/ledgerdb/ledgerdb.h"
 #include "ledger/qldb/qldb.h"
 #include "ledger/qldb/bplus_config.h"
+#include "ledger/sqlledger/sqlledger.h"
 #include "distributed/proto/strong-proto.pb.h"
 
 #include "tbb/concurrent_hash_map.h"
@@ -52,6 +53,7 @@ class VersionedKVStore {
 
   std::unique_ptr<ledgebase::ledgerdb::LedgerDB> ldb;
   std::unique_ptr<ledgebase::qldb::QLDB> qldb_;
+  std::unique_ptr<ledgebase::sqlledger::SQLLedger> sqlledger_;
 };
 
 #endif  /* _VERSIONED_KV_STORE_H_ */

distributed/store/strongstore/shardclient.cc

@@ -386,6 +386,38 @@ ShardClient::GetProofCallback(size_t uid,
       }
     }
 #endif
+#ifdef SQLLEDGER
+    res = VerifyStatus::PASS;
+    std::string ledger = "test";
+    auto digest = ledgebase::Hash::FromBase32(reply.digest().hash());
+    for (int i = 0; i < reply.sproof_size(); ++i) {
+      ledgebase::sqlledger::SQLLedgerProof prover;
+      auto curr_proof = reply.sproof(i);
+      prover.digest = curr_proof.digest();
+      for (int j = 0; j < curr_proof.blocks_size(); ++j) {
+        prover.blks.emplace_back(curr_proof.blocks(j));
+      }
+      auto blk_proof = curr_proof.blk_proof();
+      prover.blk_proof.digest = blk_proof.digest();
+      prover.blk_proof.value  = blk_proof.value();
+      for (int j = 0; j < blk_proof.proof_size(); ++j) {
+        prover.blk_proof.proof.emplace_back(blk_proof.proof(j));
+        prover.blk_proof.pos.emplace_back(blk_proof.pos(j));
+      }
+
+      auto txn_proof = curr_proof.txn_proof();
+      prover.txn_proof.digest = txn_proof.digest();
+      prover.txn_proof.value  = txn_proof.value();
+      for (int j = 0; j < txn_proof.proof_size(); ++j) {
+        prover.txn_proof.proof.emplace_back(txn_proof.proof(j));
+        prover.txn_proof.pos.emplace_back(txn_proof.pos(j));
+      }
+
+      if (!prover.Verify()) {
+        res = VerifyStatus::FAILED;
+      }
+    }
+#endif
 
     gettimeofday(&t1, NULL);
     auto elapsed = ((t1.tv_sec - t0.tv_sec)*1000000 +
@@ -409,7 +441,7 @@ ShardClient::GetRangeCallback(const string &request_str, const string &reply_str
     std::vector<std::string> unverified_keys;
     std::vector<uint64_t> estimate_blocks;
 
-#if defined(LEDGERDB)
+#if defined(LEDGERDB) || defined(SQLLEDGER)
     tip_block = reply.digest().block();
     for (size_t i = 0; i < reply.values_size(); ++i) {
       auto v = reply.values(i);
@@ -469,7 +501,7 @@ ShardClient::BatchGetCallback(const string &request_str, const string &reply_str
     std::vector<uint64_t> estimate_blocks;
     std::vector<std::string> unverified_keys;
 
-#if defined(LEDGERDB)
+#if defined(LEDGERDB) || defined(SQLLEDGER)
     tip_block = reply.digest().block();
     for (size_t i = 0; i < reply.values_size(); ++i) {
       auto v = reply.values(i);
@@ -532,7 +564,7 @@ ShardClient::CommitCallback(const string &request_str, const string &reply_str)
     std::vector<uint64_t> estimate_blocks;
     std::vector<std::string> unverified_keys;
     VerifyStatus vs;
-#if defined(LEDGERDB)
+#if defined(LEDGERDB) || defined(SQLLEDGER)
     vs = VerifyStatus::UNVERIFIED;
     tip_block = reply.digest().block();
     for (size_t i = 0; i < reply.values_size(); ++i) {

distributed/store/strongstore/shardclient.h

@@ -13,6 +13,7 @@
 #include "ledger/ledgerdb/mpt/trie.h"
 #include "ledger/ledgerdb/ledgerdb.h"
 #include "ledger/qldb/qldb.h"
+#include "ledger/sqlledger/sqlledger.h"
 #include "distributed/proto/strong-proto.pb.h"
 
 namespace strongstore {

exps/auditors

@@ -0,0 +1 @@
+10.10.10.208

exps/clients

@@ -1,8 +1,8 @@
-10.0.0.4
-10.0.0.5
-10.0.0.6
-10.0.0.7
-10.0.0.8
-10.0.0.9
-10.0.0.11
-10.0.0.12
+10.10.10.6
+10.10.10.252
+10.10.10.249
+10.10.10.248
+10.10.10.8
+10.10.10.18
+10.10.10.15
+10.10.10.207

exps/env.sh

@@ -1,17 +1,19 @@
 #===== Parameters to Fill =====
 
 # project root directory
-rootdir=
+rootdir=/users/yc/LedgerDatabase
+
+# log directory
+logdir="/data/yc/logs"
 
 # master node ip address
-master=
+master=10.10.10.206
 
 # init file directory
-initfile=
+initfile=/data/yc/ustore/tpcc/tpcc
 
 #==============================
 
 bindir="$rootdir/build/bin"
 srcdir="$rootdir/distributed"
-logdir="$rootdir/logs"
 expdir="$rootdir/exps"

exps/replicas

@@ -1,8 +1,16 @@
-10.0.0.38
-10.0.0.37
-10.0.0.36
-10.0.0.35
-10.0.0.34
-10.0.0.33
-10.0.0.32
-10.0.0.31
+10.10.10.237
+10.10.10.239
+10.10.10.244
+10.10.10.238
+10.10.10.236
+10.10.10.241
+10.10.10.242
+10.10.10.240
+10.10.10.245
+10.10.10.246
+10.10.10.250
+10.10.10.247
+10.10.10.251
+10.10.10.5
+10.10.10.253
+10.10.10.7

exps/result/50_16_10_0

@@ -0,0 +1,9 @@
+9965.0
+7794.20214501
+10247.1
+7689.38388666
+0.0275297401216
+7691.5028442
+7687.2678015
+0
+0

exps/result/50_16_1_0

@@ -0,0 +1,9 @@
+3505.25
+2256.83639541
+3541.45
+3680.36462466
+0.0102218018043
+2242.07954256
+2269.36694257
+0
+179135.863278

exps/result/50_16_20_0

@@ -0,0 +1,9 @@
+119912.1
+1322.05407524
+120880.1
+1863.39412401
+0.00800793513573
+1305.02996834
+1339.30612563
+0
+71520.5022935

exps/result/50_16_2_0

@@ -0,0 +1,9 @@
+8413.975
+1871.4728324
+8521.95
+2872.21118113
+0.0126702221909
+1859.65928002
+1883.36434218
+0
+119172.167813

exps/result/50_16_3_0

@@ -0,0 +1,9 @@
+7520.65
+3155.74825979
+7571.425
+3145.17701423
+0.00670613523874
+3138.08738918
+3152.23007509
+0
+0

exps/result/50_16_4_0

@@ -0,0 +1,9 @@
+8911.575
+3544.5701321
+8999.1
+3526.7385933
+0.00972597259726
+3519.92075715
+3533.57721971
+0
+0