From 4c2f1ab8df5fb8fc1747c211cd8de43e6810e2ae Mon Sep 17 00:00:00 2001
From: yc1111 <yuecong@gmail.com>
Date: Thu, 15 Dec 2022 17:24:25 +0000
Subject: [PATCH] add sql ledger, fix merkle tree

---
 .gitignore                                    |  2 +
 .../store/common/backend/versionstore.cc      |  5 ++-
 distributed/store/strongstore/shardclient.cc  | 14 +++++-
 exps/aggregate_ycsb.py                        |  8 ++++
 exps/parse_ycsb.py                            |  3 ++
 exps/process_ycsb.py                          |  7 +++
 exps/result/50_16_10_0                        |  9 ----
 exps/result/50_16_1_0                         |  9 ----
 exps/result/50_16_20_0                        |  9 ----
 exps/result/50_16_2_0                         |  9 ----
 exps/result/50_16_3_0                         |  9 ----
 exps/result/50_16_4_0                         |  9 ----
 exps/result/50_16_5_0                         |  9 ----
 exps/result/50_16_6_0                         |  9 ----
 exps/result/50_16_7_0                         |  9 ----
 exps/result/50_16_8_0                         |  9 ----
 exps/result/50_16_9_0                         |  9 ----
 exps/result/abort_50_0                        |  2 -
 exps/result/history                           |  2 -
 exps/result/lat_50_0                          |  2 -
 exps/result/read                              |  2 -
 exps/result/tps_50_0                          |  2 -
 exps/result/verify                            |  2 -
 exps/result/write                             |  2 -
 exps/run_exp.sh                               |  4 +-
 exps/run_tpcc.sh                              | 10 ++---
 exps/run_ycsb.sh                              | 10 ++---
 exps/shard.tss.config                         |  2 -
 exps/shard0.config                            |  2 -
 exps/shard1.config                            |  2 -
 exps/shard10.config                           |  2 -
 exps/shard11.config                           |  2 -
 exps/shard12.config                           |  2 -
 exps/shard13.config                           |  2 -
 exps/shard14.config                           |  2 -
 exps/shard15.config                           |  2 -
 exps/shard2.config                            |  2 -
 exps/shard3.config                            |  2 -
 exps/shard4.config                            |  2 -
 exps/shard5.config                            |  2 -
 exps/shard6.config                            |  2 -
 exps/shard7.config                            |  2 -
 exps/shard8.config                            |  2 -
 exps/shard9.config                            |  2 -
 exps/start_replica.sh                         |  2 +-
 ledger/ledgerdb/ledgerdb.cc                   | 11 ++---
 ledger/ledgerdb/merkletree.cc                 | 45 ++++++++++---------
 ledger/ledgerdb/merkletree.h                  |  2 +-
 ledger/ledgerdb/mpt/trie.cc                   |  1 +
 ledger/qldb/qldb.cc                           |  3 ++
 ledger/sqlledger/sqlldg_mt.cc                 |  2 +
 ledger/sqlledger/sqlledger.cc                 | 33 +++++++++++---
 test/ledger/test_merkle_tree.cc               |  4 +-
 53 files changed, 118 insertions(+), 195 deletions(-)
 delete mode 100644 exps/result/50_16_10_0
 delete mode 100644 exps/result/50_16_1_0
 delete mode 100644 exps/result/50_16_20_0
 delete mode 100644 exps/result/50_16_2_0
 delete mode 100644 exps/result/50_16_3_0
 delete mode 100644 exps/result/50_16_4_0
 delete mode 100644 exps/result/50_16_5_0
 delete mode 100644 exps/result/50_16_6_0
 delete mode 100644 exps/result/50_16_7_0
 delete mode 100644 exps/result/50_16_8_0
 delete mode 100644 exps/result/50_16_9_0
 delete mode 100644 exps/result/abort_50_0
 delete mode 100644 exps/result/history
 delete mode 100644 exps/result/lat_50_0
 delete mode 100644 exps/result/read
 delete mode 100644 exps/result/tps_50_0
 delete mode 100644 exps/result/verify
 delete mode 100644 exps/result/write
 delete mode 100644 exps/shard.tss.config
 delete mode 100644 exps/shard0.config
 delete mode 100644 exps/shard1.config
 delete mode 100644 exps/shard10.config
 delete mode 100644 exps/shard11.config
 delete mode 100644 exps/shard12.config
 delete mode 100644 exps/shard13.config
 delete mode 100644 exps/shard14.config
 delete mode 100644 exps/shard15.config
 delete mode 100644 exps/shard2.config
 delete mode 100644 exps/shard3.config
 delete mode 100644 exps/shard4.config
 delete mode 100644 exps/shard5.config
 delete mode 100644 exps/shard6.config
 delete mode 100644 exps/shard7.config
 delete mode 100644 exps/shard8.config
 delete mode 100644 exps/shard9.config

diff --git a/.gitignore b/.gitignore
index f785746..0844d7f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 build/
+exps/result/
+exps/shard*
 .vscode
diff --git a/distributed/store/common/backend/versionstore.cc b/distributed/store/common/backend/versionstore.cc
index e3f5ceb..a532162 100644
--- a/distributed/store/common/backend/versionstore.cc
+++ b/distributed/store/common/backend/versionstore.cc
@@ -227,6 +227,7 @@ bool VersionedKVStore::GetProof(
     strongstore::proto::Reply* reply) {
   timeval t0, t1;
   gettimeofday(&t0, NULL);
+  int nkey = 0;
 #ifdef LEDGERDB
   std::vector<ledgebase::ledgerdb::Proof> mtproof;
   std::vector<ledgebase::ledgerdb::MPTProof> mptproof;
@@ -243,6 +244,7 @@ bool VersionedKVStore::GetProof(
       blks.push_back(vkey.first);
     }
   }
+  nkey = ks.size();
   
   ldb->GetProofs(ks, blks, mtproof, mptproof, &mtdigest,
       &block, &mptdigest);
@@ -270,6 +272,7 @@ bool VersionedKVStore::GetProof(
   GetDigest(reply);
   for (auto& entry : keys) {
     for (auto& key : entry.second) {
+      nkey++;
       auto res = sqlledger_->getProof(key, entry.first);
       auto p = reply->add_sproof();
       auto blk_proof = p->mutable_blk_proof();
@@ -296,7 +299,7 @@ bool VersionedKVStore::GetProof(
 
   gettimeofday(&t1, NULL);
   auto lat = (t1.tv_sec - t0.tv_sec)*1000000 + t1.tv_usec - t0.tv_usec;
-  //std::cout << "getproof " << lat << " " << ks.size() << std::endl;
+  //std::cout << "getproof " << lat << " " << nkey << std::endl;
   return true;
 }
 
diff --git a/distributed/store/strongstore/shardclient.cc b/distributed/store/strongstore/shardclient.cc
index caf36c6..0812e31 100644
--- a/distributed/store/strongstore/shardclient.cc
+++ b/distributed/store/strongstore/shardclient.cc
@@ -422,7 +422,7 @@ ShardClient::GetProofCallback(size_t uid,
     gettimeofday(&t1, NULL);
     auto elapsed = ((t1.tv_sec - t0.tv_sec)*1000000 +
                     (t1.tv_usec - t0.tv_usec));
-    // std::cout << "verify " << elapsed << " " << reply.ByteSizeLong() << " " << keys.size() << " " << res << std::endl;
+    //std::cout << "verify " << elapsed << " " << reply.ByteSizeLong() << " " << keys.size() << " " << res << std::endl;
 
     Promise *w = verifyPromise[uid];
     verifyPromise.erase(uid);
@@ -514,6 +514,8 @@ ShardClient::BatchGetCallback(const string &request_str, const string &reply_str
     }
 #endif
 #ifdef AMZQLDB
+    struct timeval t0, t1;
+    gettimeofday(&t0, NULL);
     VerifyStatus vs = VerifyStatus::PASS;
     ledgebase::Hash digest;
     std::string ledger = "test";
@@ -542,6 +544,10 @@ ShardClient::BatchGetCallback(const string &request_str, const string &reply_str
         vs = VerifyStatus::FAILED;
       }
     }
+    gettimeofday(&t1, NULL);
+    auto elapsed = ((t1.tv_sec - t0.tv_sec)*1000000 +
+                    (t1.tv_usec - t0.tv_usec));
+    // std::cout << "verify " << elapsed << " " << reply.ByteSizeLong() << " " << reply.qproof_size() << " " << vs << std::endl;
 #endif
 
     Promise *w = waiting;
@@ -574,6 +580,8 @@ ShardClient::CommitCallback(const string &request_str, const string &reply_str)
     }
 #endif
 #ifdef AMZQLDB
+    struct timeval t0, t1;
+    gettimeofday(&t0, NULL);
     vs = VerifyStatus::PASS;
     std::string ledger = "test";
     auto digest = ledgebase::Hash::FromBase32(reply.digest().hash());
@@ -597,6 +605,10 @@ ShardClient::CommitCallback(const string &request_str, const string &reply_str)
         vs = VerifyStatus::FAILED;
       }
     }
+    gettimeofday(&t1, NULL);
+    auto elapsed = ((t1.tv_sec - t0.tv_sec)*1000000 +
+                    (t1.tv_usec - t0.tv_usec));
+    std::cout << "verify " << elapsed << " " << reply.ByteSizeLong() << " " << reply.qproof_size() << " " << vs << std::endl;
 #endif
 
     Promise *w = waiting;
diff --git a/exps/aggregate_ycsb.py b/exps/aggregate_ycsb.py
index 9222fba..136d271 100644
--- a/exps/aggregate_ycsb.py
+++ b/exps/aggregate_ycsb.py
@@ -14,6 +14,7 @@
 sumH = 0.0
 numV = 0.0
 sumV = 0.0
+sumN = 0.0
 
 path = sys.argv[1]
 outpath = sys.argv[2]
@@ -34,6 +35,7 @@
     sumH = sumH + float(lines[10])
     numV = numV + float(lines[11])
     sumV = sumV + float(lines[12])
+    sumN = sumN + float(lines[13])
 
 outfile = open(outpath, "w")
 outfile.write(str(numSuccess/duration) + "\n")
@@ -60,3 +62,9 @@
   outfile.write("0\n")
 else:
   outfile.write(str(sumV/numV) + "\n")
+
+if sumN > 0:
+  outfile.write(str(sumV/sumN) + "\n")
+else:
+  outfile.write("0\n")
+
diff --git a/exps/parse_ycsb.py b/exps/parse_ycsb.py
index b92f176..0f97b10 100644
--- a/exps/parse_ycsb.py
+++ b/exps/parse_ycsb.py
@@ -36,6 +36,7 @@ def print_result(fp, path, wper, servers, clients, t, lineno):
     wlat = open(path + "/write", "w")
     hlat = open(path + "/history", "w")
     vlat = open(path + "/verify", "w")
+    vpk = open(path + "/verifyperkey", "w")
 
     print_result(ftps, path, wper, servers, clients, t, 0)
     print_result(flat, path, wper, servers, clients, t, 1)
@@ -44,6 +45,7 @@ def print_result(fp, path, wper, servers, clients, t, lineno):
     print_result(wlat, path, wper, servers, clients, t, 6)
     print_result(hlat, path, wper, servers, clients, t, 7)
     print_result(vlat, path, wper, servers, clients, t, 8)
+    print_result(vpk, path, wper, servers, clients, t, 9)
 
     ftps.close()
     flat.close()
@@ -52,3 +54,4 @@ def print_result(fp, path, wper, servers, clients, t, lineno):
     wlat.close()
     hlat.close()
     vlat.close()
+    vpk.close()
diff --git a/exps/process_ycsb.py b/exps/process_ycsb.py
index a24ec1d..eea6189 100644
--- a/exps/process_ycsb.py
+++ b/exps/process_ycsb.py
@@ -13,6 +13,8 @@
 wLatency = []
 hLatency = []
 vLatency = []
+nkey = 0
+nkeys = []
 
 tExtra = 0.0
 sExtra = 0.0
@@ -23,6 +25,9 @@
 for line in open(sys.argv[1]):
   if line.startswith('#') or line.strip() == "":
     continue
+  if line.startswith("verifynkeys"):
+    nkey = float(line[12:-1])
+    continue
 
   line = line.strip().split()
   if not line[0].isdigit() or len(line) < 4:
@@ -56,6 +61,7 @@
   elif op == 2:
     hLatency.append(latency)
   elif int(line[0]) > 0:
+    nkeys.append(nkey)
     vLatency.append(latency)
 
   tLatency.append(latency) 
@@ -90,6 +96,7 @@
 outfile.write(str(sum(hLatency)) + "\n")
 outfile.write(str(len(vLatency)) + "\n")
 outfile.write(str(sum(vLatency)) + "\n")
+outfile.write(str(sum(nkeys)) + "\n")
 # print "Transactions(All/Success): ", len(tLatency), len(sLatency)
 # print "Abort Rate: ", (float)(len(tLatency)-len(sLatency))/len(tLatency)
 # print "Throughput (All/Success): ", len(tLatency)/(end-start), len(sLatency)/(end-start)
diff --git a/exps/result/50_16_10_0 b/exps/result/50_16_10_0
deleted file mode 100644
index 0425f1b..0000000
--- a/exps/result/50_16_10_0
+++ /dev/null
@@ -1,9 +0,0 @@
-9965.0
-7794.20214501
-10247.1
-7689.38388666
-0.0275297401216
-7691.5028442
-7687.2678015
-0
-0
diff --git a/exps/result/50_16_1_0 b/exps/result/50_16_1_0
deleted file mode 100644
index 9925250..0000000
--- a/exps/result/50_16_1_0
+++ /dev/null
@@ -1,9 +0,0 @@
-3505.25
-2256.83639541
-3541.45
-3680.36462466
-0.0102218018043
-2242.07954256
-2269.36694257
-0
-179135.863278
diff --git a/exps/result/50_16_20_0 b/exps/result/50_16_20_0
deleted file mode 100644
index 5f73504..0000000
--- a/exps/result/50_16_20_0
+++ /dev/null
@@ -1,9 +0,0 @@
-119912.1
-1322.05407524
-120880.1
-1863.39412401
-0.00800793513573
-1305.02996834
-1339.30612563
-0
-71520.5022935
diff --git a/exps/result/50_16_2_0 b/exps/result/50_16_2_0
deleted file mode 100644
index a1cdd23..0000000
--- a/exps/result/50_16_2_0
+++ /dev/null
@@ -1,9 +0,0 @@
-8413.975
-1871.4728324
-8521.95
-2872.21118113
-0.0126702221909
-1859.65928002
-1883.36434218
-0
-119172.167813
diff --git a/exps/result/50_16_3_0 b/exps/result/50_16_3_0
deleted file mode 100644
index cbbb0ba..0000000
--- a/exps/result/50_16_3_0
+++ /dev/null
@@ -1,9 +0,0 @@
-7520.65
-3155.74825979
-7571.425
-3145.17701423
-0.00670613523874
-3138.08738918
-3152.23007509
-0
-0
diff --git a/exps/result/50_16_4_0 b/exps/result/50_16_4_0
deleted file mode 100644
index 74e0665..0000000
--- a/exps/result/50_16_4_0
+++ /dev/null
@@ -1,9 +0,0 @@
-8911.575
-3544.5701321
-8999.1
-3526.7385933
-0.00972597259726
-3519.92075715
-3533.57721971
-0
-0
diff --git a/exps/result/50_16_5_0 b/exps/result/50_16_5_0
deleted file mode 100644
index e0c6d8b..0000000
--- a/exps/result/50_16_5_0
+++ /dev/null
@@ -1,9 +0,0 @@
-8579.0
-4589.56249854
-8689.0
-4560.50937104
-0.0126596846588
-4546.85271398
-4574.19875902
-0
-0
diff --git a/exps/result/50_16_6_0 b/exps/result/50_16_6_0
deleted file mode 100644
index 9106068..0000000
--- a/exps/result/50_16_6_0
+++ /dev/null
@@ -1,9 +0,0 @@
-9029.35
-5216.68053071
-9174.7
-5175.47914101
-0.0158424798631
-5167.86748671
-5183.06991717
-0
-0
diff --git a/exps/result/50_16_7_0 b/exps/result/50_16_7_0
deleted file mode 100644
index bea47a9..0000000
--- a/exps/result/50_16_7_0
+++ /dev/null
@@ -1,9 +0,0 @@
-9777.225
-5610.23737308
-9961.2
-5558.50138538
-0.0184691603421
-5554.86899811
-5562.13774954
-0
-0
diff --git a/exps/result/50_16_8_0 b/exps/result/50_16_8_0
deleted file mode 100644
index 129469e..0000000
--- a/exps/result/50_16_8_0
+++ /dev/null
@@ -1,9 +0,0 @@
-9544.725
-6543.80308233
-9757.2
-6473.98437052
-0.0217762267864
-6466.95812444
-6481.0489587
-0
-0
diff --git a/exps/result/50_16_9_0 b/exps/result/50_16_9_0
deleted file mode 100644
index 0b3ebc5..0000000
--- a/exps/result/50_16_9_0
+++ /dev/null
@@ -1,9 +0,0 @@
-9697.05
-7227.88689859
-9945.8
-7140.18570653
-0.0250105572201
-7149.40989231
-7130.9459265
-0
-0
diff --git a/exps/result/abort_50_0 b/exps/result/abort_50_0
deleted file mode 100644
index 1888a6b..0000000
--- a/exps/result/abort_50_0
+++ /dev/null
@@ -1,2 +0,0 @@
-"#Servers"	"1"
-16	"0.0102218018043"
diff --git a/exps/result/history b/exps/result/history
deleted file mode 100644
index 6c3677e..0000000
--- a/exps/result/history
+++ /dev/null
@@ -1,2 +0,0 @@
-"#Servers"	"1"
-16	"0"
diff --git a/exps/result/lat_50_0 b/exps/result/lat_50_0
deleted file mode 100644
index 9c934d0..0000000
--- a/exps/result/lat_50_0
+++ /dev/null
@@ -1,2 +0,0 @@
-"#Servers"	"1"
-16	"2256.83639541"
diff --git a/exps/result/read b/exps/result/read
deleted file mode 100644
index 8a5ab6f..0000000
--- a/exps/result/read
+++ /dev/null
@@ -1,2 +0,0 @@
-"#Servers"	"1"
-16	"2242.07954256"
diff --git a/exps/result/tps_50_0 b/exps/result/tps_50_0
deleted file mode 100644
index c715ddb..0000000
--- a/exps/result/tps_50_0
+++ /dev/null
@@ -1,2 +0,0 @@
-"#Servers"	"1"
-16	"3505.25"
diff --git a/exps/result/verify b/exps/result/verify
deleted file mode 100644
index 3819de0..0000000
--- a/exps/result/verify
+++ /dev/null
@@ -1,2 +0,0 @@
-"#Servers"	"1"
-16	"179135.863278"
diff --git a/exps/result/write b/exps/result/write
deleted file mode 100644
index 0366663..0000000
--- a/exps/result/write
+++ /dev/null
@@ -1,2 +0,0 @@
-"#Servers"	"1"
-16	"2269.36694257"
diff --git a/exps/run_exp.sh b/exps/run_exp.sh
index 9314e58..441e6e8 100755
--- a/exps/run_exp.sh
+++ b/exps/run_exp.sh
@@ -7,7 +7,7 @@ wperc=(50)
 nshards=(16)
 
 # Number of client processes per client node
-nclients=(1)
+nclients=(10)
 
 # Zipf factor
 theta=(0)
@@ -33,6 +33,7 @@ thetas=$( IFS=$','; echo "${theta[*]}" )
 
 for i in ${wperc[@]}
 do
+  let rper=100-$i
   for j in ${nshards[@]}
   do
     ./load.sh $j
@@ -47,6 +48,7 @@ do
         sed -i -e "s/tlen=[0-9]*/tlen=${tlen}/g" run_$driver.sh
         sed -i -e "s/rtime=[0-9]*/rtime=${rtime}/g" run_$driver.sh
         sed -i -e "s/wper=[0-9]*/wper=$i/g" run_$driver.sh
+        sed -i -e "s/rper=[0-9]*/rper=$rper/g" run_$driver.sh
         sed -i -e "s/nshard=[0-9]*/nshard=$j/g" run_$driver.sh
         sed -i -e "s/nclient=[0-9]*/nclient=$c/g" run_$driver.sh
         sed -i -e "s/zalpha=[0-9\.]*/zalpha=$k/g" run_$driver.sh
diff --git a/exps/run_tpcc.sh b/exps/run_tpcc.sh
index 1071f33..4fb3578 100755
--- a/exps/run_tpcc.sh
+++ b/exps/run_tpcc.sh
@@ -2,11 +2,11 @@
 
 #============= Parameters to fill ============
 nshard=16          # number of shards
-nclient=20         # number of clients / machine
+nclient=10         # number of clients / machine
 rtime=120          # duration to run
-delay=100          # verification delay
+delay=1000          # verification delay
 
-wper=50             # writes percentage
+wper=0             # writes percentage
 zalpha=0           # zipf alpha
 #============= Start Experiment =============
 . env.sh
@@ -34,12 +34,12 @@ for ((i=0; i<$nshard; i++))
 do
   echo "Starting shard$i replicas.."
   $expdir/start_replica.sh shard$i $expdir/shard$i.config \
-    "$bindir/$store -m $mode -e 0 -s 0 -w tpcc -f $initfile -N $nshard -n $i" $logdir
+    "$bindir/$store -m $mode -e 0 -s 0 -w tpcc -f $initfile -N $nshard -n $i -t 100" $logdir
 done
 
 
 # Wait a bit for all replicas to start up
-sleep 10
+sleep 50
 
 # Run the clients
 echo "Running the client(s)"
diff --git a/exps/run_ycsb.sh b/exps/run_ycsb.sh
index 8b0166d..afe0a5e 100755
--- a/exps/run_ycsb.sh
+++ b/exps/run_ycsb.sh
@@ -1,12 +1,12 @@
 #!/bin/bash
 
 #============= Parameters to fill ============
-nshard=16           # number of shards
-nclient=1         # number of clients / machine
+nshard=1           # number of shards
+nclient=20         # number of clients / machine
 rtime=120          # duration to run
 delay=100          # verification delay
 
-tlen=10            # transaction length
+tlen=1            # transaction length
 wper=50            # writes percentage
 rper=50            # reads percentage
 zalpha=0           # zipf alpha
@@ -19,7 +19,7 @@ clients=`cat clients`
 client="verifyClient"
 store="strongstore"
 mode="occ"
-txnrate=1000
+txnrate=120
 
 # Print out configuration being used.
 echo "Configuration:"
@@ -40,7 +40,7 @@ for ((i=0; i<$nshard; i++))
 do
   echo "Starting shard$i replicas.."
   $expdir/start_replica.sh shard$i $expdir/shard$i.config \
-    "$bindir/$store -m $mode -e 0 -s 0 -N $nshard -n $i -w ycsb -k 100000" $logdir
+    "$bindir/$store -m $mode -e 0 -s 0 -N $nshard -n $i -t 100 -w ycsb -k 100000" $logdir
 done
 
 # Wait a bit for all replicas to start up
diff --git a/exps/shard.tss.config b/exps/shard.tss.config
deleted file mode 100644
index 1a28442..0000000
--- a/exps/shard.tss.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.205:51725
diff --git a/exps/shard0.config b/exps/shard0.config
deleted file mode 100644
index 76641c2..0000000
--- a/exps/shard0.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.237:51729
diff --git a/exps/shard1.config b/exps/shard1.config
deleted file mode 100644
index 13b07e1..0000000
--- a/exps/shard1.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.239:51730
diff --git a/exps/shard10.config b/exps/shard10.config
deleted file mode 100644
index 87083e9..0000000
--- a/exps/shard10.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.250:51739
diff --git a/exps/shard11.config b/exps/shard11.config
deleted file mode 100644
index 1104366..0000000
--- a/exps/shard11.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.247:51740
diff --git a/exps/shard12.config b/exps/shard12.config
deleted file mode 100644
index bd3cdfc..0000000
--- a/exps/shard12.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.251:51741
diff --git a/exps/shard13.config b/exps/shard13.config
deleted file mode 100644
index ba694d2..0000000
--- a/exps/shard13.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.5:51742
diff --git a/exps/shard14.config b/exps/shard14.config
deleted file mode 100644
index 396d8ab..0000000
--- a/exps/shard14.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.253:51743
diff --git a/exps/shard15.config b/exps/shard15.config
deleted file mode 100644
index ff462e3..0000000
--- a/exps/shard15.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.7:51744
diff --git a/exps/shard2.config b/exps/shard2.config
deleted file mode 100644
index 3761d4e..0000000
--- a/exps/shard2.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.244:51731
diff --git a/exps/shard3.config b/exps/shard3.config
deleted file mode 100644
index be6ae52..0000000
--- a/exps/shard3.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.238:51732
diff --git a/exps/shard4.config b/exps/shard4.config
deleted file mode 100644
index 6dade57..0000000
--- a/exps/shard4.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.236:51733
diff --git a/exps/shard5.config b/exps/shard5.config
deleted file mode 100644
index 12006d4..0000000
--- a/exps/shard5.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.241:51734
diff --git a/exps/shard6.config b/exps/shard6.config
deleted file mode 100644
index 5e56953..0000000
--- a/exps/shard6.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.242:51735
diff --git a/exps/shard7.config b/exps/shard7.config
deleted file mode 100644
index ae0e36d..0000000
--- a/exps/shard7.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.240:51736
diff --git a/exps/shard8.config b/exps/shard8.config
deleted file mode 100644
index 069e293..0000000
--- a/exps/shard8.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.245:51737
diff --git a/exps/shard9.config b/exps/shard9.config
deleted file mode 100644
index a282220..0000000
--- a/exps/shard9.config
+++ /dev/null
@@ -1,2 +0,0 @@
-f 0
-replica 10.10.10.246:51738
diff --git a/exps/start_replica.sh b/exps/start_replica.sh
index a48222d..4ae2b43 100755
--- a/exps/start_replica.sh
+++ b/exps/start_replica.sh
@@ -17,7 +17,7 @@ for ((i=0; i<$n; i++))
 do
   let line=$i+2 
   server=$(cat $config | sed -n ${line}p | awk -F'[ :]' '{print $2}')
-  command="ssh $server \"source ~/.profile; source ~/.bashrc; mkdir -p $logdir; $cmd -c $config -i $i > $logdir/$shard.replica$i.log 2>&1 &\""
+  command="ssh $server \"source ~/.profile; source ~/.bashrc; mkdir -p $logdir; $cmd -c $config -i $i > $logdir/$shard.replica$i.log 2>$logdir/$shard.replica$i.err &\""
   echo $command
   eval $command
 done
diff --git a/ledger/ledgerdb/ledgerdb.cc b/ledger/ledgerdb/ledgerdb.cc
index 036f312..03f3c01 100644
--- a/ledger/ledgerdb/ledgerdb.cc
+++ b/ledger/ledgerdb/ledgerdb.cc
@@ -45,7 +45,7 @@ void LedgerDB::buildTree(int timeout) {
     uint64_t first_block;
     uint64_t last_block;
     Tree_Block blk;
-    std::vector<Hash> mt_new_hashes;
+    std::vector<std::string> mt_new_hashes;
     std::map<std::string, std::string> mpt_blks;
 
     while (tree_queue_.try_pop(blk)) {
@@ -56,7 +56,8 @@ void LedgerDB::buildTree(int timeout) {
       added = true;
       std::string mt_blk_val;
       ledger_.Get("ledger-"+std::to_string(blk.blk_seq), &mt_blk_val);
-      mt_new_hashes.push_back(Hash::ComputeFrom(mt_blk_val));
+      auto hash = Hash::ComputeFrom(mt_blk_val);
+      mt_new_hashes.push_back(hash.ToBase32());
       for (size_t i = 0; i < blk.mpt_ks.size(); i++) {
         mpt_blks[blk.mpt_ks[i]] = blk.mpt_ts;
       }
@@ -102,7 +103,7 @@ void LedgerDB::buildTree(int timeout) {
       ++commit_seq_;
       gettimeofday(&t1, NULL);
       auto latency = (t1.tv_sec - t0.tv_sec)*1000000 + t1.tv_usec - t0.tv_usec;
-      //std::cerr << "persist " << latency << " " << mpt_ks.size() << std::endl;
+      //std::cerr << "persist " << latency << " " << mpt_ks.size() << " " << mt_new_hashes.size() << std::endl;
     }
     std::this_thread::sleep_for(std::chrono::milliseconds(timeout));
   }
@@ -359,9 +360,9 @@ bool Auditor::Audit(DB* db) {
         std::to_string(cinfo.commit_seq - 1) : "";
   }
 
-  std::vector<Hash> mt_new_hashes;
+  std::vector<std::string> mt_new_hashes;
   for (size_t i = 0; i < blocks.size(); ++i) {
-    mt_new_hashes.emplace_back(Hash::ComputeFrom(blocks[i]));
+    mt_new_hashes.emplace_back(Hash::ComputeFrom(blocks[i]).ToBase32());
   }
 
   std::string root_key, root_hash;
diff --git a/ledger/ledgerdb/merkletree.cc b/ledger/ledgerdb/merkletree.cc
index 35ddb5c..fd4326a 100644
--- a/ledger/ledgerdb/merkletree.cc
+++ b/ledger/ledgerdb/merkletree.cc
@@ -6,42 +6,40 @@ namespace ledgebase {
 namespace ledgerdb {
 
 void MerkleTree::update(const uint64_t starting_block_seq,
-                        const std::vector<Hash> &leaf_block_hashes,
+                        const std::vector<std::string> &leaf_block_hashes,
                         const std::string& prev_commit_seq,
                         std::string* root_key,
                         std::string* root_hash) {
   // commit leaf hashes
   int level = 0;
   uint64_t level_starting_seq = starting_block_seq;
-  uint64_t prev_last_seq = starting_block_seq > 0 ? starting_block_seq - 1 : 0;
+  bool prev_complete = starting_block_seq > 1 ?
+      (starting_block_seq - 1) % 2 == 1 : true;
+  bool complete = true;
   std::vector<Hash> level_hashes;
   for (size_t i = 0; i < leaf_block_hashes.size(); ++i) {
     ledger_->Put("mt0-" + std::to_string(starting_block_seq + i),
-        leaf_block_hashes[i].ToBase32());
-    level_hashes.emplace_back(leaf_block_hashes[i]);
+        leaf_block_hashes[i]);
+    level_hashes.emplace_back(Hash::FromBase32(leaf_block_hashes[i]));
   }
 
   std::string commit_seq = prev_commit_seq.size() == 0 ?
       "0" : std::to_string(std::stoul(prev_commit_seq) + 1);
-  while (level_hashes.size() > 1 || prev_last_seq > 0 || level_starting_seq > 0) {
-    bool recommit = false;
+  while (level_hashes.size() > 1 || level_starting_seq > 0) {
     std::vector<Hash> parent_hashes;
     // load previous hash when needed
     if (level_starting_seq % 2 == 1) {
       std::string prev_key = "mt" + std::to_string(level) + "-" +
           std::to_string(level_starting_seq - 1);
-      if (level > 0 && level_starting_seq - 1 == prev_last_seq) {
-        prev_key += "-" + prev_commit_seq;
-        recommit = true;
+      if (level > 0 && !prev_complete) {
+          prev_key += "-" + prev_commit_seq;
       }
       std::string prev_hash;
       ledger_->Get(prev_key, &prev_hash);
       level_hashes.insert(level_hashes.begin(), Hash::FromBase32(prev_hash));
-      if (recommit) {
-        ledger_->Put("mt" + std::to_string(level) + "-" +
-            std::to_string(level_starting_seq - 1), prev_hash);
-      }
       --level_starting_seq;
+      prev_complete = level_starting_seq > 1 ?
+          (level_starting_seq - 1) % 2 == 1 : true;
     }
     for (size_t i = 0; i < level_hashes.size(); i = i + 2) {
       std::string parent_key = "mt" + std::to_string(level + 1) + "-" +
@@ -53,7 +51,7 @@ void MerkleTree::update(const uint64_t starting_block_seq,
         memcpy(data.get() + Hash::kByteLength, level_hashes[i+1].value(),
             Hash::kByteLength);
         auto parent = Hash::ComputeFrom(data.get(), Hash::kByteLength*2);
-        if (i + 1 == level_hashes.size() - 1) {
+        if (i + 1 == level_hashes.size() - 1 && !complete) {
           parent_key += "-" + commit_seq;
         }
         ledger_->Put(parent_key, parent.ToBase32());
@@ -63,6 +61,7 @@ void MerkleTree::update(const uint64_t starting_block_seq,
             Hash::kByteLength);
         if (i == level_hashes.size() - 1) {
           parent_key += "-" + commit_seq;
+          complete = false;
         }
         ledger_->Put(parent_key, parent.ToBase32());
         parent_hashes.emplace_back(parent.Clone());
@@ -70,10 +69,10 @@ void MerkleTree::update(const uint64_t starting_block_seq,
     }
     level_hashes = std::move(parent_hashes);
     level_starting_seq /= 2;
-    prev_last_seq /= 2;
     ++level;
   }
-  *root_key = "mt" + std::to_string(level) + "-0-" + commit_seq;
+  *root_key = "mt" + std::to_string(level) + "-0" +
+      (complete? "": ("-" + commit_seq));
   *root_hash = level_hashes[0].ToBase32();
 }
 
@@ -84,12 +83,14 @@ Proof MerkleTree::getProof(const std::string& commit_seq,
   Proof proof;
   std::string digest, value;
   ledger_->Get(root_key, &digest);
-  ledger_->Get("ledger-" + std::to_string(seq), &value);
+  ledger_->Get("mt0-" + std::to_string(seq), &value);
   proof.digest = digest;
   proof.value = value;
+  bool complete = (tip % 2 == 1);
 
   auto delim = root_key.find("-");
   int level = std::stoi(root_key.substr(2, delim - 2));
+  //std::cout << "height " << level << std::endl;
   uint64_t ptr = seq;
   uint64_t last = tip;
   for (int i = 0; i < level; ++i) {
@@ -97,25 +98,27 @@ Proof MerkleTree::getProof(const std::string& commit_seq,
     if (ptr % 2  == 0) {
       std::string sibling_key =
           "mt" + std::to_string(i) + "-" + std::to_string(ptr+1);
-      if (ptr + 1 == last) {
+      if (ptr + 1 == last && i > 0 && !complete) {
         sibling_key += "-" + commit_seq;
       }
       ledger_->Get(sibling_key, &res);
       proof.pos.emplace_back(1);
     } else {
-      ledger_->Get("mt" + std::to_string(i) + "-" + std::to_string(ptr-1),
-          &res);
+      std::string sibling_key =
+          "mt" + std::to_string(i) + "-" + std::to_string(ptr-1);
+      ledger_->Get(sibling_key, &res);
       proof.pos.emplace_back(0);
     }
     proof.proof.emplace_back(res);
     ptr /= 2;
     last /= 2;
+    complete = complete && (last % 2 == 1);
   }
   return proof;
 }
 
 bool Proof::Verify() const {
-  auto calc_hash = Hash::ComputeFrom(value);
+  auto calc_hash = Hash::FromBase32(value);  //Hash::ComputeFrom(value);
   for (size_t i = 0; i < proof.size(); ++i) {
     auto proof_hash = Hash::FromBase32(proof[i]);
     if (pos[i] == 0) {
diff --git a/ledger/ledgerdb/merkletree.h b/ledger/ledgerdb/merkletree.h
index 672f9ce..3374d97 100644
--- a/ledger/ledgerdb/merkletree.h
+++ b/ledger/ledgerdb/merkletree.h
@@ -38,7 +38,7 @@ class MerkleTree {
   MerkleTree(DB *db) : ledger_(db) { }
   ~MerkleTree() = default;
 
-  void update(const uint64_t blk_seq, const std::vector<Hash> &blk_hashes,
+  void update(const uint64_t blk_seq, const std::vector<std::string> &blk_hashes,
       const std::string& prev_commit_seq, std::string* root_key,
       std::string* root_hash);
   Proof getProof(const std::string& commit_seq, const std::string& root_key,
diff --git a/ledger/ledgerdb/mpt/trie.cc b/ledger/ledgerdb/mpt/trie.cc
index 3a6c1dc..1bdd3c0 100644
--- a/ledger/ledgerdb/mpt/trie.cc
+++ b/ledger/ledgerdb/mpt/trie.cc
@@ -380,6 +380,7 @@ MPTProof Trie::GetProof(const std::string& key) const {
   MPTProof proof;
   std::string encoded_key = MPTConfig::KeybytesToHex(key);
   TryGetProof(root_node_->chunk(), encoded_key, 0, &proof);
+  //std::cout << "blocks " << proof.MapSize() << std::endl;
   return proof;
 }
 
diff --git a/ledger/qldb/qldb.cc b/ledger/qldb/qldb.cc
index f62a5f1..b6c94e4 100644
--- a/ledger/qldb/qldb.cc
+++ b/ledger/qldb/qldb.cc
@@ -11,6 +11,7 @@ namespace qldb {
 bool QLProofResult::Verify(const Hash digest) {
   auto doc = Document::Encode(data.key, data.val, addr, meta);
   auto doc_hash = doc.hash();
+  //std::cout << "verify height " << proof.size() << std::endl;
   for (size_t i = 0; i < proof.size(); ++i) {
     if (proof[i].empty()) {
       std::unique_ptr<byte_t[]> node(
@@ -327,6 +328,7 @@ QLProofResult QLDB::getProof(const std::string& name, const uint64_t tip,
     curr_seq = pr_seq;
     doc_size /= 2;
   }
+  //std::cout << "block " << level << std::endl;
 
   level = 0;
   curr_seq = block_addr;
@@ -350,6 +352,7 @@ QLProofResult QLDB::getProof(const std::string& name, const uint64_t tip,
     curr_seq = pr_seq;
     latest /= 2;
   }
+  //std::cout << "height " << level << std::endl;
   return result;
 }
 
diff --git a/ledger/sqlledger/sqlldg_mt.cc b/ledger/sqlledger/sqlldg_mt.cc
index 1ad7359..15fb443 100644
--- a/ledger/sqlledger/sqlldg_mt.cc
+++ b/ledger/sqlledger/sqlldg_mt.cc
@@ -8,6 +8,7 @@ namespace sqlledger {
 void MerkleTree::Build(const std::string& mtid,
     const std::vector<std::string> &leaves, std::string* toplevel,
     std::string* root_hash) {
+  if (leaves.size() == 0) return;
   // commit leaf hashes
   int level = 0;
   std::vector<Hash> level_hashes;
@@ -92,6 +93,7 @@ bool Proof::Verify() const {
       calc_hash = Hash::ComputeFrom(calc_hash.value(), Hash::kByteLength);
     }
   }
+  //std::cout << "tree height " << proof.size() << std::endl;
   if (calc_hash == Hash::FromBase32(digest)) return true;
   return false;
 }
diff --git a/ledger/sqlledger/sqlledger.cc b/ledger/sqlledger/sqlledger.cc
index 0aa2a4d..a2d408e 100644
--- a/ledger/sqlledger/sqlledger.cc
+++ b/ledger/sqlledger/sqlledger.cc
@@ -26,7 +26,9 @@ void SQLLedger::updateLedger(int timeout) {
       continue;
     }
 
-    std::unique_ptr<std::map<uint64_t, std::vector<std::string>>>   new_txns;
+    timeval t0, t1;
+    gettimeofday(&t0, NULL);
+    std::unique_ptr<std::map<uint64_t, std::vector<std::string>>> new_txns;
     uint64_t new_blk_seq;
     {
       boost::unique_lock<boost::shared_mutex> blockseqlock(lock_);
@@ -37,9 +39,11 @@ void SQLLedger::updateLedger(int timeout) {
     }
 
     int txn_seq = 0;
+    long nkey = 0;
     std::vector<std::string> leaves;
     for (auto& entry : *(new_txns.get())) {
       std::string txnid = "txn" + std::to_string(entry.first);
+      nkey += entry.second.size();
 
       // build merkle txn merkle tree
       std::string level, txnroot;
@@ -69,6 +73,10 @@ void SQLLedger::updateLedger(int timeout) {
     std::string newhash = Hash::ComputeFrom(newblock).ToBase32();
     db_.Put(block_key, newblock);
     db_.Put("digest", newhash + "|" + std::to_string(new_blk_seq));
+
+    gettimeofday(&t1, NULL);
+    auto lat = (t1.tv_sec - t0.tv_sec)*1000000 + t1.tv_usec - t0.tv_usec;
+    //std::cerr << "persist " << lat << " " << nkey << " " << new_txns->size() << std::endl;
   }
 }
 
@@ -186,6 +194,7 @@ SQLLedgerProof SQLLedger::getProof(const std::string& key,
   // get block
   int level;
   std::string blkroot;
+  //std::cout << "blocks " << (tip - block_addr) << std::endl;
   for (size_t i = block_addr; i <= tip; ++i) {
     std::string block;
     db_.Get("blk" + std::to_string(i), &block);
@@ -214,25 +223,39 @@ SQLLedgerProof SQLLedger::getProof(const std::string& key,
   proof.blk_proof.value = txn;
   proof.txn_proof = mt_->GetProof("txn" + txnid, txnrootlevel, docseq);
   proof.txn_proof.value = doc;
+  //std::cout << "height " << level + txnrootlevel << std::endl;
   return proof;
 }
 
 bool SQLLedgerProof::Verify() {
   std::string target = digest;
   std::string blk_mt_root;
+  int blocks = 0;
   for (auto it = blks.rbegin(); it != blks.rend(); ++it) {
-    if (Hash::ComputeFrom(*it).ToBase32().compare(target) != 0) return false;
+    if (Hash::ComputeFrom(*it).ToBase32().compare(target) != 0) {
+      //std::cout << "verification failed at block" << std::endl;
+    }
     auto block_info = Utils::splitBy(*it, '|');
     target = block_info[0];
     blk_mt_root = block_info[1];
+    blocks++;
   }
+  //std::cout << "block " << blks.size() << " " << blocks << std::endl;
 
-  if (blk_mt_root.compare(blk_proof.digest) != 0) return false;
-  if (!blk_proof.Verify()) return false;
+  if (blk_mt_root.compare(blk_proof.digest) != 0) {
+    //std::cout << "verification failed at txn mt root" << std::endl;
+  }
+  std::cout << "txn ";
+  if (!blk_proof.Verify()) {
+    //std::cout << "verification failed at txn tree" << std::endl;
+  }
 
   auto txn_info = Utils::splitBy(blk_proof.value, '|');
   std::string txn_mt_root = txn_info[3];
-  if (txn_mt_root != txn_proof.digest) return false;
+  if (txn_mt_root != txn_proof.digest) {
+    //std::cout << "verification failed at row mt root" << std::endl;
+  }
+  //std::cout << "row ";
   return txn_proof.Verify();
 }
 
diff --git a/test/ledger/test_merkle_tree.cc b/test/ledger/test_merkle_tree.cc
index b743c45..5cc60b1 100644
--- a/test/ledger/test_merkle_tree.cc
+++ b/test/ledger/test_merkle_tree.cc
@@ -8,7 +8,7 @@
 #include "ledger/ledgerdb/merkletree.h"
 
 TEST(MERKLETREE, TPS) {
-  std::vector<ledgebase::Hash> hashes;
+  std::vector<std::string> hashes;
 
   ledgebase::DB db;
   db.Open("testdb");
@@ -19,7 +19,7 @@ TEST(MERKLETREE, TPS) {
   timeval t0, t1;
   gettimeofday(&t0, NULL);
   for (size_t i = 0; i < 100000; ++i) {
-    hashes.emplace_back(ledgebase::Hash::ComputeFrom(std::to_string(i)));
+    hashes.emplace_back(ledgebase::Hash::ComputeFrom(std::to_string(i)).ToBase32());
 
     if (hashes.size() == 100) {
       std::string root_key, root_hash;