sync with upstream @ 21409451

Check the NEWS file for details of what changed.

git-svn-id: https://leveldb.googlecode.com/svn/trunk@28 62dab493-f737-651d-591e-8d6aee1b9529

Author: [email protected]

NEWS

@@ -0,0 +1,17 @@
+Release 1.2 2011-05-16
+----------------------
+
+Fixes for larger databases (tested up to one billion 100-byte entries,
+i.e., ~100GB).
+
+(1) Place hard limit on number of level-0 files.  This fixes errors
+of the form "too many open files".
+
+(2) Fixed memtable management.  Before the fix, a heavy write burst
+could cause unbounded memory usage.
+
+A fix for a logging bug where the reader would incorrectly complain
+about corruption.
+
+Allow public access to WriteBatch contents so that users can easily
+wrap a DB.

db/db_bench.cc

@@ -24,9 +24,10 @@
 //      overwrite     -- overwrite N values in random key order in async mode
 //      fillsync      -- write N/100 values in random key order in sync mode
 //      fill100K      -- write N/1000 100K values in random order in async mode
-//      readseq       -- read N values sequentially
-//      readreverse   -- read N values in reverse order
-//      readrandom    -- read N values in random order
+//      readseq       -- read N times sequentially
+//      readreverse   -- read N times in reverse order
+//      readrandom    -- read N times in random order
+//      readhot       -- read N times in random order from 1% section of DB
 //      crc32c        -- repeated crc32c of 4K of data
 //   Meta operations:
 //      compact     -- Compact the entire DB
@@ -54,6 +55,9 @@ static const char* FLAGS_benchmarks =
 // Number of key/values to place in database
 static int FLAGS_num = 1000000;
 
+// Number of read operations to do.  If negative, do FLAGS_num reads.
+static int FLAGS_reads = -1;
+
 // Size of each value
 static int FLAGS_value_size = 100;
 
@@ -72,6 +76,14 @@ static int FLAGS_write_buffer_size = 0;
 // Negative means use default settings.
 static int FLAGS_cache_size = -1;
 
+// Maximum number of files to keep open at the same time (use default if == 0)
+static int FLAGS_open_files = 0;
+
+// If true, do not destroy the existing database.  If you set this
+// flag and also specify a benchmark that wants a fresh database, that
+// benchmark will fail.
+static bool FLAGS_use_existing_db = false;
+
 namespace leveldb {
 
 // Helper for quickly generating random data.
@@ -126,6 +138,7 @@ class Benchmark {
   Cache* cache_;
   DB* db_;
   int num_;
+  int reads_;
   int heap_counter_;
   double start_;
   double last_op_finish_;
@@ -298,6 +311,7 @@ class Benchmark {
   : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL),
     db_(NULL),
     num_(FLAGS_num),
+    reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
     heap_counter_(0),
     bytes_(0),
     rand_(301) {
@@ -308,7 +322,9 @@ class Benchmark {
         Env::Default()->DeleteFile("/tmp/dbbench/" + files[i]);
       }
     }
-    DestroyDB("/tmp/dbbench", Options());
+    if (!FLAGS_use_existing_db) {
+      DestroyDB("/tmp/dbbench", Options());
+    }
   }
 
   ~Benchmark() {
@@ -355,11 +371,13 @@ class Benchmark {
         ReadReverse();
       } else if (name == Slice("readrandom")) {
         ReadRandom();
+      } else if (name == Slice("readhot")) {
+        ReadHot();
       } else if (name == Slice("readrandomsmall")) {
-        int n = num_;
-        num_ /= 1000;
+        int n = reads_;
+        reads_ /= 1000;
         ReadRandom();
-        num_ = n;
+        reads_ = n;
       } else if (name == Slice("compact")) {
         Compact();
       } else if (name == Slice("crc32c")) {
@@ -449,7 +467,7 @@ class Benchmark {
   void Open() {
     assert(db_ == NULL);
     Options options;
-    options.create_if_missing = true;
+    options.create_if_missing = !FLAGS_use_existing_db;
     options.block_cache = cache_;
     options.write_buffer_size = FLAGS_write_buffer_size;
     Status s = DB::Open(options, "/tmp/dbbench", &db_);
@@ -462,6 +480,10 @@ class Benchmark {
   void Write(const WriteOptions& options, Order order, DBState state,
              int num_entries, int value_size, int entries_per_batch) {
     if (state == FRESH) {
+      if (FLAGS_use_existing_db) {
+        message_ = "skipping (--use_existing_db is true)";
+        return;
+      }
       delete db_;
       db_ = NULL;
       DestroyDB("/tmp/dbbench", Options());
@@ -499,7 +521,7 @@ class Benchmark {
   void ReadSequential() {
     Iterator* iter = db_->NewIterator(ReadOptions());
     int i = 0;
-    for (iter->SeekToFirst(); i < num_ && iter->Valid(); iter->Next()) {
+    for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
       bytes_ += iter->key().size() + iter->value().size();
       FinishedSingleOp();
       ++i;
@@ -510,7 +532,7 @@ class Benchmark {
   void ReadReverse() {
     Iterator* iter = db_->NewIterator(ReadOptions());
     int i = 0;
-    for (iter->SeekToLast(); i < num_ && iter->Valid(); iter->Prev()) {
+    for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) {
       bytes_ += iter->key().size() + iter->value().size();
       FinishedSingleOp();
       ++i;
@@ -521,7 +543,7 @@ class Benchmark {
   void ReadRandom() {
     ReadOptions options;
     std::string value;
-    for (int i = 0; i < num_; i++) {
+    for (int i = 0; i < reads_; i++) {
       char key[100];
       const int k = rand_.Next() % FLAGS_num;
       snprintf(key, sizeof(key), "%016d", k);
@@ -530,6 +552,19 @@ class Benchmark {
     }
   }
 
+  void ReadHot() {
+    ReadOptions options;
+    std::string value;
+    const int range = (FLAGS_num + 99) / 100;
+    for (int i = 0; i < reads_; i++) {
+      char key[100];
+      const int k = rand_.Next() % range;
+      snprintf(key, sizeof(key), "%016d", k);
+      db_->Get(options, key, &value);
+      FinishedSingleOp();
+    }
+  }
+
   void Compact() {
     DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
     dbi->TEST_CompactMemTable();
@@ -582,6 +617,8 @@ class Benchmark {
 
 int main(int argc, char** argv) {
   FLAGS_write_buffer_size = leveldb::Options().write_buffer_size;
+  FLAGS_open_files = leveldb::Options().max_open_files;
+
   for (int i = 1; i < argc; i++) {
     double d;
     int n;
@@ -593,14 +630,21 @@ int main(int argc, char** argv) {
     } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
                (n == 0 || n == 1)) {
       FLAGS_histogram = n;
+    } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
+               (n == 0 || n == 1)) {
+      FLAGS_use_existing_db = n;
     } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
       FLAGS_num = n;
+    } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
+      FLAGS_reads = n;
     } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
       FLAGS_value_size = n;
     } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
       FLAGS_write_buffer_size = n;
     } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
       FLAGS_cache_size = n;
+    } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {
+      FLAGS_open_files = n;
     } else {
       fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
       exit(1);