A number of smaller fixes and performance improvements:

- Implemented Get() directly instead of building on top of a full
  merging iterator stack.  This speeds up the "readrandom" benchmark
  by up to 15-30%.

- Fixed an opensource compilation problem.
  Added --db=<name> flag to control where the database is placed.

- Automatically compact a file when we have done enough
  overlapping seeks to that file.

- Fixed a performance bug where we would read from at least one
  file in a level even if none of the files overlapped the key
  being read.

- Makefile fix for Mac OSX installations that have XCode 4 without XCode 3.

- Unified the two occurrences of binary search in a file-list
  into one routine.

- Found and fixed a bug where we would unnecessarily search the
  last file when looking for a key larger than all data in the
  level.

- A fix to avoid the need for trivial move compactions and
  therefore gets rid of two out of five syncs in "fillseq".

- Removed the MANIFEST file write when switching to a new
  memtable/log-file for a 10-20% improvement on fill speed on ext4.

- Adding a SNAPPY setting in the Makefile for folks who have
  Snappy installed. Snappy compresses values and speeds up writes.



git-svn-id: https://leveldb.googlecode.com/svn/trunk@32 62dab493-f737-651d-591e-8d6aee1b9529

Author: [email protected]

Makefile

@@ -28,9 +28,22 @@ PLATFORM_CFLAGS = -DLEVELDB_PLATFORM_POSIX -std=c++0x
 PORT_MODULE = port_posix.o
 endif # UNAME
 
-CFLAGS = -c -I. -I./include $(PLATFORM_CFLAGS) $(OPT)
+# Set 'SNAPPY' to 1 if you have the Snappy compression library
+# installed and want to enable its use in LevelDB
+# (see http://code.google.com/p/snappy/)
+SNAPPY=0
+
+ifeq ($(SNAPPY), 0)
+SNAPPY_CFLAGS=
+SNAPPY_LDFLAGS=
+else
+SNAPPY_CFLAGS=-DSNAPPY
+SNAPPY_LDFLAGS=-lsnappy
+endif
 
-LDFLAGS=-lpthread
+CFLAGS = -c -I. -I./include $(PLATFORM_CFLAGS) $(OPT) $(SNAPPY_CFLAGS)
+
+LDFLAGS=-lpthread $(SNAPPY_LDFLAGS)
 
 LIBOBJECTS = \
 	./db/builder.o \
@@ -85,6 +98,7 @@ TESTS = \
 	skiplist_test \
 	table_test \
 	version_edit_test \
+	version_set_test \
 	write_batch_test
 
 PROGRAMS = db_bench $(TESTS)
@@ -151,17 +165,23 @@ skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CC) $(LDFLAGS) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
 
+version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CC) $(LDFLAGS) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
+
 write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CC) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
 
 ifeq ($(PLATFORM), IOS)
 # For iOS, create universal object files to be used on both the simulator and
 # a device.
+SIMULATORROOT=/Developer/Platforms/iPhoneSimulator.platform/Developer
+DEVICEROOT=/Developer/Platforms/iPhoneOS.platform/Developer
+IOSVERSION=$(shell defaults read /Developer/Platforms/iPhoneOS.platform/version CFBundleShortVersionString)
 .cc.o:
 	mkdir -p ios-x86/$(dir $@)
-	$(CC) $(CFLAGS) -isysroot /Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator4.3.sdk -arch i686 $< -o ios-x86/$@
+	$(SIMULATORROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 $< -o ios-x86/$@
 	mkdir -p ios-arm/$(dir $@)
-	$(CC) $(CFLAGS) -isysroot /Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS4.3.sdk -arch armv6 -arch armv7 $< -o ios-arm/$@
+	$(DEVICEROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 $< -o ios-arm/$@
 	lipo ios-x86/$@ ios-arm/$@ -create -output $@
 else
 .cc.o:

TODO

@@ -8,7 +8,6 @@ db
   object stores, etc. can be done in the background anyway, so
   probably not that important.
 
-api changes:
-- Make it wrappable
-
-Faster Get implementation
+After a range is completely deleted, what gets rid of the
+corresponding files if we do no future changes to that range.  Make
+the conditions for triggering compactions fire in more situations?

db/builder.cc

@@ -19,8 +19,7 @@ Status BuildTable(const std::string& dbname,
                   const Options& options,
                   TableCache* table_cache,
                   Iterator* iter,
-                  FileMetaData* meta,
-                  VersionEdit* edit) {
+                  FileMetaData* meta) {
   Status s;
   meta->file_size = 0;
   iter->SeekToFirst();
@@ -79,8 +78,7 @@ Status BuildTable(const std::string& dbname,
   }
 
   if (s.ok() && meta->file_size > 0) {
-    edit->AddFile(0, meta->number, meta->file_size,
-                  meta->smallest, meta->largest);
+    // Keep it
   } else {
     env->DeleteFile(fname);
   }

db/builder.h

@@ -19,17 +19,15 @@ class VersionEdit;
 
 // Build a Table file from the contents of *iter.  The generated file
 // will be named according to meta->number.  On success, the rest of
-// *meta will be filled with metadata about the generated table, and
-// the file information will be added to *edit.  If no data is present
-// in *iter, meta->file_size will be set to zero, and no Table file
-// will be produced.
+// *meta will be filled with metadata about the generated table.
+// If no data is present in *iter, meta->file_size will be set to
+// zero, and no Table file will be produced.
 extern Status BuildTable(const std::string& dbname,
                          Env* env,
                          const Options& options,
                          TableCache* table_cache,
                          Iterator* iter,
-                         FileMetaData* meta,
-                         VersionEdit* edit);
+                         FileMetaData* meta);
 
 }
 

db/corruption_test.cc

@@ -27,13 +27,12 @@ static const int kValueSize = 1000;
 class CorruptionTest {
  public:
   test::ErrorEnv env_;
-  Random rnd_;
   std::string dbname_;
   Cache* tiny_cache_;
   Options options_;
   DB* db_;
 
-  CorruptionTest() : rnd_(test::RandomSeed()) {
+  CorruptionTest() {
     tiny_cache_ = NewLRUCache(100);
     options_.env = &env_;
     dbname_ = test::TmpDir() + "/db_test";
@@ -122,15 +121,17 @@ class CorruptionTest {
     ASSERT_OK(env_.GetChildren(dbname_, &filenames));
     uint64_t number;
     FileType type;
-    std::vector<std::string> candidates;
+    std::string fname;
+    int picked_number = -1;
     for (int i = 0; i < filenames.size(); i++) {
       if (ParseFileName(filenames[i], &number, &type) &&
-          type == filetype) {
-        candidates.push_back(dbname_ + "/" + filenames[i]);
+          type == filetype &&
+          int(number) > picked_number) {  // Pick latest file
+        fname = dbname_ + "/" + filenames[i];
+        picked_number = number;
       }
     }
-    ASSERT_TRUE(!candidates.empty()) << filetype;
-    std::string fname = candidates[rnd_.Uniform(candidates.size())];
+    ASSERT_TRUE(!fname.empty()) << filetype;
 
     struct stat sbuf;
     if (stat(fname.c_str(), &sbuf) != 0) {
@@ -239,8 +240,6 @@ TEST(CorruptionTest, TableFileIndexData) {
   Build(10000);  // Enough to build multiple Tables
   DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
   dbi->TEST_CompactMemTable();
-  dbi->TEST_CompactRange(0, "", "~");
-  dbi->TEST_CompactRange(1, "", "~");
 
   Corrupt(kTableFile, -2000, 500);
   Reopen();
@@ -296,26 +295,32 @@ TEST(CorruptionTest, CompactionInputError) {
   Build(10);
   DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
   dbi->TEST_CompactMemTable();
-  ASSERT_EQ(1, Property("leveldb.num-files-at-level0"));
+  const int last = config::kNumLevels - 1;
+  ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
 
   Corrupt(kTableFile, 100, 1);
   Check(9, 9);
 
   // Force compactions by writing lots of values
   Build(10000);
   Check(10000, 10000);
-  dbi->TEST_CompactRange(0, "", "~");
-  ASSERT_EQ(0, Property("leveldb.num-files-at-level0"));
 }
 
 TEST(CorruptionTest, CompactionInputErrorParanoid) {
   Options options;
   options.paranoid_checks = true;
   options.write_buffer_size = 1048576;
   Reopen(&options);
+  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+
+  // Fill levels >= 1 so memtable compaction outputs to level 1
+  for (int level = 1; level < config::kNumLevels; level++) {
+    dbi->Put(WriteOptions(), "", "begin");
+    dbi->Put(WriteOptions(), "~", "end");
+    dbi->TEST_CompactMemTable();
+  }
 
   Build(10);
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
   dbi->TEST_CompactMemTable();
   ASSERT_EQ(1, Property("leveldb.num-files-at-level0"));
 

db/db_bench.cc

@@ -86,6 +86,9 @@ static int FLAGS_open_files = 0;
 // benchmark will fail.
 static bool FLAGS_use_existing_db = false;
 
+// Use the db with the following name.
+static const char* FLAGS_db = "/tmp/dbbench";
+
 namespace leveldb {
 
 // Helper for quickly generating random data.
@@ -318,14 +321,14 @@ class Benchmark {
     bytes_(0),
     rand_(301) {
     std::vector<std::string> files;
-    Env::Default()->GetChildren("/tmp/dbbench", &files);
+    Env::Default()->GetChildren(FLAGS_db, &files);
     for (int i = 0; i < files.size(); i++) {
       if (Slice(files[i]).starts_with("heap-")) {
-        Env::Default()->DeleteFile("/tmp/dbbench/" + files[i]);
+        Env::Default()->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
       }
     }
     if (!FLAGS_use_existing_db) {
-      DestroyDB("/tmp/dbbench", Options());
+      DestroyDB(FLAGS_db, Options());
     }
   }
 
@@ -364,7 +367,7 @@ class Benchmark {
         Write(write_options, RANDOM, EXISTING, num_, FLAGS_value_size, 1);
       } else if (name == Slice("fillsync")) {
         write_options.sync = true;
-        Write(write_options, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1);
+        Write(write_options, RANDOM, FRESH, num_ / 1000, FLAGS_value_size, 1);
       } else if (name == Slice("fill100K")) {
         Write(write_options, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1);
       } else if (name == Slice("readseq")) {
@@ -490,7 +493,7 @@ class Benchmark {
     options.create_if_missing = !FLAGS_use_existing_db;
     options.block_cache = cache_;
     options.write_buffer_size = FLAGS_write_buffer_size;
-    Status s = DB::Open(options, "/tmp/dbbench", &db_);
+    Status s = DB::Open(options, FLAGS_db, &db_);
     if (!s.ok()) {
       fprintf(stderr, "open error: %s\n", s.ToString().c_str());
       exit(1);
@@ -506,7 +509,7 @@ class Benchmark {
       }
       delete db_;
       db_ = NULL;
-      DestroyDB("/tmp/dbbench", Options());
+      DestroyDB(FLAGS_db, Options());
       Open();
       Start();  // Do not count time taken to destroy/open
     }
@@ -617,7 +620,7 @@ class Benchmark {
 
   void HeapProfile() {
     char fname[100];
-    snprintf(fname, sizeof(fname), "/tmp/dbbench/heap-%04d", ++heap_counter_);
+    snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_);
     WritableFile* file;
     Status s = Env::Default()->NewWritableFile(fname, &file);
     if (!s.ok()) {
@@ -665,6 +668,8 @@ int main(int argc, char** argv) {
       FLAGS_cache_size = n;
     } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {
       FLAGS_open_files = n;
+    } else if (strncmp(argv[i], "--db=", 5) == 0) {
+      FLAGS_db = argv[i] + 5;
     } else {
       fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
       exit(1);