From 426dcc4ffd12d430b505d18b081e089d9925ed37 Mon Sep 17 00:00:00 2001 From: speedplane Date: Wed, 11 Nov 2015 21:52:38 -0500 Subject: [PATCH] Add support for Google App Engine to the SemiDBM. This commit will allow GAE users to create a semidbm database offline, then upload it to GAE, and then use that database in read-only mode within GAE. Google App Engine runs python in a sandbox, and does not support os file descriptor access that is used by semidbm. It also does not support opening files for writing (but it does support read only). This commit changes those function calls to the standard python open, read, and write functions, which are supported in the GAE sandbox. Further, if the semidbm database is opened in read-only mode, then the appropriate flags are used which work on GAE. --- semidbm/compat.py | 6 +++-- semidbm/db.py | 59 +++++++++++++++++++++++++---------------------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/semidbm/compat.py b/semidbm/compat.py index 3785353..914a581 100644 --- a/semidbm/compat.py +++ b/semidbm/compat.py @@ -15,9 +15,11 @@ str_type = str -DATA_OPEN_FLAGS = os.O_RDWR | os.O_CREAT | os.O_APPEND +DATA_OPEN_FLAGS = "r+" +DATA_OPEN_FLAGS_READ = "r" if sys.platform.startswith('win'): # On windows we need to specify that we should be # reading the file as a binary file so it doesn't # change any line ending characters. - DATA_OPEN_FLAGS = DATA_OPEN_FLAGS | os.O_BINARY + DATA_OPEN_FLAGS += "b" + DATA_OPEN_FLAGS_READ += "b" diff --git a/semidbm/db.py b/semidbm/db.py index 5530508..62b1f07 100644 --- a/semidbm/db.py +++ b/semidbm/db.py @@ -18,6 +18,8 @@ _open = compat.file_open +SEEK_SET = 0 +SEEK_END = 2 class _SemiDBM(object): """ @@ -26,6 +28,8 @@ class _SemiDBM(object): does not exist it will be created. """ + open_flags = compat.DATA_OPEN_FLAGS + def __init__(self, dbdir, renamer, data_loader=None, verify_checksums=False): self._renamer = renamer @@ -36,7 +40,6 @@ def __init__(self, dbdir, renamer, data_loader=None, self._index = None self._data_fd = None self._verify_checksums = verify_checksums - self._current_offset = 0 self._load_db() def _create_db_dir(self): @@ -46,8 +49,7 @@ def _create_db_dir(self): def _load_db(self): self._create_db_dir() self._index = self._load_index(self._data_filename) - self._data_fd = os.open(self._data_filename, compat.DATA_OPEN_FLAGS) - self._current_offset = os.lseek(self._data_fd, 0, os.SEEK_END) + self._data_fd = _open(self._data_filename, self.open_flags) def _load_index(self, filename): # This method is only used upon instantiation to populate @@ -85,25 +87,22 @@ def _load_index_from_fileobj(self, filename): index[key_name] = (offset, size) return index - def __getitem__(self, key, read=os.read, lseek=os.lseek, - seek_set=os.SEEK_SET, str_type=compat.str_type, - isinstance=isinstance): + def __getitem__(self, key, str_type = compat.str_type, + isinstance = isinstance): if isinstance(key, str_type): key = key.encode('utf-8') offset, size = self._index[key] - lseek(self._data_fd, offset, seek_set) - if not self._verify_checksums: - return read(self._data_fd, size) - else: + self._data_fd.seek(offset, SEEK_SET) + value = self._data_fd.read(size) + if self._verify_checksums: # Checksum is at the end of the value. - data = read(self._data_fd, size + 4) - return self._verify_checksum_data(key, data) + return self._verify_checksum_data(key, value, self._data_fd.read(4)) + return value - def _verify_checksum_data(self, key, data): + def _verify_checksum_data(self, key, value, chk): # key is the bytes of the key, # data is the bytes of the value + 4 byte checksum at the end. - value = data[:-4] - expected = struct.unpack('!I', data[-4:])[0] + expected = struct.unpack('!I', chk)[0] actual = crc32(key) actual = crc32(value, actual) if actual & 0xffffffff != expected: @@ -111,7 +110,7 @@ def _verify_checksum_data(self, key, data): "Corrupt data detected: invalid checksum for key %s" % key) return value - def __setitem__(self, key, value, len=len, crc32=crc32, write=os.write, + def __setitem__(self, key, value, len=len, crc32=crc32, str_type=compat.str_type, pack=struct.pack, isinstance=isinstance): if isinstance(key, str_type): @@ -119,6 +118,7 @@ def __setitem__(self, key, value, len=len, crc32=crc32, write=os.write, if isinstance(value, str_type): value = value.encode('utf-8') # Write the new data out at the end of the file. + self._data_fd.seek(0, SEEK_END) # Format is # 4 bytes 4bytes 4bytes # @@ -130,16 +130,15 @@ def __setitem__(self, key, value, len=len, crc32=crc32, write=os.write, checksum = pack('!I', crc32(keyval) & 0xffffffff) blob = keyval_size + keyval + checksum - write(self._data_fd, blob) - # Update the in memory index. - self._index[key] = (self._current_offset + 8 + key_size, - val_size) - self._current_offset += len(blob) + # Update the in memory index to point to the val. + self._index[key] = self._data_fd.tell() + 8 + key_size, val_size + # Write the blob + self._data_fd.write(blob) def __contains__(self, key): return key in self._index - def __delitem__(self, key, len=len, write=os.write, deleted=_DELETED, + def __delitem__(self, key, len=len, deleted=_DELETED, str_type=compat.str_type, isinstance=isinstance, crc32=crc32, pack=struct.pack): if isinstance(key, str_type): @@ -148,9 +147,8 @@ def __delitem__(self, key, len=len, write=os.write, deleted=_DELETED, crc = pack('!I', crc32(key) & 0xffffffff) blob = key_size + key + crc - write(self._data_fd, blob) + self._data_fd.write(blob) del self._index[key] - self._current_offset += len(blob) def __iter__(self): for key in self._index: @@ -181,7 +179,8 @@ def close(self, compact=False): if compact: self.compact() self.sync() - os.close(self._data_fd) + self._data_fd.close() + self._data_fd = None def sync(self): """Sync the db to disk. @@ -196,7 +195,7 @@ def sync(self): """ # The files are opened unbuffered so we don't technically # need to flush the file objects. - os.fsync(self._data_fd) + self._data_fd.flush() def compact(self): """Compact the db to reduce space. @@ -223,7 +222,8 @@ def compact(self): new_db[key] = self[key] new_db.sync() new_db.close() - os.close(self._data_fd) + self._data_fd.close() + self._data_fd = None self._renamer(new_db._data_filename, self._data_filename) os.rmdir(new_db._dbdir) # The index is already compacted so we don't need to compact it. @@ -231,6 +231,8 @@ def compact(self): class _SemiDBMReadOnly(_SemiDBM): + open_flags = compat.DATA_OPEN_FLAGS_READ + def __delitem__(self, key): self._method_not_allowed('delitem') @@ -247,7 +249,8 @@ def _method_not_allowed(self, method_name): raise DBMError("Can't %s: db opened in read only mode." % method_name) def close(self, compact=False): - os.close(self._data_fd) + self._data_fd.close() + self._data_fd = None class _SemiDBMReadWrite(_SemiDBM):