From 3725179736e2b5372664163470e7ef3dc76529a4 Mon Sep 17 00:00:00 2001 From: Sfan5 Date: Mon, 9 Sep 2013 22:46:18 +0200 Subject: Use system-wide LevelDB instead of bundled one --- src/leveldb/helpers/memenv/memenv.cc | 384 ------------------------------ src/leveldb/helpers/memenv/memenv.h | 20 -- src/leveldb/helpers/memenv/memenv_test.cc | 232 ------------------ 3 files changed, 636 deletions(-) delete mode 100644 src/leveldb/helpers/memenv/memenv.cc delete mode 100644 src/leveldb/helpers/memenv/memenv.h delete mode 100644 src/leveldb/helpers/memenv/memenv_test.cc (limited to 'src/leveldb/helpers') diff --git a/src/leveldb/helpers/memenv/memenv.cc b/src/leveldb/helpers/memenv/memenv.cc deleted file mode 100644 index 5879de121..000000000 --- a/src/leveldb/helpers/memenv/memenv.cc +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include "helpers/memenv/memenv.h" - -#include "leveldb/env.h" -#include "leveldb/status.h" -#include "port/port.h" -#include "util/mutexlock.h" -#include -#include -#include -#include - -namespace leveldb { - -namespace { - -class FileState { - public: - // FileStates are reference counted. The initial reference count is zero - // and the caller must call Ref() at least once. - FileState() : refs_(0), size_(0) {} - - // Increase the reference count. - void Ref() { - MutexLock lock(&refs_mutex_); - ++refs_; - } - - // Decrease the reference count. Delete if this is the last reference. - void Unref() { - bool do_delete = false; - - { - MutexLock lock(&refs_mutex_); - --refs_; - assert(refs_ >= 0); - if (refs_ <= 0) { - do_delete = true; - } - } - - if (do_delete) { - delete this; - } - } - - uint64_t Size() const { return size_; } - - Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { - if (offset > size_) { - return Status::IOError("Offset greater than file size."); - } - const uint64_t available = size_ - offset; - if (n > available) { - n = available; - } - if (n == 0) { - *result = Slice(); - return Status::OK(); - } - - size_t block = offset / kBlockSize; - size_t block_offset = offset % kBlockSize; - - if (n <= kBlockSize - block_offset) { - // The requested bytes are all in the first block. - *result = Slice(blocks_[block] + block_offset, n); - return Status::OK(); - } - - size_t bytes_to_copy = n; - char* dst = scratch; - - while (bytes_to_copy > 0) { - size_t avail = kBlockSize - block_offset; - if (avail > bytes_to_copy) { - avail = bytes_to_copy; - } - memcpy(dst, blocks_[block] + block_offset, avail); - - bytes_to_copy -= avail; - dst += avail; - block++; - block_offset = 0; - } - - *result = Slice(scratch, n); - return Status::OK(); - } - - Status Append(const Slice& data) { - const char* src = data.data(); - size_t src_len = data.size(); - - while (src_len > 0) { - size_t avail; - size_t offset = size_ % kBlockSize; - - if (offset != 0) { - // There is some room in the last block. - avail = kBlockSize - offset; - } else { - // No room in the last block; push new one. - blocks_.push_back(new char[kBlockSize]); - avail = kBlockSize; - } - - if (avail > src_len) { - avail = src_len; - } - memcpy(blocks_.back() + offset, src, avail); - src_len -= avail; - src += avail; - size_ += avail; - } - - return Status::OK(); - } - - private: - // Private since only Unref() should be used to delete it. - ~FileState() { - for (std::vector::iterator i = blocks_.begin(); i != blocks_.end(); - ++i) { - delete [] *i; - } - } - - // No copying allowed. - FileState(const FileState&); - void operator=(const FileState&); - - port::Mutex refs_mutex_; - int refs_; // Protected by refs_mutex_; - - // The following fields are not protected by any mutex. They are only mutable - // while the file is being written, and concurrent access is not allowed - // to writable files. - std::vector blocks_; - uint64_t size_; - - enum { kBlockSize = 8 * 1024 }; -}; - -class SequentialFileImpl : public SequentialFile { - public: - explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) { - file_->Ref(); - } - - ~SequentialFileImpl() { - file_->Unref(); - } - - virtual Status Read(size_t n, Slice* result, char* scratch) { - Status s = file_->Read(pos_, n, result, scratch); - if (s.ok()) { - pos_ += result->size(); - } - return s; - } - - virtual Status Skip(uint64_t n) { - if (pos_ > file_->Size()) { - return Status::IOError("pos_ > file_->Size()"); - } - const size_t available = file_->Size() - pos_; - if (n > available) { - n = available; - } - pos_ += n; - return Status::OK(); - } - - private: - FileState* file_; - size_t pos_; -}; - -class RandomAccessFileImpl : public RandomAccessFile { - public: - explicit RandomAccessFileImpl(FileState* file) : file_(file) { - file_->Ref(); - } - - ~RandomAccessFileImpl() { - file_->Unref(); - } - - virtual Status Read(uint64_t offset, size_t n, Slice* result, - char* scratch) const { - return file_->Read(offset, n, result, scratch); - } - - private: - FileState* file_; -}; - -class WritableFileImpl : public WritableFile { - public: - WritableFileImpl(FileState* file) : file_(file) { - file_->Ref(); - } - - ~WritableFileImpl() { - file_->Unref(); - } - - virtual Status Append(const Slice& data) { - return file_->Append(data); - } - - virtual Status Close() { return Status::OK(); } - virtual Status Flush() { return Status::OK(); } - virtual Status Sync() { return Status::OK(); } - - private: - FileState* file_; -}; - -class NoOpLogger : public Logger { - public: - virtual void Logv(const char* format, va_list ap) { } -}; - -class InMemoryEnv : public EnvWrapper { - public: - explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { } - - virtual ~InMemoryEnv() { - for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ - i->second->Unref(); - } - } - - // Partial implementation of the Env interface. - virtual Status NewSequentialFile(const std::string& fname, - SequentialFile** result) { - MutexLock lock(&mutex_); - if (file_map_.find(fname) == file_map_.end()) { - *result = NULL; - return Status::IOError(fname, "File not found"); - } - - *result = new SequentialFileImpl(file_map_[fname]); - return Status::OK(); - } - - virtual Status NewRandomAccessFile(const std::string& fname, - RandomAccessFile** result) { - MutexLock lock(&mutex_); - if (file_map_.find(fname) == file_map_.end()) { - *result = NULL; - return Status::IOError(fname, "File not found"); - } - - *result = new RandomAccessFileImpl(file_map_[fname]); - return Status::OK(); - } - - virtual Status NewWritableFile(const std::string& fname, - WritableFile** result) { - MutexLock lock(&mutex_); - if (file_map_.find(fname) != file_map_.end()) { - DeleteFileInternal(fname); - } - - FileState* file = new FileState(); - file->Ref(); - file_map_[fname] = file; - - *result = new WritableFileImpl(file); - return Status::OK(); - } - - virtual bool FileExists(const std::string& fname) { - MutexLock lock(&mutex_); - return file_map_.find(fname) != file_map_.end(); - } - - virtual Status GetChildren(const std::string& dir, - std::vector* result) { - MutexLock lock(&mutex_); - result->clear(); - - for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ - const std::string& filename = i->first; - - if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' && - Slice(filename).starts_with(Slice(dir))) { - result->push_back(filename.substr(dir.size() + 1)); - } - } - - return Status::OK(); - } - - void DeleteFileInternal(const std::string& fname) { - if (file_map_.find(fname) == file_map_.end()) { - return; - } - - file_map_[fname]->Unref(); - file_map_.erase(fname); - } - - virtual Status DeleteFile(const std::string& fname) { - MutexLock lock(&mutex_); - if (file_map_.find(fname) == file_map_.end()) { - return Status::IOError(fname, "File not found"); - } - - DeleteFileInternal(fname); - return Status::OK(); - } - - virtual Status CreateDir(const std::string& dirname) { - return Status::OK(); - } - - virtual Status DeleteDir(const std::string& dirname) { - return Status::OK(); - } - - virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) { - MutexLock lock(&mutex_); - if (file_map_.find(fname) == file_map_.end()) { - return Status::IOError(fname, "File not found"); - } - - *file_size = file_map_[fname]->Size(); - return Status::OK(); - } - - virtual Status RenameFile(const std::string& src, - const std::string& target) { - MutexLock lock(&mutex_); - if (file_map_.find(src) == file_map_.end()) { - return Status::IOError(src, "File not found"); - } - - DeleteFileInternal(target); - file_map_[target] = file_map_[src]; - file_map_.erase(src); - return Status::OK(); - } - - virtual Status LockFile(const std::string& fname, FileLock** lock) { - *lock = new FileLock; - return Status::OK(); - } - - virtual Status UnlockFile(FileLock* lock) { - delete lock; - return Status::OK(); - } - - virtual Status GetTestDirectory(std::string* path) { - *path = "/test"; - return Status::OK(); - } - - virtual Status NewLogger(const std::string& fname, Logger** result) { - *result = new NoOpLogger; - return Status::OK(); - } - - private: - // Map from filenames to FileState objects, representing a simple file system. - typedef std::map FileSystem; - port::Mutex mutex_; - FileSystem file_map_; // Protected by mutex_. -}; - -} // namespace - -Env* NewMemEnv(Env* base_env) { - return new InMemoryEnv(base_env); -} - -} // namespace leveldb diff --git a/src/leveldb/helpers/memenv/memenv.h b/src/leveldb/helpers/memenv/memenv.h deleted file mode 100644 index 03b88de76..000000000 --- a/src/leveldb/helpers/memenv/memenv.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ -#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ - -namespace leveldb { - -class Env; - -// Returns a new environment that stores its data in memory and delegates -// all non-file-storage tasks to base_env. The caller must delete the result -// when it is no longer needed. -// *base_env must remain live while the result is in use. -Env* NewMemEnv(Env* base_env); - -} // namespace leveldb - -#endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ diff --git a/src/leveldb/helpers/memenv/memenv_test.cc b/src/leveldb/helpers/memenv/memenv_test.cc deleted file mode 100644 index a44310fed..000000000 --- a/src/leveldb/helpers/memenv/memenv_test.cc +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include "helpers/memenv/memenv.h" - -#include "db/db_impl.h" -#include "leveldb/db.h" -#include "leveldb/env.h" -#include "util/testharness.h" -#include -#include - -namespace leveldb { - -class MemEnvTest { - public: - Env* env_; - - MemEnvTest() - : env_(NewMemEnv(Env::Default())) { - } - ~MemEnvTest() { - delete env_; - } -}; - -TEST(MemEnvTest, Basics) { - uint64_t file_size; - WritableFile* writable_file; - std::vector children; - - ASSERT_OK(env_->CreateDir("/dir")); - - // Check that the directory is empty. - ASSERT_TRUE(!env_->FileExists("/dir/non_existent")); - ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok()); - ASSERT_OK(env_->GetChildren("/dir", &children)); - ASSERT_EQ(0, children.size()); - - // Create a file. - ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); - delete writable_file; - - // Check that the file exists. - ASSERT_TRUE(env_->FileExists("/dir/f")); - ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); - ASSERT_EQ(0, file_size); - ASSERT_OK(env_->GetChildren("/dir", &children)); - ASSERT_EQ(1, children.size()); - ASSERT_EQ("f", children[0]); - - // Write to the file. - ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); - ASSERT_OK(writable_file->Append("abc")); - delete writable_file; - - // Check for expected size. - ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); - ASSERT_EQ(3, file_size); - - // Check that renaming works. - ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); - ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g")); - ASSERT_TRUE(!env_->FileExists("/dir/f")); - ASSERT_TRUE(env_->FileExists("/dir/g")); - ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); - ASSERT_EQ(3, file_size); - - // Check that opening non-existent file fails. - SequentialFile* seq_file; - RandomAccessFile* rand_file; - ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok()); - ASSERT_TRUE(!seq_file); - ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok()); - ASSERT_TRUE(!rand_file); - - // Check that deleting works. - ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok()); - ASSERT_OK(env_->DeleteFile("/dir/g")); - ASSERT_TRUE(!env_->FileExists("/dir/g")); - ASSERT_OK(env_->GetChildren("/dir", &children)); - ASSERT_EQ(0, children.size()); - ASSERT_OK(env_->DeleteDir("/dir")); -} - -TEST(MemEnvTest, ReadWrite) { - WritableFile* writable_file; - SequentialFile* seq_file; - RandomAccessFile* rand_file; - Slice result; - char scratch[100]; - - ASSERT_OK(env_->CreateDir("/dir")); - - ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); - ASSERT_OK(writable_file->Append("hello ")); - ASSERT_OK(writable_file->Append("world")); - delete writable_file; - - // Read sequentially. - ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file)); - ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello". - ASSERT_EQ(0, result.compare("hello")); - ASSERT_OK(seq_file->Skip(1)); - ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world". - ASSERT_EQ(0, result.compare("world")); - ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF. - ASSERT_EQ(0, result.size()); - ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file. - ASSERT_OK(seq_file->Read(1000, &result, scratch)); - ASSERT_EQ(0, result.size()); - delete seq_file; - - // Random reads. - ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file)); - ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world". - ASSERT_EQ(0, result.compare("world")); - ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello". - ASSERT_EQ(0, result.compare("hello")); - ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d". - ASSERT_EQ(0, result.compare("d")); - - // Too high offset. - ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok()); - delete rand_file; -} - -TEST(MemEnvTest, Locks) { - FileLock* lock; - - // These are no-ops, but we test they return success. - ASSERT_OK(env_->LockFile("some file", &lock)); - ASSERT_OK(env_->UnlockFile(lock)); -} - -TEST(MemEnvTest, Misc) { - std::string test_dir; - ASSERT_OK(env_->GetTestDirectory(&test_dir)); - ASSERT_TRUE(!test_dir.empty()); - - WritableFile* writable_file; - ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file)); - - // These are no-ops, but we test they return success. - ASSERT_OK(writable_file->Sync()); - ASSERT_OK(writable_file->Flush()); - ASSERT_OK(writable_file->Close()); - delete writable_file; -} - -TEST(MemEnvTest, LargeWrite) { - const size_t kWriteSize = 300 * 1024; - char* scratch = new char[kWriteSize * 2]; - - std::string write_data; - for (size_t i = 0; i < kWriteSize; ++i) { - write_data.append(1, static_cast(i)); - } - - WritableFile* writable_file; - ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); - ASSERT_OK(writable_file->Append("foo")); - ASSERT_OK(writable_file->Append(write_data)); - delete writable_file; - - SequentialFile* seq_file; - Slice result; - ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file)); - ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo". - ASSERT_EQ(0, result.compare("foo")); - - size_t read = 0; - std::string read_data; - while (read < kWriteSize) { - ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch)); - read_data.append(result.data(), result.size()); - read += result.size(); - } - ASSERT_TRUE(write_data == read_data); - delete seq_file; - delete [] scratch; -} - -TEST(MemEnvTest, DBTest) { - Options options; - options.create_if_missing = true; - options.env = env_; - DB* db; - - const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")}; - const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")}; - - ASSERT_OK(DB::Open(options, "/dir/db", &db)); - for (size_t i = 0; i < 3; ++i) { - ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i])); - } - - for (size_t i = 0; i < 3; ++i) { - std::string res; - ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); - ASSERT_TRUE(res == vals[i]); - } - - Iterator* iterator = db->NewIterator(ReadOptions()); - iterator->SeekToFirst(); - for (size_t i = 0; i < 3; ++i) { - ASSERT_TRUE(iterator->Valid()); - ASSERT_TRUE(keys[i] == iterator->key()); - ASSERT_TRUE(vals[i] == iterator->value()); - iterator->Next(); - } - ASSERT_TRUE(!iterator->Valid()); - delete iterator; - - DBImpl* dbi = reinterpret_cast(db); - ASSERT_OK(dbi->TEST_CompactMemTable()); - - for (size_t i = 0; i < 3; ++i) { - std::string res; - ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); - ASSERT_TRUE(res == vals[i]); - } - - delete db; -} - -} // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} -- cgit v1.2.3