diff --git a/src/leveldb/db/memtable.cc b/src/leveldb/db/memtable.cc index bfec0a7e7..287afdbdc 100644 --- a/src/leveldb/db/memtable.cc +++ b/src/leveldb/db/memtable.cc @@ -1,145 +1,145 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/memtable.h" #include "db/dbformat.h" #include "leveldb/comparator.h" #include "leveldb/env.h" #include "leveldb/iterator.h" #include "util/coding.h" namespace leveldb { static Slice GetLengthPrefixedSlice(const char* data) { uint32_t len; const char* p = data; p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted return Slice(p, len); } MemTable::MemTable(const InternalKeyComparator& cmp) : comparator_(cmp), refs_(0), table_(comparator_, &arena_) { } MemTable::~MemTable() { assert(refs_ == 0); } size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); } int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr) const { // Internal keys are encoded as length-prefixed strings. Slice a = GetLengthPrefixedSlice(aptr); Slice b = GetLengthPrefixedSlice(bptr); return comparator.Compare(a, b); } // Encode a suitable internal key target for "target" and return it. // Uses *scratch as scratch space, and the returned pointer will point // into this scratch space. static const char* EncodeKey(std::string* scratch, const Slice& target) { scratch->clear(); PutVarint32(scratch, target.size()); scratch->append(target.data(), target.size()); return scratch->data(); } class MemTableIterator: public Iterator { public: explicit MemTableIterator(MemTable::Table* table) : iter_(table) { } virtual bool Valid() const { return iter_.Valid(); } virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); } virtual void SeekToFirst() { iter_.SeekToFirst(); } virtual void SeekToLast() { iter_.SeekToLast(); } virtual void Next() { iter_.Next(); } virtual void Prev() { iter_.Prev(); } virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); } virtual Slice value() const { Slice key_slice = GetLengthPrefixedSlice(iter_.key()); return GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); } virtual Status status() const { return Status::OK(); } private: MemTable::Table::Iterator iter_; std::string tmp_; // For passing to EncodeKey // No copying allowed MemTableIterator(const MemTableIterator&); void operator=(const MemTableIterator&); }; Iterator* MemTable::NewIterator() { return new MemTableIterator(&table_); } void MemTable::Add(SequenceNumber s, ValueType type, const Slice& key, const Slice& value) { // Format of an entry is concatenation of: // key_size : varint32 of internal_key.size() // key bytes : char[internal_key.size()] // value_size : varint32 of value.size() // value bytes : char[value.size()] size_t key_size = key.size(); size_t val_size = value.size(); size_t internal_key_size = key_size + 8; const size_t encoded_len = VarintLength(internal_key_size) + internal_key_size + VarintLength(val_size) + val_size; char* buf = arena_.Allocate(encoded_len); char* p = EncodeVarint32(buf, internal_key_size); memcpy(p, key.data(), key_size); p += key_size; EncodeFixed64(p, (s << 8) | type); p += 8; p = EncodeVarint32(p, val_size); memcpy(p, value.data(), val_size); - assert((p + val_size) - buf == encoded_len); + assert(p + val_size == buf + encoded_len); table_.Insert(buf); } bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) { Slice memkey = key.memtable_key(); Table::Iterator iter(&table_); iter.Seek(memkey.data()); if (iter.Valid()) { // entry format is: // klength varint32 // userkey char[klength] // tag uint64 // vlength varint32 // value char[vlength] // Check that it belongs to same user key. We do not check the // sequence number since the Seek() call above should have skipped // all entries with overly large sequence numbers. const char* entry = iter.key(); uint32_t key_length; const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length); if (comparator_.comparator.user_comparator()->Compare( Slice(key_ptr, key_length - 8), key.user_key()) == 0) { // Correct user key const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); switch (static_cast(tag & 0xff)) { case kTypeValue: { Slice v = GetLengthPrefixedSlice(key_ptr + key_length); value->assign(v.data(), v.size()); return true; } case kTypeDeletion: *s = Status::NotFound(Slice()); return true; } } } return false; } } // namespace leveldb diff --git a/src/leveldb/db/version_set.cc b/src/leveldb/db/version_set.cc index b1256f90e..2cb6d80ed 100644 --- a/src/leveldb/db/version_set.cc +++ b/src/leveldb/db/version_set.cc @@ -1,1535 +1,1535 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/version_set.h" #include #include #include "db/filename.h" #include "db/log_reader.h" #include "db/log_writer.h" #include "db/memtable.h" #include "db/table_cache.h" #include "leveldb/env.h" #include "leveldb/table_builder.h" #include "table/merger.h" #include "table/two_level_iterator.h" #include "util/coding.h" #include "util/logging.h" namespace leveldb { -static int TargetFileSize(const Options* options) { +static size_t TargetFileSize(const Options* options) { return options->max_file_size; } // Maximum bytes of overlaps in grandparent (i.e., level+2) before we // stop building a single file in a level->level+1 compaction. static int64_t MaxGrandParentOverlapBytes(const Options* options) { return 10 * TargetFileSize(options); } // Maximum number of bytes in all compacted files. We avoid expanding // the lower level file set of a compaction if it would make the // total compaction cover more than this many bytes. static int64_t ExpandedCompactionByteSizeLimit(const Options* options) { return 25 * TargetFileSize(options); } static double MaxBytesForLevel(const Options* options, int level) { // Note: the result for level zero is not really used since we set // the level-0 compaction threshold based on number of files. // Result for both level-0 and level-1 double result = 10. * 1048576.0; while (level > 1) { result *= 10; level--; } return result; } static uint64_t MaxFileSizeForLevel(const Options* options, int level) { // We could vary per level to reduce number of files? return TargetFileSize(options); } static int64_t TotalFileSize(const std::vector& files) { int64_t sum = 0; for (size_t i = 0; i < files.size(); i++) { sum += files[i]->file_size; } return sum; } Version::~Version() { assert(refs_ == 0); // Remove from linked list prev_->next_ = next_; next_->prev_ = prev_; // Drop references to files for (int level = 0; level < config::kNumLevels; level++) { for (size_t i = 0; i < files_[level].size(); i++) { FileMetaData* f = files_[level][i]; assert(f->refs > 0); f->refs--; if (f->refs <= 0) { delete f; } } } } int FindFile(const InternalKeyComparator& icmp, const std::vector& files, const Slice& key) { uint32_t left = 0; uint32_t right = files.size(); while (left < right) { uint32_t mid = (left + right) / 2; const FileMetaData* f = files[mid]; if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) { // Key at "mid.largest" is < "target". Therefore all // files at or before "mid" are uninteresting. left = mid + 1; } else { // Key at "mid.largest" is >= "target". Therefore all files // after "mid" are uninteresting. right = mid; } } return right; } static bool AfterFile(const Comparator* ucmp, const Slice* user_key, const FileMetaData* f) { // NULL user_key occurs before all keys and is therefore never after *f return (user_key != NULL && ucmp->Compare(*user_key, f->largest.user_key()) > 0); } static bool BeforeFile(const Comparator* ucmp, const Slice* user_key, const FileMetaData* f) { // NULL user_key occurs after all keys and is therefore never before *f return (user_key != NULL && ucmp->Compare(*user_key, f->smallest.user_key()) < 0); } bool SomeFileOverlapsRange( const InternalKeyComparator& icmp, bool disjoint_sorted_files, const std::vector& files, const Slice* smallest_user_key, const Slice* largest_user_key) { const Comparator* ucmp = icmp.user_comparator(); if (!disjoint_sorted_files) { // Need to check against all files for (size_t i = 0; i < files.size(); i++) { const FileMetaData* f = files[i]; if (AfterFile(ucmp, smallest_user_key, f) || BeforeFile(ucmp, largest_user_key, f)) { // No overlap } else { return true; // Overlap } } return false; } // Binary search over file list uint32_t index = 0; if (smallest_user_key != NULL) { // Find the earliest possible internal key for smallest_user_key InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek); index = FindFile(icmp, files, small.Encode()); } if (index >= files.size()) { // beginning of range is after all files, so no overlap. return false; } return !BeforeFile(ucmp, largest_user_key, files[index]); } // An internal iterator. For a given version/level pair, yields // information about the files in the level. For a given entry, key() // is the largest key that occurs in the file, and value() is an // 16-byte value containing the file number and file size, both // encoded using EncodeFixed64. class Version::LevelFileNumIterator : public Iterator { public: LevelFileNumIterator(const InternalKeyComparator& icmp, const std::vector* flist) : icmp_(icmp), flist_(flist), index_(flist->size()) { // Marks as invalid } virtual bool Valid() const { return index_ < flist_->size(); } virtual void Seek(const Slice& target) { index_ = FindFile(icmp_, *flist_, target); } virtual void SeekToFirst() { index_ = 0; } virtual void SeekToLast() { index_ = flist_->empty() ? 0 : flist_->size() - 1; } virtual void Next() { assert(Valid()); index_++; } virtual void Prev() { assert(Valid()); if (index_ == 0) { index_ = flist_->size(); // Marks as invalid } else { index_--; } } Slice key() const { assert(Valid()); return (*flist_)[index_]->largest.Encode(); } Slice value() const { assert(Valid()); EncodeFixed64(value_buf_, (*flist_)[index_]->number); EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size); return Slice(value_buf_, sizeof(value_buf_)); } virtual Status status() const { return Status::OK(); } private: const InternalKeyComparator icmp_; const std::vector* const flist_; uint32_t index_; // Backing store for value(). Holds the file number and size. mutable char value_buf_[16]; }; static Iterator* GetFileIterator(void* arg, const ReadOptions& options, const Slice& file_value) { TableCache* cache = reinterpret_cast(arg); if (file_value.size() != 16) { return NewErrorIterator( Status::Corruption("FileReader invoked with unexpected value")); } else { return cache->NewIterator(options, DecodeFixed64(file_value.data()), DecodeFixed64(file_value.data() + 8)); } } Iterator* Version::NewConcatenatingIterator(const ReadOptions& options, int level) const { return NewTwoLevelIterator( new LevelFileNumIterator(vset_->icmp_, &files_[level]), &GetFileIterator, vset_->table_cache_, options); } void Version::AddIterators(const ReadOptions& options, std::vector* iters) { // Merge all level zero files together since they may overlap for (size_t i = 0; i < files_[0].size(); i++) { iters->push_back( vset_->table_cache_->NewIterator( options, files_[0][i]->number, files_[0][i]->file_size)); } // For levels > 0, we can use a concatenating iterator that sequentially // walks through the non-overlapping files in the level, opening them // lazily. for (int level = 1; level < config::kNumLevels; level++) { if (!files_[level].empty()) { iters->push_back(NewConcatenatingIterator(options, level)); } } } // Callback from TableCache::Get() namespace { enum SaverState { kNotFound, kFound, kDeleted, kCorrupt, }; struct Saver { SaverState state; const Comparator* ucmp; Slice user_key; std::string* value; }; } static void SaveValue(void* arg, const Slice& ikey, const Slice& v) { Saver* s = reinterpret_cast(arg); ParsedInternalKey parsed_key; if (!ParseInternalKey(ikey, &parsed_key)) { s->state = kCorrupt; } else { if (s->ucmp->Compare(parsed_key.user_key, s->user_key) == 0) { s->state = (parsed_key.type == kTypeValue) ? kFound : kDeleted; if (s->state == kFound) { s->value->assign(v.data(), v.size()); } } } } static bool NewestFirst(FileMetaData* a, FileMetaData* b) { return a->number > b->number; } void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void* arg, bool (*func)(void*, int, FileMetaData*)) { // TODO(sanjay): Change Version::Get() to use this function. const Comparator* ucmp = vset_->icmp_.user_comparator(); // Search level-0 in order from newest to oldest. std::vector tmp; tmp.reserve(files_[0].size()); for (uint32_t i = 0; i < files_[0].size(); i++) { FileMetaData* f = files_[0][i]; if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 && ucmp->Compare(user_key, f->largest.user_key()) <= 0) { tmp.push_back(f); } } if (!tmp.empty()) { std::sort(tmp.begin(), tmp.end(), NewestFirst); for (uint32_t i = 0; i < tmp.size(); i++) { if (!(*func)(arg, 0, tmp[i])) { return; } } } // Search other levels. for (int level = 1; level < config::kNumLevels; level++) { size_t num_files = files_[level].size(); if (num_files == 0) continue; // Binary search to find earliest index whose largest key >= internal_key. uint32_t index = FindFile(vset_->icmp_, files_[level], internal_key); if (index < num_files) { FileMetaData* f = files_[level][index]; if (ucmp->Compare(user_key, f->smallest.user_key()) < 0) { // All of "f" is past any data for user_key } else { if (!(*func)(arg, level, f)) { return; } } } } } Status Version::Get(const ReadOptions& options, const LookupKey& k, std::string* value, GetStats* stats) { Slice ikey = k.internal_key(); Slice user_key = k.user_key(); const Comparator* ucmp = vset_->icmp_.user_comparator(); Status s; stats->seek_file = NULL; stats->seek_file_level = -1; FileMetaData* last_file_read = NULL; int last_file_read_level = -1; // We can search level-by-level since entries never hop across // levels. Therefore we are guaranteed that if we find data // in an smaller level, later levels are irrelevant. std::vector tmp; FileMetaData* tmp2; for (int level = 0; level < config::kNumLevels; level++) { size_t num_files = files_[level].size(); if (num_files == 0) continue; // Get the list of files to search in this level FileMetaData* const* files = &files_[level][0]; if (level == 0) { // Level-0 files may overlap each other. Find all files that // overlap user_key and process them in order from newest to oldest. tmp.reserve(num_files); for (uint32_t i = 0; i < num_files; i++) { FileMetaData* f = files[i]; if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 && ucmp->Compare(user_key, f->largest.user_key()) <= 0) { tmp.push_back(f); } } if (tmp.empty()) continue; std::sort(tmp.begin(), tmp.end(), NewestFirst); files = &tmp[0]; num_files = tmp.size(); } else { // Binary search to find earliest index whose largest key >= ikey. uint32_t index = FindFile(vset_->icmp_, files_[level], ikey); if (index >= num_files) { files = NULL; num_files = 0; } else { tmp2 = files[index]; if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) { // All of "tmp2" is past any data for user_key files = NULL; num_files = 0; } else { files = &tmp2; num_files = 1; } } } for (uint32_t i = 0; i < num_files; ++i) { if (last_file_read != NULL && stats->seek_file == NULL) { // We have had more than one seek for this read. Charge the 1st file. stats->seek_file = last_file_read; stats->seek_file_level = last_file_read_level; } FileMetaData* f = files[i]; last_file_read = f; last_file_read_level = level; Saver saver; saver.state = kNotFound; saver.ucmp = ucmp; saver.user_key = user_key; saver.value = value; s = vset_->table_cache_->Get(options, f->number, f->file_size, ikey, &saver, SaveValue); if (!s.ok()) { return s; } switch (saver.state) { case kNotFound: break; // Keep searching in other files case kFound: return s; case kDeleted: s = Status::NotFound(Slice()); // Use empty error message for speed return s; case kCorrupt: s = Status::Corruption("corrupted key for ", user_key); return s; } } } return Status::NotFound(Slice()); // Use an empty error message for speed } bool Version::UpdateStats(const GetStats& stats) { FileMetaData* f = stats.seek_file; if (f != NULL) { f->allowed_seeks--; if (f->allowed_seeks <= 0 && file_to_compact_ == NULL) { file_to_compact_ = f; file_to_compact_level_ = stats.seek_file_level; return true; } } return false; } bool Version::RecordReadSample(Slice internal_key) { ParsedInternalKey ikey; if (!ParseInternalKey(internal_key, &ikey)) { return false; } struct State { GetStats stats; // Holds first matching file int matches; static bool Match(void* arg, int level, FileMetaData* f) { State* state = reinterpret_cast(arg); state->matches++; if (state->matches == 1) { // Remember first match. state->stats.seek_file = f; state->stats.seek_file_level = level; } // We can stop iterating once we have a second match. return state->matches < 2; } }; State state; state.matches = 0; ForEachOverlapping(ikey.user_key, internal_key, &state, &State::Match); // Must have at least two matches since we want to merge across // files. But what if we have a single file that contains many // overwrites and deletions? Should we have another mechanism for // finding such files? if (state.matches >= 2) { // 1MB cost is about 1 seek (see comment in Builder::Apply). return UpdateStats(state.stats); } return false; } void Version::Ref() { ++refs_; } void Version::Unref() { assert(this != &vset_->dummy_versions_); assert(refs_ >= 1); --refs_; if (refs_ == 0) { delete this; } } bool Version::OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key) { return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level], smallest_user_key, largest_user_key); } int Version::PickLevelForMemTableOutput( const Slice& smallest_user_key, const Slice& largest_user_key) { int level = 0; if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) { // Push to next level if there is no overlap in next level, // and the #bytes overlapping in the level after that are limited. InternalKey start(smallest_user_key, kMaxSequenceNumber, kValueTypeForSeek); InternalKey limit(largest_user_key, 0, static_cast(0)); std::vector overlaps; while (level < config::kMaxMemCompactLevel) { if (OverlapInLevel(level + 1, &smallest_user_key, &largest_user_key)) { break; } if (level + 2 < config::kNumLevels) { // Check that file does not overlap too many grandparent bytes. GetOverlappingInputs(level + 2, &start, &limit, &overlaps); const int64_t sum = TotalFileSize(overlaps); if (sum > MaxGrandParentOverlapBytes(vset_->options_)) { break; } } level++; } } return level; } // Store in "*inputs" all files in "level" that overlap [begin,end] void Version::GetOverlappingInputs( int level, const InternalKey* begin, const InternalKey* end, std::vector* inputs) { assert(level >= 0); assert(level < config::kNumLevels); inputs->clear(); Slice user_begin, user_end; if (begin != NULL) { user_begin = begin->user_key(); } if (end != NULL) { user_end = end->user_key(); } const Comparator* user_cmp = vset_->icmp_.user_comparator(); for (size_t i = 0; i < files_[level].size(); ) { FileMetaData* f = files_[level][i++]; const Slice file_start = f->smallest.user_key(); const Slice file_limit = f->largest.user_key(); if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) { // "f" is completely before specified range; skip it } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) { // "f" is completely after specified range; skip it } else { inputs->push_back(f); if (level == 0) { // Level-0 files may overlap each other. So check if the newly // added file has expanded the range. If so, restart search. if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) { user_begin = file_start; inputs->clear(); i = 0; } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) { user_end = file_limit; inputs->clear(); i = 0; } } } } } std::string Version::DebugString() const { std::string r; for (int level = 0; level < config::kNumLevels; level++) { // E.g., // --- level 1 --- // 17:123['a' .. 'd'] // 20:43['e' .. 'g'] r.append("--- level "); AppendNumberTo(&r, level); r.append(" ---\n"); const std::vector& files = files_[level]; for (size_t i = 0; i < files.size(); i++) { r.push_back(' '); AppendNumberTo(&r, files[i]->number); r.push_back(':'); AppendNumberTo(&r, files[i]->file_size); r.append("["); r.append(files[i]->smallest.DebugString()); r.append(" .. "); r.append(files[i]->largest.DebugString()); r.append("]\n"); } } return r; } // A helper class so we can efficiently apply a whole sequence // of edits to a particular state without creating intermediate // Versions that contain full copies of the intermediate state. class VersionSet::Builder { private: // Helper to sort by v->files_[file_number].smallest struct BySmallestKey { const InternalKeyComparator* internal_comparator; bool operator()(FileMetaData* f1, FileMetaData* f2) const { int r = internal_comparator->Compare(f1->smallest, f2->smallest); if (r != 0) { return (r < 0); } else { // Break ties by file number return (f1->number < f2->number); } } }; typedef std::set FileSet; struct LevelState { std::set deleted_files; FileSet* added_files; }; VersionSet* vset_; Version* base_; LevelState levels_[config::kNumLevels]; public: // Initialize a builder with the files from *base and other info from *vset Builder(VersionSet* vset, Version* base) : vset_(vset), base_(base) { base_->Ref(); BySmallestKey cmp; cmp.internal_comparator = &vset_->icmp_; for (int level = 0; level < config::kNumLevels; level++) { levels_[level].added_files = new FileSet(cmp); } } ~Builder() { for (int level = 0; level < config::kNumLevels; level++) { const FileSet* added = levels_[level].added_files; std::vector to_unref; to_unref.reserve(added->size()); for (FileSet::const_iterator it = added->begin(); it != added->end(); ++it) { to_unref.push_back(*it); } delete added; for (uint32_t i = 0; i < to_unref.size(); i++) { FileMetaData* f = to_unref[i]; f->refs--; if (f->refs <= 0) { delete f; } } } base_->Unref(); } // Apply all of the edits in *edit to the current state. void Apply(VersionEdit* edit) { // Update compaction pointers for (size_t i = 0; i < edit->compact_pointers_.size(); i++) { const int level = edit->compact_pointers_[i].first; vset_->compact_pointer_[level] = edit->compact_pointers_[i].second.Encode().ToString(); } // Delete files const VersionEdit::DeletedFileSet& del = edit->deleted_files_; for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin(); iter != del.end(); ++iter) { const int level = iter->first; const uint64_t number = iter->second; levels_[level].deleted_files.insert(number); } // Add new files for (size_t i = 0; i < edit->new_files_.size(); i++) { const int level = edit->new_files_[i].first; FileMetaData* f = new FileMetaData(edit->new_files_[i].second); f->refs = 1; // We arrange to automatically compact this file after // a certain number of seeks. Let's assume: // (1) One seek costs 10ms // (2) Writing or reading 1MB costs 10ms (100MB/s) // (3) A compaction of 1MB does 25MB of IO: // 1MB read from this level // 10-12MB read from next level (boundaries may be misaligned) // 10-12MB written to next level // This implies that 25 seeks cost the same as the compaction // of 1MB of data. I.e., one seek costs approximately the // same as the compaction of 40KB of data. We are a little // conservative and allow approximately one seek for every 16KB // of data before triggering a compaction. f->allowed_seeks = (f->file_size / 16384); if (f->allowed_seeks < 100) f->allowed_seeks = 100; levels_[level].deleted_files.erase(f->number); levels_[level].added_files->insert(f); } } // Save the current state in *v. void SaveTo(Version* v) { BySmallestKey cmp; cmp.internal_comparator = &vset_->icmp_; for (int level = 0; level < config::kNumLevels; level++) { // Merge the set of added files with the set of pre-existing files. // Drop any deleted files. Store the result in *v. const std::vector& base_files = base_->files_[level]; std::vector::const_iterator base_iter = base_files.begin(); std::vector::const_iterator base_end = base_files.end(); const FileSet* added = levels_[level].added_files; v->files_[level].reserve(base_files.size() + added->size()); for (FileSet::const_iterator added_iter = added->begin(); added_iter != added->end(); ++added_iter) { // Add all smaller files listed in base_ for (std::vector::const_iterator bpos = std::upper_bound(base_iter, base_end, *added_iter, cmp); base_iter != bpos; ++base_iter) { MaybeAddFile(v, level, *base_iter); } MaybeAddFile(v, level, *added_iter); } // Add remaining base files for (; base_iter != base_end; ++base_iter) { MaybeAddFile(v, level, *base_iter); } #ifndef NDEBUG // Make sure there is no overlap in levels > 0 if (level > 0) { for (uint32_t i = 1; i < v->files_[level].size(); i++) { const InternalKey& prev_end = v->files_[level][i-1]->largest; const InternalKey& this_begin = v->files_[level][i]->smallest; if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) { fprintf(stderr, "overlapping ranges in same level %s vs. %s\n", prev_end.DebugString().c_str(), this_begin.DebugString().c_str()); abort(); } } } #endif } } void MaybeAddFile(Version* v, int level, FileMetaData* f) { if (levels_[level].deleted_files.count(f->number) > 0) { // File is deleted: do nothing } else { std::vector* files = &v->files_[level]; if (level > 0 && !files->empty()) { // Must not overlap assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest, f->smallest) < 0); } f->refs++; files->push_back(f); } } }; VersionSet::VersionSet(const std::string& dbname, const Options* options, TableCache* table_cache, const InternalKeyComparator* cmp) : env_(options->env), dbname_(dbname), options_(options), table_cache_(table_cache), icmp_(*cmp), next_file_number_(2), manifest_file_number_(0), // Filled by Recover() last_sequence_(0), log_number_(0), prev_log_number_(0), descriptor_file_(NULL), descriptor_log_(NULL), dummy_versions_(this), current_(NULL) { AppendVersion(new Version(this)); } VersionSet::~VersionSet() { current_->Unref(); assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty delete descriptor_log_; delete descriptor_file_; } void VersionSet::AppendVersion(Version* v) { // Make "v" current assert(v->refs_ == 0); assert(v != current_); if (current_ != NULL) { current_->Unref(); } current_ = v; v->Ref(); // Append to linked list v->prev_ = dummy_versions_.prev_; v->next_ = &dummy_versions_; v->prev_->next_ = v; v->next_->prev_ = v; } Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { if (edit->has_log_number_) { assert(edit->log_number_ >= log_number_); assert(edit->log_number_ < next_file_number_); } else { edit->SetLogNumber(log_number_); } if (!edit->has_prev_log_number_) { edit->SetPrevLogNumber(prev_log_number_); } edit->SetNextFile(next_file_number_); edit->SetLastSequence(last_sequence_); Version* v = new Version(this); { Builder builder(this, current_); builder.Apply(edit); builder.SaveTo(v); } Finalize(v); // Initialize new descriptor log file if necessary by creating // a temporary file that contains a snapshot of the current version. std::string new_manifest_file; Status s; if (descriptor_log_ == NULL) { // No reason to unlock *mu here since we only hit this path in the // first call to LogAndApply (when opening the database). assert(descriptor_file_ == NULL); new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_); edit->SetNextFile(next_file_number_); s = env_->NewWritableFile(new_manifest_file, &descriptor_file_); if (s.ok()) { descriptor_log_ = new log::Writer(descriptor_file_); s = WriteSnapshot(descriptor_log_); } } // Unlock during expensive MANIFEST log write { mu->Unlock(); // Write new record to MANIFEST log if (s.ok()) { std::string record; edit->EncodeTo(&record); s = descriptor_log_->AddRecord(record); if (s.ok()) { s = descriptor_file_->Sync(); } if (!s.ok()) { Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str()); } } // If we just created a new descriptor file, install it by writing a // new CURRENT file that points to it. if (s.ok() && !new_manifest_file.empty()) { s = SetCurrentFile(env_, dbname_, manifest_file_number_); } mu->Lock(); } // Install the new version if (s.ok()) { AppendVersion(v); log_number_ = edit->log_number_; prev_log_number_ = edit->prev_log_number_; } else { delete v; if (!new_manifest_file.empty()) { delete descriptor_log_; delete descriptor_file_; descriptor_log_ = NULL; descriptor_file_ = NULL; env_->DeleteFile(new_manifest_file); } } return s; } Status VersionSet::Recover(bool *save_manifest) { struct LogReporter : public log::Reader::Reporter { Status* status; virtual void Corruption(size_t bytes, const Status& s) { if (this->status->ok()) *this->status = s; } }; // Read "CURRENT" file, which contains a pointer to the current manifest file std::string current; Status s = ReadFileToString(env_, CurrentFileName(dbname_), ¤t); if (!s.ok()) { return s; } if (current.empty() || current[current.size()-1] != '\n') { return Status::Corruption("CURRENT file does not end with newline"); } current.resize(current.size() - 1); std::string dscname = dbname_ + "/" + current; SequentialFile* file; s = env_->NewSequentialFile(dscname, &file); if (!s.ok()) { return s; } bool have_log_number = false; bool have_prev_log_number = false; bool have_next_file = false; bool have_last_sequence = false; uint64_t next_file = 0; uint64_t last_sequence = 0; uint64_t log_number = 0; uint64_t prev_log_number = 0; Builder builder(this, current_); { LogReporter reporter; reporter.status = &s; log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { VersionEdit edit; s = edit.DecodeFrom(record); if (s.ok()) { if (edit.has_comparator_ && edit.comparator_ != icmp_.user_comparator()->Name()) { s = Status::InvalidArgument( edit.comparator_ + " does not match existing comparator ", icmp_.user_comparator()->Name()); } } if (s.ok()) { builder.Apply(&edit); } if (edit.has_log_number_) { log_number = edit.log_number_; have_log_number = true; } if (edit.has_prev_log_number_) { prev_log_number = edit.prev_log_number_; have_prev_log_number = true; } if (edit.has_next_file_number_) { next_file = edit.next_file_number_; have_next_file = true; } if (edit.has_last_sequence_) { last_sequence = edit.last_sequence_; have_last_sequence = true; } } } delete file; file = NULL; if (s.ok()) { if (!have_next_file) { s = Status::Corruption("no meta-nextfile entry in descriptor"); } else if (!have_log_number) { s = Status::Corruption("no meta-lognumber entry in descriptor"); } else if (!have_last_sequence) { s = Status::Corruption("no last-sequence-number entry in descriptor"); } if (!have_prev_log_number) { prev_log_number = 0; } MarkFileNumberUsed(prev_log_number); MarkFileNumberUsed(log_number); } if (s.ok()) { Version* v = new Version(this); builder.SaveTo(v); // Install recovered version Finalize(v); AppendVersion(v); manifest_file_number_ = next_file; next_file_number_ = next_file + 1; last_sequence_ = last_sequence; log_number_ = log_number; prev_log_number_ = prev_log_number; // See if we can reuse the existing MANIFEST file. if (ReuseManifest(dscname, current)) { // No need to save new manifest } else { *save_manifest = true; } } return s; } bool VersionSet::ReuseManifest(const std::string& dscname, const std::string& dscbase) { if (!options_->reuse_logs) { return false; } FileType manifest_type; uint64_t manifest_number; uint64_t manifest_size; if (!ParseFileName(dscbase, &manifest_number, &manifest_type) || manifest_type != kDescriptorFile || !env_->GetFileSize(dscname, &manifest_size).ok() || // Make new compacted MANIFEST if old one is too big manifest_size >= TargetFileSize(options_)) { return false; } assert(descriptor_file_ == NULL); assert(descriptor_log_ == NULL); Status r = env_->NewAppendableFile(dscname, &descriptor_file_); if (!r.ok()) { Log(options_->info_log, "Reuse MANIFEST: %s\n", r.ToString().c_str()); assert(descriptor_file_ == NULL); return false; } Log(options_->info_log, "Reusing MANIFEST %s\n", dscname.c_str()); descriptor_log_ = new log::Writer(descriptor_file_, manifest_size); manifest_file_number_ = manifest_number; return true; } void VersionSet::MarkFileNumberUsed(uint64_t number) { if (next_file_number_ <= number) { next_file_number_ = number + 1; } } void VersionSet::Finalize(Version* v) { // Precomputed best level for next compaction int best_level = -1; double best_score = -1; for (int level = 0; level < config::kNumLevels-1; level++) { double score; if (level == 0) { // We treat level-0 specially by bounding the number of files // instead of number of bytes for two reasons: // // (1) With larger write-buffer sizes, it is nice not to do too // many level-0 compactions. // // (2) The files in level-0 are merged on every read and // therefore we wish to avoid too many files when the individual // file size is small (perhaps because of a small write-buffer // setting, or very high compression ratios, or lots of // overwrites/deletions). score = v->files_[level].size() / static_cast(config::kL0_CompactionTrigger); } else { // Compute the ratio of current size to size limit. const uint64_t level_bytes = TotalFileSize(v->files_[level]); score = static_cast(level_bytes) / MaxBytesForLevel(options_, level); } if (score > best_score) { best_level = level; best_score = score; } } v->compaction_level_ = best_level; v->compaction_score_ = best_score; } Status VersionSet::WriteSnapshot(log::Writer* log) { // TODO: Break up into multiple records to reduce memory usage on recovery? // Save metadata VersionEdit edit; edit.SetComparatorName(icmp_.user_comparator()->Name()); // Save compaction pointers for (int level = 0; level < config::kNumLevels; level++) { if (!compact_pointer_[level].empty()) { InternalKey key; key.DecodeFrom(compact_pointer_[level]); edit.SetCompactPointer(level, key); } } // Save files for (int level = 0; level < config::kNumLevels; level++) { const std::vector& files = current_->files_[level]; for (size_t i = 0; i < files.size(); i++) { const FileMetaData* f = files[i]; edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest); } } std::string record; edit.EncodeTo(&record); return log->AddRecord(record); } int VersionSet::NumLevelFiles(int level) const { assert(level >= 0); assert(level < config::kNumLevels); return current_->files_[level].size(); } const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const { // Update code if kNumLevels changes assert(config::kNumLevels == 7); snprintf(scratch->buffer, sizeof(scratch->buffer), "files[ %d %d %d %d %d %d %d ]", int(current_->files_[0].size()), int(current_->files_[1].size()), int(current_->files_[2].size()), int(current_->files_[3].size()), int(current_->files_[4].size()), int(current_->files_[5].size()), int(current_->files_[6].size())); return scratch->buffer; } uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) { uint64_t result = 0; for (int level = 0; level < config::kNumLevels; level++) { const std::vector& files = v->files_[level]; for (size_t i = 0; i < files.size(); i++) { if (icmp_.Compare(files[i]->largest, ikey) <= 0) { // Entire file is before "ikey", so just add the file size result += files[i]->file_size; } else if (icmp_.Compare(files[i]->smallest, ikey) > 0) { // Entire file is after "ikey", so ignore if (level > 0) { // Files other than level 0 are sorted by meta->smallest, so // no further files in this level will contain data for // "ikey". break; } } else { // "ikey" falls in the range for this table. Add the // approximate offset of "ikey" within the table. Table* tableptr; Iterator* iter = table_cache_->NewIterator( ReadOptions(), files[i]->number, files[i]->file_size, &tableptr); if (tableptr != NULL) { result += tableptr->ApproximateOffsetOf(ikey.Encode()); } delete iter; } } } return result; } void VersionSet::AddLiveFiles(std::set* live) { for (Version* v = dummy_versions_.next_; v != &dummy_versions_; v = v->next_) { for (int level = 0; level < config::kNumLevels; level++) { const std::vector& files = v->files_[level]; for (size_t i = 0; i < files.size(); i++) { live->insert(files[i]->number); } } } } int64_t VersionSet::NumLevelBytes(int level) const { assert(level >= 0); assert(level < config::kNumLevels); return TotalFileSize(current_->files_[level]); } int64_t VersionSet::MaxNextLevelOverlappingBytes() { int64_t result = 0; std::vector overlaps; for (int level = 1; level < config::kNumLevels - 1; level++) { for (size_t i = 0; i < current_->files_[level].size(); i++) { const FileMetaData* f = current_->files_[level][i]; current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest, &overlaps); const int64_t sum = TotalFileSize(overlaps); if (sum > result) { result = sum; } } } return result; } // Stores the minimal range that covers all entries in inputs in // *smallest, *largest. // REQUIRES: inputs is not empty void VersionSet::GetRange(const std::vector& inputs, InternalKey* smallest, InternalKey* largest) { assert(!inputs.empty()); smallest->Clear(); largest->Clear(); for (size_t i = 0; i < inputs.size(); i++) { FileMetaData* f = inputs[i]; if (i == 0) { *smallest = f->smallest; *largest = f->largest; } else { if (icmp_.Compare(f->smallest, *smallest) < 0) { *smallest = f->smallest; } if (icmp_.Compare(f->largest, *largest) > 0) { *largest = f->largest; } } } } // Stores the minimal range that covers all entries in inputs1 and inputs2 // in *smallest, *largest. // REQUIRES: inputs is not empty void VersionSet::GetRange2(const std::vector& inputs1, const std::vector& inputs2, InternalKey* smallest, InternalKey* largest) { std::vector all = inputs1; all.insert(all.end(), inputs2.begin(), inputs2.end()); GetRange(all, smallest, largest); } Iterator* VersionSet::MakeInputIterator(Compaction* c) { ReadOptions options; options.verify_checksums = options_->paranoid_checks; options.fill_cache = false; // Level-0 files have to be merged together. For other levels, // we will make a concatenating iterator per level. // TODO(opt): use concatenating iterator for level-0 if there is no overlap const int space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2); Iterator** list = new Iterator*[space]; int num = 0; for (int which = 0; which < 2; which++) { if (!c->inputs_[which].empty()) { if (c->level() + which == 0) { const std::vector& files = c->inputs_[which]; for (size_t i = 0; i < files.size(); i++) { list[num++] = table_cache_->NewIterator( options, files[i]->number, files[i]->file_size); } } else { // Create concatenating iterator for the files from this level list[num++] = NewTwoLevelIterator( new Version::LevelFileNumIterator(icmp_, &c->inputs_[which]), &GetFileIterator, table_cache_, options); } } } assert(num <= space); Iterator* result = NewMergingIterator(&icmp_, list, num); delete[] list; return result; } Compaction* VersionSet::PickCompaction() { Compaction* c; int level; // We prefer compactions triggered by too much data in a level over // the compactions triggered by seeks. const bool size_compaction = (current_->compaction_score_ >= 1); const bool seek_compaction = (current_->file_to_compact_ != NULL); if (size_compaction) { level = current_->compaction_level_; assert(level >= 0); assert(level+1 < config::kNumLevels); c = new Compaction(options_, level); // Pick the first file that comes after compact_pointer_[level] for (size_t i = 0; i < current_->files_[level].size(); i++) { FileMetaData* f = current_->files_[level][i]; if (compact_pointer_[level].empty() || icmp_.Compare(f->largest.Encode(), compact_pointer_[level]) > 0) { c->inputs_[0].push_back(f); break; } } if (c->inputs_[0].empty()) { // Wrap-around to the beginning of the key space c->inputs_[0].push_back(current_->files_[level][0]); } } else if (seek_compaction) { level = current_->file_to_compact_level_; c = new Compaction(options_, level); c->inputs_[0].push_back(current_->file_to_compact_); } else { return NULL; } c->input_version_ = current_; c->input_version_->Ref(); // Files in level 0 may overlap each other, so pick up all overlapping ones if (level == 0) { InternalKey smallest, largest; GetRange(c->inputs_[0], &smallest, &largest); // Note that the next call will discard the file we placed in // c->inputs_[0] earlier and replace it with an overlapping set // which will include the picked file. current_->GetOverlappingInputs(0, &smallest, &largest, &c->inputs_[0]); assert(!c->inputs_[0].empty()); } SetupOtherInputs(c); return c; } void VersionSet::SetupOtherInputs(Compaction* c) { const int level = c->level(); InternalKey smallest, largest; GetRange(c->inputs_[0], &smallest, &largest); current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]); // Get entire range covered by compaction InternalKey all_start, all_limit; GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit); // See if we can grow the number of inputs in "level" without // changing the number of "level+1" files we pick up. if (!c->inputs_[1].empty()) { std::vector expanded0; current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0); const int64_t inputs0_size = TotalFileSize(c->inputs_[0]); const int64_t inputs1_size = TotalFileSize(c->inputs_[1]); const int64_t expanded0_size = TotalFileSize(expanded0); if (expanded0.size() > c->inputs_[0].size() && inputs1_size + expanded0_size < ExpandedCompactionByteSizeLimit(options_)) { InternalKey new_start, new_limit; GetRange(expanded0, &new_start, &new_limit); std::vector expanded1; current_->GetOverlappingInputs(level+1, &new_start, &new_limit, &expanded1); if (expanded1.size() == c->inputs_[1].size()) { Log(options_->info_log, "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n", level, int(c->inputs_[0].size()), int(c->inputs_[1].size()), long(inputs0_size), long(inputs1_size), int(expanded0.size()), int(expanded1.size()), long(expanded0_size), long(inputs1_size)); smallest = new_start; largest = new_limit; c->inputs_[0] = expanded0; c->inputs_[1] = expanded1; GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit); } } } // Compute the set of grandparent files that overlap this compaction // (parent == level+1; grandparent == level+2) if (level + 2 < config::kNumLevels) { current_->GetOverlappingInputs(level + 2, &all_start, &all_limit, &c->grandparents_); } if (false) { Log(options_->info_log, "Compacting %d '%s' .. '%s'", level, smallest.DebugString().c_str(), largest.DebugString().c_str()); } // Update the place where we will do the next compaction for this level. // We update this immediately instead of waiting for the VersionEdit // to be applied so that if the compaction fails, we will try a different // key range next time. compact_pointer_[level] = largest.Encode().ToString(); c->edit_.SetCompactPointer(level, largest); } Compaction* VersionSet::CompactRange( int level, const InternalKey* begin, const InternalKey* end) { std::vector inputs; current_->GetOverlappingInputs(level, begin, end, &inputs); if (inputs.empty()) { return NULL; } // Avoid compacting too much in one shot in case the range is large. // But we cannot do this for level-0 since level-0 files can overlap // and we must not pick one file and drop another older file if the // two files overlap. if (level > 0) { const uint64_t limit = MaxFileSizeForLevel(options_, level); uint64_t total = 0; for (size_t i = 0; i < inputs.size(); i++) { uint64_t s = inputs[i]->file_size; total += s; if (total >= limit) { inputs.resize(i + 1); break; } } } Compaction* c = new Compaction(options_, level); c->input_version_ = current_; c->input_version_->Ref(); c->inputs_[0] = inputs; SetupOtherInputs(c); return c; } Compaction::Compaction(const Options* options, int level) : level_(level), max_output_file_size_(MaxFileSizeForLevel(options, level)), input_version_(NULL), grandparent_index_(0), seen_key_(false), overlapped_bytes_(0) { for (int i = 0; i < config::kNumLevels; i++) { level_ptrs_[i] = 0; } } Compaction::~Compaction() { if (input_version_ != NULL) { input_version_->Unref(); } } bool Compaction::IsTrivialMove() const { const VersionSet* vset = input_version_->vset_; // Avoid a move if there is lots of overlapping grandparent data. // Otherwise, the move could create a parent file that will require // a very expensive merge later on. return (num_input_files(0) == 1 && num_input_files(1) == 0 && TotalFileSize(grandparents_) <= MaxGrandParentOverlapBytes(vset->options_)); } void Compaction::AddInputDeletions(VersionEdit* edit) { for (int which = 0; which < 2; which++) { for (size_t i = 0; i < inputs_[which].size(); i++) { edit->DeleteFile(level_ + which, inputs_[which][i]->number); } } } bool Compaction::IsBaseLevelForKey(const Slice& user_key) { // Maybe use binary search to find right entry instead of linear search? const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator(); for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) { const std::vector& files = input_version_->files_[lvl]; for (; level_ptrs_[lvl] < files.size(); ) { FileMetaData* f = files[level_ptrs_[lvl]]; if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) { // We've advanced far enough if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) { // Key falls in this file's range, so definitely not base level return false; } break; } level_ptrs_[lvl]++; } } return true; } bool Compaction::ShouldStopBefore(const Slice& internal_key) { const VersionSet* vset = input_version_->vset_; // Scan to find earliest grandparent file that contains key. const InternalKeyComparator* icmp = &vset->icmp_; while (grandparent_index_ < grandparents_.size() && icmp->Compare(internal_key, grandparents_[grandparent_index_]->largest.Encode()) > 0) { if (seen_key_) { overlapped_bytes_ += grandparents_[grandparent_index_]->file_size; } grandparent_index_++; } seen_key_ = true; if (overlapped_bytes_ > MaxGrandParentOverlapBytes(vset->options_)) { // Too much overlap for current output; start new output overlapped_bytes_ = 0; return true; } else { return false; } } void Compaction::ReleaseInputs() { if (input_version_ != NULL) { input_version_->Unref(); input_version_ = NULL; } } } // namespace leveldb diff --git a/src/leveldb/port/atomic_pointer.h b/src/leveldb/port/atomic_pointer.h index 1c4c7aafc..d79a02230 100644 --- a/src/leveldb/port/atomic_pointer.h +++ b/src/leveldb/port/atomic_pointer.h @@ -1,242 +1,245 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // AtomicPointer provides storage for a lock-free pointer. // Platform-dependent implementation of AtomicPointer: // - If the platform provides a cheap barrier, we use it with raw pointers // - If is present (on newer versions of gcc, it is), we use // a -based AtomicPointer. However we prefer the memory // barrier based version, because at least on a gcc 4.4 32-bit build // on linux, we have encountered a buggy implementation. // Also, some implementations are much slower than a memory-barrier // based implementation (~16ns for based acquire-load vs. ~1ns for // a barrier based acquire-load). // This code is based on atomicops-internals-* in Google's perftools: // http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase #ifndef PORT_ATOMIC_POINTER_H_ #define PORT_ATOMIC_POINTER_H_ #include #ifdef LEVELDB_ATOMIC_PRESENT #include #endif #ifdef OS_WIN #include #endif #ifdef OS_MACOSX #include #endif #if defined(_M_X64) || defined(__x86_64__) #define ARCH_CPU_X86_FAMILY 1 #elif defined(_M_IX86) || defined(__i386__) || defined(__i386) #define ARCH_CPU_X86_FAMILY 1 #elif defined(__ARMEL__) #define ARCH_CPU_ARM_FAMILY 1 #elif defined(__aarch64__) #define ARCH_CPU_ARM64_FAMILY 1 #elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__) #define ARCH_CPU_PPC_FAMILY 1 #elif defined(__mips__) #define ARCH_CPU_MIPS_FAMILY 1 #endif namespace leveldb { namespace port { +// AtomicPointer based on if available +#if defined(LEVELDB_ATOMIC_PRESENT) +class AtomicPointer { + private: + std::atomic rep_; + public: + AtomicPointer() { } + explicit AtomicPointer(void* v) : rep_(v) { } + inline void* Acquire_Load() const { + return rep_.load(std::memory_order_acquire); + } + inline void Release_Store(void* v) { + rep_.store(v, std::memory_order_release); + } + inline void* NoBarrier_Load() const { + return rep_.load(std::memory_order_relaxed); + } + inline void NoBarrier_Store(void* v) { + rep_.store(v, std::memory_order_relaxed); + } +}; + +#else + // Define MemoryBarrier() if available // Windows on x86 #if defined(OS_WIN) && defined(COMPILER_MSVC) && defined(ARCH_CPU_X86_FAMILY) // windows.h already provides a MemoryBarrier(void) macro // http://msdn.microsoft.com/en-us/library/ms684208(v=vs.85).aspx #define LEVELDB_HAVE_MEMORY_BARRIER // Mac OS #elif defined(OS_MACOSX) inline void MemoryBarrier() { OSMemoryBarrier(); } #define LEVELDB_HAVE_MEMORY_BARRIER // Gcc on x86 #elif defined(ARCH_CPU_X86_FAMILY) && defined(__GNUC__) inline void MemoryBarrier() { // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. __asm__ __volatile__("" : : : "memory"); } #define LEVELDB_HAVE_MEMORY_BARRIER // Sun Studio #elif defined(ARCH_CPU_X86_FAMILY) && defined(__SUNPRO_CC) inline void MemoryBarrier() { // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. asm volatile("" : : : "memory"); } #define LEVELDB_HAVE_MEMORY_BARRIER // ARM Linux #elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__) typedef void (*LinuxKernelMemoryBarrierFunc)(void); // The Linux ARM kernel provides a highly optimized device-specific memory // barrier function at a fixed memory address that is mapped in every // user-level process. // // This beats using CPU-specific instructions which are, on single-core // devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more // than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking // shows that the extra function call cost is completely negligible on // multi-core devices. // inline void MemoryBarrier() { (*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)(); } #define LEVELDB_HAVE_MEMORY_BARRIER // ARM64 #elif defined(ARCH_CPU_ARM64_FAMILY) inline void MemoryBarrier() { asm volatile("dmb sy" : : : "memory"); } #define LEVELDB_HAVE_MEMORY_BARRIER // PPC #elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__) inline void MemoryBarrier() { // TODO for some powerpc expert: is there a cheaper suitable variant? // Perhaps by having separate barriers for acquire and release ops. asm volatile("sync" : : : "memory"); } #define LEVELDB_HAVE_MEMORY_BARRIER // MIPS #elif defined(ARCH_CPU_MIPS_FAMILY) && defined(__GNUC__) inline void MemoryBarrier() { __asm__ __volatile__("sync" : : : "memory"); } #define LEVELDB_HAVE_MEMORY_BARRIER #endif // AtomicPointer built using platform-specific MemoryBarrier() #if defined(LEVELDB_HAVE_MEMORY_BARRIER) class AtomicPointer { private: void* rep_; public: AtomicPointer() { } explicit AtomicPointer(void* p) : rep_(p) {} inline void* NoBarrier_Load() const { return rep_; } inline void NoBarrier_Store(void* v) { rep_ = v; } inline void* Acquire_Load() const { void* result = rep_; MemoryBarrier(); return result; } inline void Release_Store(void* v) { MemoryBarrier(); rep_ = v; } }; -// AtomicPointer based on -#elif defined(LEVELDB_ATOMIC_PRESENT) -class AtomicPointer { - private: - std::atomic rep_; - public: - AtomicPointer() { } - explicit AtomicPointer(void* v) : rep_(v) { } - inline void* Acquire_Load() const { - return rep_.load(std::memory_order_acquire); - } - inline void Release_Store(void* v) { - rep_.store(v, std::memory_order_release); - } - inline void* NoBarrier_Load() const { - return rep_.load(std::memory_order_relaxed); - } - inline void NoBarrier_Store(void* v) { - rep_.store(v, std::memory_order_relaxed); - } -}; - // Atomic pointer based on sparc memory barriers #elif defined(__sparcv9) && defined(__GNUC__) class AtomicPointer { private: void* rep_; public: AtomicPointer() { } explicit AtomicPointer(void* v) : rep_(v) { } inline void* Acquire_Load() const { void* val; __asm__ __volatile__ ( "ldx [%[rep_]], %[val] \n\t" "membar #LoadLoad|#LoadStore \n\t" : [val] "=r" (val) : [rep_] "r" (&rep_) : "memory"); return val; } inline void Release_Store(void* v) { __asm__ __volatile__ ( "membar #LoadStore|#StoreStore \n\t" "stx %[v], [%[rep_]] \n\t" : : [rep_] "r" (&rep_), [v] "r" (v) : "memory"); } inline void* NoBarrier_Load() const { return rep_; } inline void NoBarrier_Store(void* v) { rep_ = v; } }; // Atomic pointer based on ia64 acq/rel #elif defined(__ia64) && defined(__GNUC__) class AtomicPointer { private: void* rep_; public: AtomicPointer() { } explicit AtomicPointer(void* v) : rep_(v) { } inline void* Acquire_Load() const { void* val ; __asm__ __volatile__ ( "ld8.acq %[val] = [%[rep_]] \n\t" : [val] "=r" (val) : [rep_] "r" (&rep_) : "memory" ); return val; } inline void Release_Store(void* v) { __asm__ __volatile__ ( "st8.rel [%[rep_]] = %[v] \n\t" : : [rep_] "r" (&rep_), [v] "r" (v) : "memory" ); } inline void* NoBarrier_Load() const { return rep_; } inline void NoBarrier_Store(void* v) { rep_ = v; } }; // We have neither MemoryBarrier(), nor #else #error Please implement AtomicPointer for this platform. +#endif #endif #undef LEVELDB_HAVE_MEMORY_BARRIER #undef ARCH_CPU_X86_FAMILY #undef ARCH_CPU_ARM_FAMILY #undef ARCH_CPU_ARM64_FAMILY #undef ARCH_CPU_PPC_FAMILY } // namespace port } // namespace leveldb #endif // PORT_ATOMIC_POINTER_H_ diff --git a/src/leveldb/port/port_example.h b/src/leveldb/port/port_example.h index 97bd669a5..5b1d027de 100644 --- a/src/leveldb/port/port_example.h +++ b/src/leveldb/port/port_example.h @@ -1,141 +1,145 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // This file contains the specification, but not the implementations, // of the types/operations/etc. that should be defined by a platform // specific port_.h file. Use this file as a reference for // how to port this package to a new platform. #ifndef STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ #define STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ namespace leveldb { namespace port { // TODO(jorlow): Many of these belong more in the environment class rather than // here. We should try moving them and see if it affects perf. // The following boolean constant must be true on a little-endian machine // and false otherwise. static const bool kLittleEndian = true /* or some other expression */; // ------------------ Threading ------------------- // A Mutex represents an exclusive lock. class Mutex { public: Mutex(); ~Mutex(); // Lock the mutex. Waits until other lockers have exited. // Will deadlock if the mutex is already locked by this thread. void Lock(); // Unlock the mutex. // REQUIRES: This mutex was locked by this thread. void Unlock(); // Optionally crash if this thread does not hold this mutex. // The implementation must be fast, especially if NDEBUG is // defined. The implementation is allowed to skip all checks. void AssertHeld(); }; class CondVar { public: explicit CondVar(Mutex* mu); ~CondVar(); // Atomically release *mu and block on this condition variable until // either a call to SignalAll(), or a call to Signal() that picks // this thread to wakeup. // REQUIRES: this thread holds *mu void Wait(); // If there are some threads waiting, wake up at least one of them. void Signal(); // Wake up all waiting threads. void SignallAll(); }; // Thread-safe initialization. // Used as follows: // static port::OnceType init_control = LEVELDB_ONCE_INIT; // static void Initializer() { ... do something ...; } // ... // port::InitOnce(&init_control, &Initializer); typedef intptr_t OnceType; #define LEVELDB_ONCE_INIT 0 extern void InitOnce(port::OnceType*, void (*initializer)()); // A type that holds a pointer that can be read or written atomically // (i.e., without word-tearing.) class AtomicPointer { private: intptr_t rep_; public: // Initialize to arbitrary value AtomicPointer(); // Initialize to hold v explicit AtomicPointer(void* v) : rep_(v) { } // Read and return the stored pointer with the guarantee that no // later memory access (read or write) by this thread can be // reordered ahead of this read. void* Acquire_Load() const; // Set v as the stored pointer with the guarantee that no earlier // memory access (read or write) by this thread can be reordered // after this store. void Release_Store(void* v); // Read the stored pointer with no ordering guarantees. void* NoBarrier_Load() const; // Set va as the stored pointer with no ordering guarantees. void NoBarrier_Store(void* v); }; // ------------------ Compression ------------------- // Store the snappy compression of "input[0,input_length-1]" in *output. // Returns false if snappy is not supported by this port. extern bool Snappy_Compress(const char* input, size_t input_length, std::string* output); // If input[0,input_length-1] looks like a valid snappy compressed // buffer, store the size of the uncompressed data in *result and // return true. Else return false. extern bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result); // Attempt to snappy uncompress input[0,input_length-1] into *output. // Returns true if successful, false if the input is invalid lightweight // compressed data. // // REQUIRES: at least the first "n" bytes of output[] must be writable // where "n" is the result of a successful call to // Snappy_GetUncompressedLength. extern bool Snappy_Uncompress(const char* input_data, size_t input_length, char* output); // ------------------ Miscellaneous ------------------- // If heap profiling is not supported, returns false. // Else repeatedly calls (*func)(arg, data, n) and then returns true. // The concatenation of all "data[0,n-1]" fragments is the heap profile. extern bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg); +// Determine whether a working accelerated crc32 implementation exists +// Returns true if AcceleratedCRC32C is safe to call +bool HasAcceleratedCRC32C(); + // Extend the CRC to include the first n bytes of buf. // // Returns zero if the CRC cannot be extended using acceleration, else returns // the newly extended CRC value (which may also be zero). uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size); } // namespace port } // namespace leveldb #endif // STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ diff --git a/src/leveldb/port/port_posix.cc b/src/leveldb/port/port_posix.cc index 30e8007ae..ec39e9219 100644 --- a/src/leveldb/port/port_posix.cc +++ b/src/leveldb/port/port_posix.cc @@ -1,53 +1,67 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "port/port_posix.h" #include #include #include +#if (defined(__x86_64__) || defined(__i386__)) && defined(__GNUC__) +#include +#endif + namespace leveldb { namespace port { static void PthreadCall(const char* label, int result) { if (result != 0) { fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); abort(); } } Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } CondVar::CondVar(Mutex* mu) : mu_(mu) { PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); } CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } void CondVar::Wait() { PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); } void CondVar::Signal() { PthreadCall("signal", pthread_cond_signal(&cv_)); } void CondVar::SignalAll() { PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); } void InitOnce(OnceType* once, void (*initializer)()) { PthreadCall("once", pthread_once(once, initializer)); } +bool HasAcceleratedCRC32C() { +#if (defined(__x86_64__) || defined(__i386__)) && defined(__GNUC__) + unsigned int eax, ebx, ecx, edx; + __get_cpuid(1, &eax, &ebx, &ecx, &edx); + return (ecx & (1 << 20)) != 0; +#else + return false; +#endif +} + } // namespace port } // namespace leveldb diff --git a/src/leveldb/port/port_posix.h b/src/leveldb/port/port_posix.h index 7e8213b22..d85fa5d63 100644 --- a/src/leveldb/port/port_posix.h +++ b/src/leveldb/port/port_posix.h @@ -1,160 +1,161 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // See port_example.h for documentation for the following types/functions. #ifndef STORAGE_LEVELDB_PORT_PORT_POSIX_H_ #define STORAGE_LEVELDB_PORT_PORT_POSIX_H_ #undef PLATFORM_IS_LITTLE_ENDIAN #if defined(OS_MACOSX) #include #if defined(__DARWIN_LITTLE_ENDIAN) && defined(__DARWIN_BYTE_ORDER) #define PLATFORM_IS_LITTLE_ENDIAN \ (__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN) #endif #elif defined(OS_SOLARIS) #include #ifdef _LITTLE_ENDIAN #define PLATFORM_IS_LITTLE_ENDIAN true #else #define PLATFORM_IS_LITTLE_ENDIAN false #endif #elif defined(OS_FREEBSD) || defined(OS_OPENBSD) ||\ defined(OS_NETBSD) || defined(OS_DRAGONFLYBSD) #include #include #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #elif defined(OS_HPUX) #define PLATFORM_IS_LITTLE_ENDIAN false #elif defined(OS_ANDROID) // Due to a bug in the NDK x86 definition, // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android. // See http://code.google.com/p/android/issues/detail?id=39824 #include #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #else #include #endif #include #ifdef SNAPPY #include #endif #include #include #include "port/atomic_pointer.h" #ifndef PLATFORM_IS_LITTLE_ENDIAN #define PLATFORM_IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN) #endif #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\ defined(OS_ANDROID) || defined(OS_HPUX) || defined(CYGWIN) // Use fread/fwrite/fflush on platforms without _unlocked variants #define fread_unlocked fread #define fwrite_unlocked fwrite #define fflush_unlocked fflush #endif #if defined(OS_FREEBSD) ||\ defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) // Use fsync() on platforms without fdatasync() #define fdatasync fsync #endif #if defined(OS_MACOSX) #define fdatasync(fd) fcntl(fd, F_FULLFSYNC, 0) #endif #if defined(OS_ANDROID) && __ANDROID_API__ < 9 // fdatasync() was only introduced in API level 9 on Android. Use fsync() // when targetting older platforms. #define fdatasync fsync #endif namespace leveldb { namespace port { static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN; #undef PLATFORM_IS_LITTLE_ENDIAN class CondVar; class Mutex { public: Mutex(); ~Mutex(); void Lock(); void Unlock(); void AssertHeld() { } private: friend class CondVar; pthread_mutex_t mu_; // No copying Mutex(const Mutex&); void operator=(const Mutex&); }; class CondVar { public: explicit CondVar(Mutex* mu); ~CondVar(); void Wait(); void Signal(); void SignalAll(); private: pthread_cond_t cv_; Mutex* mu_; }; typedef pthread_once_t OnceType; #define LEVELDB_ONCE_INIT PTHREAD_ONCE_INIT extern void InitOnce(OnceType* once, void (*initializer)()); inline bool Snappy_Compress(const char* input, size_t length, ::std::string* output) { #ifdef SNAPPY output->resize(snappy::MaxCompressedLength(length)); size_t outlen; snappy::RawCompress(input, length, &(*output)[0], &outlen); output->resize(outlen); return true; #endif return false; } inline bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result) { #ifdef SNAPPY return snappy::GetUncompressedLength(input, length, result); #else return false; #endif } inline bool Snappy_Uncompress(const char* input, size_t length, char* output) { #ifdef SNAPPY return snappy::RawUncompress(input, length, output); #else return false; #endif } inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { return false; } +bool HasAcceleratedCRC32C(); uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size); } // namespace port } // namespace leveldb #endif // STORAGE_LEVELDB_PORT_PORT_POSIX_H_ diff --git a/src/leveldb/port/port_posix_sse.cc b/src/leveldb/port/port_posix_sse.cc index 1e519ba0b..2d49c21dd 100644 --- a/src/leveldb/port/port_posix_sse.cc +++ b/src/leveldb/port/port_posix_sse.cc @@ -1,129 +1,110 @@ // Copyright 2016 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // A portable implementation of crc32c, optimized to handle // four bytes at a time. // // In a separate source file to allow this accelerated CRC32C function to be // compiled with the appropriate compiler flags to enable x86 SSE 4.2 // instructions. #include #include #include "port/port.h" #if defined(LEVELDB_PLATFORM_POSIX_SSE) #if defined(_MSC_VER) #include #elif defined(__GNUC__) && defined(__SSE4_2__) #include -#include #endif #endif // defined(LEVELDB_PLATFORM_POSIX_SSE) namespace leveldb { namespace port { #if defined(LEVELDB_PLATFORM_POSIX_SSE) // Used to fetch a naturally-aligned 32-bit word in little endian byte-order static inline uint32_t LE_LOAD32(const uint8_t *p) { // SSE is x86 only, so ensured that |p| is always little-endian. uint32_t word; memcpy(&word, p, sizeof(word)); return word; } #if defined(_M_X64) || defined(__x86_64__) // LE_LOAD64 is only used on x64. // Used to fetch a naturally-aligned 64-bit word in little endian byte-order static inline uint64_t LE_LOAD64(const uint8_t *p) { uint64_t dword; memcpy(&dword, p, sizeof(dword)); return dword; } #endif // defined(_M_X64) || defined(__x86_64__) -static inline bool HaveSSE42() { -#if defined(_MSC_VER) - int cpu_info[4]; - __cpuid(cpu_info, 1); - return (cpu_info[2] & (1 << 20)) != 0; -#elif defined(__GNUC__) - unsigned int eax, ebx, ecx, edx; - __get_cpuid(1, &eax, &ebx, &ecx, &edx); - return (ecx & (1 << 20)) != 0; -#else - return false; -#endif -} - #endif // defined(LEVELDB_PLATFORM_POSIX_SSE) // For further improvements see Intel publication at: // http://download.intel.com/design/intarch/papers/323405.pdf uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size) { #if !defined(LEVELDB_PLATFORM_POSIX_SSE) return 0; #else - static bool have = HaveSSE42(); - if (!have) { - return 0; - } const uint8_t *p = reinterpret_cast(buf); const uint8_t *e = p + size; uint32_t l = crc ^ 0xffffffffu; #define STEP1 do { \ l = _mm_crc32_u8(l, *p++); \ } while (0) #define STEP4 do { \ l = _mm_crc32_u32(l, LE_LOAD32(p)); \ p += 4; \ } while (0) #define STEP8 do { \ l = _mm_crc32_u64(l, LE_LOAD64(p)); \ p += 8; \ } while (0) if (size > 16) { // Process unaligned bytes for (unsigned int i = reinterpret_cast(p) % 8; i; --i) { STEP1; } // _mm_crc32_u64 is only available on x64. #if defined(_M_X64) || defined(__x86_64__) // Process 8 bytes at a time while ((e-p) >= 8) { STEP8; } // Process 4 bytes at a time if ((e-p) >= 4) { STEP4; } #else // !(defined(_M_X64) || defined(__x86_64__)) // Process 4 bytes at a time while ((e-p) >= 4) { STEP4; } #endif // defined(_M_X64) || defined(__x86_64__) } // Process the last few bytes while (p != e) { STEP1; } #undef STEP8 #undef STEP4 #undef STEP1 return l ^ 0xffffffffu; #endif // defined(LEVELDB_PLATFORM_POSIX_SSE) } } // namespace port } // namespace leveldb diff --git a/src/leveldb/port/port_win.cc b/src/leveldb/port/port_win.cc index 1b0f060a1..1be9e8d5b 100644 --- a/src/leveldb/port/port_win.cc +++ b/src/leveldb/port/port_win.cc @@ -1,147 +1,158 @@ // LevelDB Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // See port_example.h for documentation for the following types/functions. // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of the University of California, Berkeley nor the // names of its contributors may be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #include "port/port_win.h" #include #include +#include namespace leveldb { namespace port { Mutex::Mutex() : cs_(NULL) { assert(!cs_); cs_ = static_cast(new CRITICAL_SECTION()); ::InitializeCriticalSection(static_cast(cs_)); assert(cs_); } Mutex::~Mutex() { assert(cs_); ::DeleteCriticalSection(static_cast(cs_)); delete static_cast(cs_); cs_ = NULL; assert(!cs_); } void Mutex::Lock() { assert(cs_); ::EnterCriticalSection(static_cast(cs_)); } void Mutex::Unlock() { assert(cs_); ::LeaveCriticalSection(static_cast(cs_)); } void Mutex::AssertHeld() { assert(cs_); assert(1); } CondVar::CondVar(Mutex* mu) : waiting_(0), mu_(mu), sem1_(::CreateSemaphore(NULL, 0, 10000, NULL)), sem2_(::CreateSemaphore(NULL, 0, 10000, NULL)) { assert(mu_); } CondVar::~CondVar() { ::CloseHandle(sem1_); ::CloseHandle(sem2_); } void CondVar::Wait() { mu_->AssertHeld(); wait_mtx_.Lock(); ++waiting_; wait_mtx_.Unlock(); mu_->Unlock(); // initiate handshake ::WaitForSingleObject(sem1_, INFINITE); ::ReleaseSemaphore(sem2_, 1, NULL); mu_->Lock(); } void CondVar::Signal() { wait_mtx_.Lock(); if (waiting_ > 0) { --waiting_; // finalize handshake ::ReleaseSemaphore(sem1_, 1, NULL); ::WaitForSingleObject(sem2_, INFINITE); } wait_mtx_.Unlock(); } void CondVar::SignalAll() { wait_mtx_.Lock(); ::ReleaseSemaphore(sem1_, waiting_, NULL); while(waiting_ > 0) { --waiting_; ::WaitForSingleObject(sem2_, INFINITE); } wait_mtx_.Unlock(); } AtomicPointer::AtomicPointer(void* v) { Release_Store(v); } void InitOnce(OnceType* once, void (*initializer)()) { once->InitOnce(initializer); } void* AtomicPointer::Acquire_Load() const { void * p = NULL; InterlockedExchangePointer(&p, rep_); return p; } void AtomicPointer::Release_Store(void* v) { InterlockedExchangePointer(&rep_, v); } void* AtomicPointer::NoBarrier_Load() const { return rep_; } void AtomicPointer::NoBarrier_Store(void* v) { rep_ = v; } +bool HasAcceleratedCRC32C() { +#if defined(__x86_64__) || defined(__i386__) + int cpu_info[4]; + __cpuid(cpu_info, 1); + return (cpu_info[2] & (1 << 20)) != 0; +#else + return false; +#endif +} + } } diff --git a/src/leveldb/port/port_win.h b/src/leveldb/port/port_win.h index 500900482..e8bf46ef2 100644 --- a/src/leveldb/port/port_win.h +++ b/src/leveldb/port/port_win.h @@ -1,176 +1,177 @@ // LevelDB Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // See port_example.h for documentation for the following types/functions. // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of the University of California, Berkeley nor the // names of its contributors may be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #ifndef STORAGE_LEVELDB_PORT_PORT_WIN_H_ #define STORAGE_LEVELDB_PORT_PORT_WIN_H_ #ifdef _MSC_VER #define snprintf _snprintf #define close _close #define fread_unlocked _fread_nolock #endif #include #include #ifdef SNAPPY #include #endif namespace leveldb { namespace port { // Windows is little endian (for now :p) static const bool kLittleEndian = true; class CondVar; class Mutex { public: Mutex(); ~Mutex(); void Lock(); void Unlock(); void AssertHeld(); private: friend class CondVar; // critical sections are more efficient than mutexes // but they are not recursive and can only be used to synchronize threads within the same process // we use opaque void * to avoid including windows.h in port_win.h void * cs_; // No copying Mutex(const Mutex&); void operator=(const Mutex&); }; // the Win32 API offers a dependable condition variable mechanism, but only starting with // Windows 2008 and Vista // no matter what we will implement our own condition variable with a semaphore // implementation as described in a paper written by Andrew D. Birrell in 2003 class CondVar { public: explicit CondVar(Mutex* mu); ~CondVar(); void Wait(); void Signal(); void SignalAll(); private: Mutex* mu_; Mutex wait_mtx_; long waiting_; void * sem1_; void * sem2_; }; class OnceType { public: // OnceType() : init_(false) {} OnceType(const OnceType &once) : init_(once.init_) {} OnceType(bool f) : init_(f) {} void InitOnce(void (*initializer)()) { mutex_.Lock(); if (!init_) { init_ = true; initializer(); } mutex_.Unlock(); } private: bool init_; Mutex mutex_; }; #define LEVELDB_ONCE_INIT false extern void InitOnce(port::OnceType*, void (*initializer)()); // Storage for a lock-free pointer class AtomicPointer { private: void * rep_; public: AtomicPointer() : rep_(NULL) { } explicit AtomicPointer(void* v); void* Acquire_Load() const; void Release_Store(void* v); void* NoBarrier_Load() const; void NoBarrier_Store(void* v); }; inline bool Snappy_Compress(const char* input, size_t length, ::std::string* output) { #ifdef SNAPPY output->resize(snappy::MaxCompressedLength(length)); size_t outlen; snappy::RawCompress(input, length, &(*output)[0], &outlen); output->resize(outlen); return true; #endif return false; } inline bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result) { #ifdef SNAPPY return snappy::GetUncompressedLength(input, length, result); #else return false; #endif } inline bool Snappy_Uncompress(const char* input, size_t length, char* output) { #ifdef SNAPPY return snappy::RawUncompress(input, length, output); #else return false; #endif } inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { return false; } +bool HasAcceleratedCRC32C(); uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size); } } #endif // STORAGE_LEVELDB_PORT_PORT_WIN_H_ diff --git a/src/leveldb/util/crc32c.cc b/src/leveldb/util/crc32c.cc index edd61cfd6..b3f40eeee 100644 --- a/src/leveldb/util/crc32c.cc +++ b/src/leveldb/util/crc32c.cc @@ -1,350 +1,354 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // A portable implementation of crc32c, optimized to handle // four bytes at a time. #include "util/crc32c.h" #include #include "port/port.h" #include "util/coding.h" namespace leveldb { namespace crc32c { static const uint32_t table0_[256] = { 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 }; static const uint32_t table1_[256] = { 0x00000000, 0x13a29877, 0x274530ee, 0x34e7a899, 0x4e8a61dc, 0x5d28f9ab, 0x69cf5132, 0x7a6dc945, 0x9d14c3b8, 0x8eb65bcf, 0xba51f356, 0xa9f36b21, 0xd39ea264, 0xc03c3a13, 0xf4db928a, 0xe7790afd, 0x3fc5f181, 0x2c6769f6, 0x1880c16f, 0x0b225918, 0x714f905d, 0x62ed082a, 0x560aa0b3, 0x45a838c4, 0xa2d13239, 0xb173aa4e, 0x859402d7, 0x96369aa0, 0xec5b53e5, 0xfff9cb92, 0xcb1e630b, 0xd8bcfb7c, 0x7f8be302, 0x6c297b75, 0x58ced3ec, 0x4b6c4b9b, 0x310182de, 0x22a31aa9, 0x1644b230, 0x05e62a47, 0xe29f20ba, 0xf13db8cd, 0xc5da1054, 0xd6788823, 0xac154166, 0xbfb7d911, 0x8b507188, 0x98f2e9ff, 0x404e1283, 0x53ec8af4, 0x670b226d, 0x74a9ba1a, 0x0ec4735f, 0x1d66eb28, 0x298143b1, 0x3a23dbc6, 0xdd5ad13b, 0xcef8494c, 0xfa1fe1d5, 0xe9bd79a2, 0x93d0b0e7, 0x80722890, 0xb4958009, 0xa737187e, 0xff17c604, 0xecb55e73, 0xd852f6ea, 0xcbf06e9d, 0xb19da7d8, 0xa23f3faf, 0x96d89736, 0x857a0f41, 0x620305bc, 0x71a19dcb, 0x45463552, 0x56e4ad25, 0x2c896460, 0x3f2bfc17, 0x0bcc548e, 0x186eccf9, 0xc0d23785, 0xd370aff2, 0xe797076b, 0xf4359f1c, 0x8e585659, 0x9dface2e, 0xa91d66b7, 0xbabffec0, 0x5dc6f43d, 0x4e646c4a, 0x7a83c4d3, 0x69215ca4, 0x134c95e1, 0x00ee0d96, 0x3409a50f, 0x27ab3d78, 0x809c2506, 0x933ebd71, 0xa7d915e8, 0xb47b8d9f, 0xce1644da, 0xddb4dcad, 0xe9537434, 0xfaf1ec43, 0x1d88e6be, 0x0e2a7ec9, 0x3acdd650, 0x296f4e27, 0x53028762, 0x40a01f15, 0x7447b78c, 0x67e52ffb, 0xbf59d487, 0xacfb4cf0, 0x981ce469, 0x8bbe7c1e, 0xf1d3b55b, 0xe2712d2c, 0xd69685b5, 0xc5341dc2, 0x224d173f, 0x31ef8f48, 0x050827d1, 0x16aabfa6, 0x6cc776e3, 0x7f65ee94, 0x4b82460d, 0x5820de7a, 0xfbc3faf9, 0xe861628e, 0xdc86ca17, 0xcf245260, 0xb5499b25, 0xa6eb0352, 0x920cabcb, 0x81ae33bc, 0x66d73941, 0x7575a136, 0x419209af, 0x523091d8, 0x285d589d, 0x3bffc0ea, 0x0f186873, 0x1cbaf004, 0xc4060b78, 0xd7a4930f, 0xe3433b96, 0xf0e1a3e1, 0x8a8c6aa4, 0x992ef2d3, 0xadc95a4a, 0xbe6bc23d, 0x5912c8c0, 0x4ab050b7, 0x7e57f82e, 0x6df56059, 0x1798a91c, 0x043a316b, 0x30dd99f2, 0x237f0185, 0x844819fb, 0x97ea818c, 0xa30d2915, 0xb0afb162, 0xcac27827, 0xd960e050, 0xed8748c9, 0xfe25d0be, 0x195cda43, 0x0afe4234, 0x3e19eaad, 0x2dbb72da, 0x57d6bb9f, 0x447423e8, 0x70938b71, 0x63311306, 0xbb8de87a, 0xa82f700d, 0x9cc8d894, 0x8f6a40e3, 0xf50789a6, 0xe6a511d1, 0xd242b948, 0xc1e0213f, 0x26992bc2, 0x353bb3b5, 0x01dc1b2c, 0x127e835b, 0x68134a1e, 0x7bb1d269, 0x4f567af0, 0x5cf4e287, 0x04d43cfd, 0x1776a48a, 0x23910c13, 0x30339464, 0x4a5e5d21, 0x59fcc556, 0x6d1b6dcf, 0x7eb9f5b8, 0x99c0ff45, 0x8a626732, 0xbe85cfab, 0xad2757dc, 0xd74a9e99, 0xc4e806ee, 0xf00fae77, 0xe3ad3600, 0x3b11cd7c, 0x28b3550b, 0x1c54fd92, 0x0ff665e5, 0x759baca0, 0x663934d7, 0x52de9c4e, 0x417c0439, 0xa6050ec4, 0xb5a796b3, 0x81403e2a, 0x92e2a65d, 0xe88f6f18, 0xfb2df76f, 0xcfca5ff6, 0xdc68c781, 0x7b5fdfff, 0x68fd4788, 0x5c1aef11, 0x4fb87766, 0x35d5be23, 0x26772654, 0x12908ecd, 0x013216ba, 0xe64b1c47, 0xf5e98430, 0xc10e2ca9, 0xd2acb4de, 0xa8c17d9b, 0xbb63e5ec, 0x8f844d75, 0x9c26d502, 0x449a2e7e, 0x5738b609, 0x63df1e90, 0x707d86e7, 0x0a104fa2, 0x19b2d7d5, 0x2d557f4c, 0x3ef7e73b, 0xd98eedc6, 0xca2c75b1, 0xfecbdd28, 0xed69455f, 0x97048c1a, 0x84a6146d, 0xb041bcf4, 0xa3e32483 }; static const uint32_t table2_[256] = { 0x00000000, 0xa541927e, 0x4f6f520d, 0xea2ec073, 0x9edea41a, 0x3b9f3664, 0xd1b1f617, 0x74f06469, 0x38513ec5, 0x9d10acbb, 0x773e6cc8, 0xd27ffeb6, 0xa68f9adf, 0x03ce08a1, 0xe9e0c8d2, 0x4ca15aac, 0x70a27d8a, 0xd5e3eff4, 0x3fcd2f87, 0x9a8cbdf9, 0xee7cd990, 0x4b3d4bee, 0xa1138b9d, 0x045219e3, 0x48f3434f, 0xedb2d131, 0x079c1142, 0xa2dd833c, 0xd62de755, 0x736c752b, 0x9942b558, 0x3c032726, 0xe144fb14, 0x4405696a, 0xae2ba919, 0x0b6a3b67, 0x7f9a5f0e, 0xdadbcd70, 0x30f50d03, 0x95b49f7d, 0xd915c5d1, 0x7c5457af, 0x967a97dc, 0x333b05a2, 0x47cb61cb, 0xe28af3b5, 0x08a433c6, 0xade5a1b8, 0x91e6869e, 0x34a714e0, 0xde89d493, 0x7bc846ed, 0x0f382284, 0xaa79b0fa, 0x40577089, 0xe516e2f7, 0xa9b7b85b, 0x0cf62a25, 0xe6d8ea56, 0x43997828, 0x37691c41, 0x92288e3f, 0x78064e4c, 0xdd47dc32, 0xc76580d9, 0x622412a7, 0x880ad2d4, 0x2d4b40aa, 0x59bb24c3, 0xfcfab6bd, 0x16d476ce, 0xb395e4b0, 0xff34be1c, 0x5a752c62, 0xb05bec11, 0x151a7e6f, 0x61ea1a06, 0xc4ab8878, 0x2e85480b, 0x8bc4da75, 0xb7c7fd53, 0x12866f2d, 0xf8a8af5e, 0x5de93d20, 0x29195949, 0x8c58cb37, 0x66760b44, 0xc337993a, 0x8f96c396, 0x2ad751e8, 0xc0f9919b, 0x65b803e5, 0x1148678c, 0xb409f5f2, 0x5e273581, 0xfb66a7ff, 0x26217bcd, 0x8360e9b3, 0x694e29c0, 0xcc0fbbbe, 0xb8ffdfd7, 0x1dbe4da9, 0xf7908dda, 0x52d11fa4, 0x1e704508, 0xbb31d776, 0x511f1705, 0xf45e857b, 0x80aee112, 0x25ef736c, 0xcfc1b31f, 0x6a802161, 0x56830647, 0xf3c29439, 0x19ec544a, 0xbcadc634, 0xc85da25d, 0x6d1c3023, 0x8732f050, 0x2273622e, 0x6ed23882, 0xcb93aafc, 0x21bd6a8f, 0x84fcf8f1, 0xf00c9c98, 0x554d0ee6, 0xbf63ce95, 0x1a225ceb, 0x8b277743, 0x2e66e53d, 0xc448254e, 0x6109b730, 0x15f9d359, 0xb0b84127, 0x5a968154, 0xffd7132a, 0xb3764986, 0x1637dbf8, 0xfc191b8b, 0x595889f5, 0x2da8ed9c, 0x88e97fe2, 0x62c7bf91, 0xc7862def, 0xfb850ac9, 0x5ec498b7, 0xb4ea58c4, 0x11abcaba, 0x655baed3, 0xc01a3cad, 0x2a34fcde, 0x8f756ea0, 0xc3d4340c, 0x6695a672, 0x8cbb6601, 0x29faf47f, 0x5d0a9016, 0xf84b0268, 0x1265c21b, 0xb7245065, 0x6a638c57, 0xcf221e29, 0x250cde5a, 0x804d4c24, 0xf4bd284d, 0x51fcba33, 0xbbd27a40, 0x1e93e83e, 0x5232b292, 0xf77320ec, 0x1d5de09f, 0xb81c72e1, 0xccec1688, 0x69ad84f6, 0x83834485, 0x26c2d6fb, 0x1ac1f1dd, 0xbf8063a3, 0x55aea3d0, 0xf0ef31ae, 0x841f55c7, 0x215ec7b9, 0xcb7007ca, 0x6e3195b4, 0x2290cf18, 0x87d15d66, 0x6dff9d15, 0xc8be0f6b, 0xbc4e6b02, 0x190ff97c, 0xf321390f, 0x5660ab71, 0x4c42f79a, 0xe90365e4, 0x032da597, 0xa66c37e9, 0xd29c5380, 0x77ddc1fe, 0x9df3018d, 0x38b293f3, 0x7413c95f, 0xd1525b21, 0x3b7c9b52, 0x9e3d092c, 0xeacd6d45, 0x4f8cff3b, 0xa5a23f48, 0x00e3ad36, 0x3ce08a10, 0x99a1186e, 0x738fd81d, 0xd6ce4a63, 0xa23e2e0a, 0x077fbc74, 0xed517c07, 0x4810ee79, 0x04b1b4d5, 0xa1f026ab, 0x4bdee6d8, 0xee9f74a6, 0x9a6f10cf, 0x3f2e82b1, 0xd50042c2, 0x7041d0bc, 0xad060c8e, 0x08479ef0, 0xe2695e83, 0x4728ccfd, 0x33d8a894, 0x96993aea, 0x7cb7fa99, 0xd9f668e7, 0x9557324b, 0x3016a035, 0xda386046, 0x7f79f238, 0x0b899651, 0xaec8042f, 0x44e6c45c, 0xe1a75622, 0xdda47104, 0x78e5e37a, 0x92cb2309, 0x378ab177, 0x437ad51e, 0xe63b4760, 0x0c158713, 0xa954156d, 0xe5f54fc1, 0x40b4ddbf, 0xaa9a1dcc, 0x0fdb8fb2, 0x7b2bebdb, 0xde6a79a5, 0x3444b9d6, 0x91052ba8 }; static const uint32_t table3_[256] = { 0x00000000, 0xdd45aab8, 0xbf672381, 0x62228939, 0x7b2231f3, 0xa6679b4b, 0xc4451272, 0x1900b8ca, 0xf64463e6, 0x2b01c95e, 0x49234067, 0x9466eadf, 0x8d665215, 0x5023f8ad, 0x32017194, 0xef44db2c, 0xe964b13d, 0x34211b85, 0x560392bc, 0x8b463804, 0x924680ce, 0x4f032a76, 0x2d21a34f, 0xf06409f7, 0x1f20d2db, 0xc2657863, 0xa047f15a, 0x7d025be2, 0x6402e328, 0xb9474990, 0xdb65c0a9, 0x06206a11, 0xd725148b, 0x0a60be33, 0x6842370a, 0xb5079db2, 0xac072578, 0x71428fc0, 0x136006f9, 0xce25ac41, 0x2161776d, 0xfc24ddd5, 0x9e0654ec, 0x4343fe54, 0x5a43469e, 0x8706ec26, 0xe524651f, 0x3861cfa7, 0x3e41a5b6, 0xe3040f0e, 0x81268637, 0x5c632c8f, 0x45639445, 0x98263efd, 0xfa04b7c4, 0x27411d7c, 0xc805c650, 0x15406ce8, 0x7762e5d1, 0xaa274f69, 0xb327f7a3, 0x6e625d1b, 0x0c40d422, 0xd1057e9a, 0xaba65fe7, 0x76e3f55f, 0x14c17c66, 0xc984d6de, 0xd0846e14, 0x0dc1c4ac, 0x6fe34d95, 0xb2a6e72d, 0x5de23c01, 0x80a796b9, 0xe2851f80, 0x3fc0b538, 0x26c00df2, 0xfb85a74a, 0x99a72e73, 0x44e284cb, 0x42c2eeda, 0x9f874462, 0xfda5cd5b, 0x20e067e3, 0x39e0df29, 0xe4a57591, 0x8687fca8, 0x5bc25610, 0xb4868d3c, 0x69c32784, 0x0be1aebd, 0xd6a40405, 0xcfa4bccf, 0x12e11677, 0x70c39f4e, 0xad8635f6, 0x7c834b6c, 0xa1c6e1d4, 0xc3e468ed, 0x1ea1c255, 0x07a17a9f, 0xdae4d027, 0xb8c6591e, 0x6583f3a6, 0x8ac7288a, 0x57828232, 0x35a00b0b, 0xe8e5a1b3, 0xf1e51979, 0x2ca0b3c1, 0x4e823af8, 0x93c79040, 0x95e7fa51, 0x48a250e9, 0x2a80d9d0, 0xf7c57368, 0xeec5cba2, 0x3380611a, 0x51a2e823, 0x8ce7429b, 0x63a399b7, 0xbee6330f, 0xdcc4ba36, 0x0181108e, 0x1881a844, 0xc5c402fc, 0xa7e68bc5, 0x7aa3217d, 0x52a0c93f, 0x8fe56387, 0xedc7eabe, 0x30824006, 0x2982f8cc, 0xf4c75274, 0x96e5db4d, 0x4ba071f5, 0xa4e4aad9, 0x79a10061, 0x1b838958, 0xc6c623e0, 0xdfc69b2a, 0x02833192, 0x60a1b8ab, 0xbde41213, 0xbbc47802, 0x6681d2ba, 0x04a35b83, 0xd9e6f13b, 0xc0e649f1, 0x1da3e349, 0x7f816a70, 0xa2c4c0c8, 0x4d801be4, 0x90c5b15c, 0xf2e73865, 0x2fa292dd, 0x36a22a17, 0xebe780af, 0x89c50996, 0x5480a32e, 0x8585ddb4, 0x58c0770c, 0x3ae2fe35, 0xe7a7548d, 0xfea7ec47, 0x23e246ff, 0x41c0cfc6, 0x9c85657e, 0x73c1be52, 0xae8414ea, 0xcca69dd3, 0x11e3376b, 0x08e38fa1, 0xd5a62519, 0xb784ac20, 0x6ac10698, 0x6ce16c89, 0xb1a4c631, 0xd3864f08, 0x0ec3e5b0, 0x17c35d7a, 0xca86f7c2, 0xa8a47efb, 0x75e1d443, 0x9aa50f6f, 0x47e0a5d7, 0x25c22cee, 0xf8878656, 0xe1873e9c, 0x3cc29424, 0x5ee01d1d, 0x83a5b7a5, 0xf90696d8, 0x24433c60, 0x4661b559, 0x9b241fe1, 0x8224a72b, 0x5f610d93, 0x3d4384aa, 0xe0062e12, 0x0f42f53e, 0xd2075f86, 0xb025d6bf, 0x6d607c07, 0x7460c4cd, 0xa9256e75, 0xcb07e74c, 0x16424df4, 0x106227e5, 0xcd278d5d, 0xaf050464, 0x7240aedc, 0x6b401616, 0xb605bcae, 0xd4273597, 0x09629f2f, 0xe6264403, 0x3b63eebb, 0x59416782, 0x8404cd3a, 0x9d0475f0, 0x4041df48, 0x22635671, 0xff26fcc9, 0x2e238253, 0xf36628eb, 0x9144a1d2, 0x4c010b6a, 0x5501b3a0, 0x88441918, 0xea669021, 0x37233a99, 0xd867e1b5, 0x05224b0d, 0x6700c234, 0xba45688c, 0xa345d046, 0x7e007afe, 0x1c22f3c7, 0xc167597f, 0xc747336e, 0x1a0299d6, 0x782010ef, 0xa565ba57, 0xbc65029d, 0x6120a825, 0x0302211c, 0xde478ba4, 0x31035088, 0xec46fa30, 0x8e647309, 0x5321d9b1, 0x4a21617b, 0x9764cbc3, 0xf54642fa, 0x2803e842 }; // Used to fetch a naturally-aligned 32-bit word in little endian byte-order static inline uint32_t LE_LOAD32(const uint8_t *p) { return DecodeFixed32(reinterpret_cast(p)); } // Determine if the CPU running this program can accelerate the CRC32C // calculation. static bool CanAccelerateCRC32C() { + if (!port::HasAcceleratedCRC32C()) + return false; + + // Double-check that the accelerated implementation functions correctly. // port::AcceleretedCRC32C returns zero when unable to accelerate. static const char kTestCRCBuffer[] = "TestCRCBuffer"; static const char kBufSize = sizeof(kTestCRCBuffer) - 1; static const uint32_t kTestCRCValue = 0xdcbc59fa; return port::AcceleratedCRC32C(0, kTestCRCBuffer, kBufSize) == kTestCRCValue; } uint32_t Extend(uint32_t crc, const char* buf, size_t size) { static bool accelerate = CanAccelerateCRC32C(); if (accelerate) { return port::AcceleratedCRC32C(crc, buf, size); } const uint8_t *p = reinterpret_cast(buf); const uint8_t *e = p + size; uint32_t l = crc ^ 0xffffffffu; #define STEP1 do { \ int c = (l & 0xff) ^ *p++; \ l = table0_[c] ^ (l >> 8); \ } while (0) #define STEP4 do { \ uint32_t c = l ^ LE_LOAD32(p); \ p += 4; \ l = table3_[c & 0xff] ^ \ table2_[(c >> 8) & 0xff] ^ \ table1_[(c >> 16) & 0xff] ^ \ table0_[c >> 24]; \ } while (0) // Point x at first 4-byte aligned byte in string. This might be // just past the end of the string. const uintptr_t pval = reinterpret_cast(p); const uint8_t* x = reinterpret_cast(((pval + 3) >> 2) << 2); if (x <= e) { // Process bytes until finished or p is 4-byte aligned while (p != x) { STEP1; } } // Process bytes 16 at a time while ((e-p) >= 16) { STEP4; STEP4; STEP4; STEP4; } // Process bytes 4 at a time while ((e-p) >= 4) { STEP4; } // Process the last few bytes while (p != e) { STEP1; } #undef STEP4 #undef STEP1 return l ^ 0xffffffffu; } } // namespace crc32c } // namespace leveldb diff --git a/src/leveldb/util/logging.cc b/src/leveldb/util/logging.cc index ca6b32440..db6160c8f 100644 --- a/src/leveldb/util/logging.cc +++ b/src/leveldb/util/logging.cc @@ -1,72 +1,72 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "util/logging.h" #include #include #include #include #include "leveldb/env.h" #include "leveldb/slice.h" namespace leveldb { void AppendNumberTo(std::string* str, uint64_t num) { char buf[30]; snprintf(buf, sizeof(buf), "%llu", (unsigned long long) num); str->append(buf); } void AppendEscapedStringTo(std::string* str, const Slice& value) { for (size_t i = 0; i < value.size(); i++) { char c = value[i]; if (c >= ' ' && c <= '~') { str->push_back(c); } else { char buf[10]; snprintf(buf, sizeof(buf), "\\x%02x", static_cast(c) & 0xff); str->append(buf); } } } std::string NumberToString(uint64_t num) { std::string r; AppendNumberTo(&r, num); return r; } std::string EscapeString(const Slice& value) { std::string r; AppendEscapedStringTo(&r, value); return r; } bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { uint64_t v = 0; int digits = 0; while (!in->empty()) { - char c = (*in)[0]; + unsigned char c = (*in)[0]; if (c >= '0' && c <= '9') { ++digits; const int delta = (c - '0'); static const uint64_t kMaxUint64 = ~static_cast(0); if (v > kMaxUint64/10 || (v == kMaxUint64/10 && delta > kMaxUint64%10)) { // Overflow return false; } v = (v * 10) + delta; in->remove_prefix(1); } else { break; } } *val = v; return (digits > 0); } } // namespace leveldb