Revision 346d1069c3031468cf5cb06299f018a873c9fb33 authored by Daniel Black on 26 June 2018, 05:42:35 UTC, committed by Facebook Github Bot on 26 June 2018, 05:58:19 UTC
Summary: Pinned the alignment of StatisticsData to the cacheline size rather than just extending its size (which could go over two cache lines)if unaligned in allocation. Avoid compile errors in the process as per individual commit messages. strengthen static_assert to CACHELINE rather than the highest common multiple. Closes https://github.com/facebook/rocksdb/pull/4036 Differential Revision: D8582844 Pulled By: yiwu-arbug fbshipit-source-id: 363c37029f28e6093e06c60b987bca9aa204bc71
1 parent 6d454d7
version_builder_test.cc
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include <string>
#include "db/version_edit.h"
#include "db/version_set.h"
#include "util/logging.h"
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/testutil.h"
namespace rocksdb {
class VersionBuilderTest : public testing::Test {
public:
const Comparator* ucmp_;
InternalKeyComparator icmp_;
Options options_;
ImmutableCFOptions ioptions_;
MutableCFOptions mutable_cf_options_;
VersionStorageInfo vstorage_;
uint32_t file_num_;
CompactionOptionsFIFO fifo_options_;
std::vector<uint64_t> size_being_compacted_;
VersionBuilderTest()
: ucmp_(BytewiseComparator()),
icmp_(ucmp_),
ioptions_(options_),
mutable_cf_options_(options_),
vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
nullptr, false),
file_num_(1) {
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
size_being_compacted_.resize(options_.num_levels);
}
~VersionBuilderTest() {
for (int i = 0; i < vstorage_.num_levels(); i++) {
for (auto* f : vstorage_.LevelFiles(i)) {
if (--f->refs == 0) {
delete f;
}
}
}
}
InternalKey GetInternalKey(const char* ukey,
SequenceNumber smallest_seq = 100) {
return InternalKey(ukey, smallest_seq, kTypeValue);
}
void Add(int level, uint32_t file_number, const char* smallest,
const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100,
uint64_t num_entries = 0, uint64_t num_deletions = 0,
bool sampled = false, SequenceNumber smallest_seqno = 0,
SequenceNumber largest_seqno = 0) {
assert(level < vstorage_.num_levels());
FileMetaData* f = new FileMetaData;
f->fd = FileDescriptor(file_number, path_id, file_size);
f->smallest = GetInternalKey(smallest, smallest_seq);
f->largest = GetInternalKey(largest, largest_seq);
f->smallest_seqno = smallest_seqno;
f->largest_seqno = largest_seqno;
f->compensated_file_size = file_size;
f->refs = 0;
f->num_entries = num_entries;
f->num_deletions = num_deletions;
vstorage_.AddFile(level, f);
if (sampled) {
f->init_stats_from_file = true;
vstorage_.UpdateAccumulatedStats(f);
}
}
void UpdateVersionStorageInfo() {
vstorage_.UpdateFilesByCompactionPri(ioptions_.compaction_pri);
vstorage_.UpdateNumNonEmptyLevels();
vstorage_.GenerateFileIndexer();
vstorage_.GenerateLevelFilesBrief();
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
vstorage_.GenerateLevel0NonOverlapping();
vstorage_.SetFinalized();
}
};
void UnrefFilesInVersion(VersionStorageInfo* new_vstorage) {
for (int i = 0; i < new_vstorage->num_levels(); i++) {
for (auto* f : new_vstorage->LevelFiles(i)) {
if (--f->refs == 0) {
delete f;
}
}
}
}
TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
Add(0, 1U, "150", "200", 100U);
Add(1, 66U, "150", "200", 100U);
Add(1, 88U, "201", "300", 100U);
Add(2, 6U, "150", "179", 100U);
Add(2, 7U, "180", "220", 100U);
Add(2, 8U, "221", "300", 100U);
Add(3, 26U, "150", "170", 100U);
Add(3, 27U, "171", "179", 100U);
Add(3, 28U, "191", "220", 100U);
Add(3, 29U, "221", "300", 100U);
UpdateVersionStorageInfo();
VersionEdit version_edit;
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
GetInternalKey("350"), 200, 200, false);
version_edit.DeleteFile(3, 27U);
EnvOptions env_options;
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
kCompactionStyleLevel, nullptr, false);
version_builder.Apply(&version_edit);
version_builder.SaveTo(&new_vstorage);
ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2));
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3));
UnrefFilesInVersion(&new_vstorage);
}
TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
ioptions_.level_compaction_dynamic_level_bytes = true;
Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U);
Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U);
Add(4, 6U, "150", "179", 100U);
Add(4, 7U, "180", "220", 100U);
Add(4, 8U, "221", "300", 100U);
Add(5, 26U, "150", "170", 100U);
Add(5, 27U, "171", "179", 100U);
UpdateVersionStorageInfo();
VersionEdit version_edit;
version_edit.AddFile(3, 666, 0, 100U, GetInternalKey("301"),
GetInternalKey("350"), 200, 200, false);
version_edit.DeleteFile(0, 1U);
version_edit.DeleteFile(0, 88U);
EnvOptions env_options;
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
kCompactionStyleLevel, nullptr, false);
version_builder.Apply(&version_edit);
version_builder.SaveTo(&new_vstorage);
ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
ASSERT_EQ(100U, new_vstorage.NumLevelBytes(3));
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(4));
ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5));
UnrefFilesInVersion(&new_vstorage);
}
TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
ioptions_.level_compaction_dynamic_level_bytes = true;
Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U);
Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U);
Add(4, 6U, "150", "179", 100U);
Add(4, 7U, "180", "220", 100U);
Add(4, 8U, "221", "300", 100U);
Add(5, 26U, "150", "170", 100U);
Add(5, 27U, "171", "179", 100U);
UpdateVersionStorageInfo();
VersionEdit version_edit;
version_edit.AddFile(4, 666, 0, 100U, GetInternalKey("301"),
GetInternalKey("350"), 200, 200, false);
version_edit.DeleteFile(0, 1U);
version_edit.DeleteFile(0, 88U);
version_edit.DeleteFile(4, 6U);
version_edit.DeleteFile(4, 7U);
version_edit.DeleteFile(4, 8U);
EnvOptions env_options;
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
kCompactionStyleLevel, nullptr, false);
version_builder.Apply(&version_edit);
version_builder.SaveTo(&new_vstorage);
ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
ASSERT_EQ(100U, new_vstorage.NumLevelBytes(4));
ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5));
UnrefFilesInVersion(&new_vstorage);
}
TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
UpdateVersionStorageInfo();
VersionEdit version_edit;
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
GetInternalKey("350"), 200, 200, false);
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
GetInternalKey("450"), 200, 200, false);
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
GetInternalKey("650"), 200, 200, false);
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
GetInternalKey("550"), 200, 200, false);
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
GetInternalKey("750"), 200, 200, false);
EnvOptions env_options;
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
kCompactionStyleLevel, nullptr, false);
version_builder.Apply(&version_edit);
version_builder.SaveTo(&new_vstorage);
ASSERT_EQ(500U, new_vstorage.NumLevelBytes(2));
UnrefFilesInVersion(&new_vstorage);
}
TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
UpdateVersionStorageInfo();
EnvOptions env_options;
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
kCompactionStyleLevel, nullptr, false);
VersionEdit version_edit;
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
GetInternalKey("350"), 200, 200, false);
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
GetInternalKey("450"), 200, 200, false);
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
GetInternalKey("650"), 200, 200, false);
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
GetInternalKey("550"), 200, 200, false);
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
GetInternalKey("750"), 200, 200, false);
version_builder.Apply(&version_edit);
VersionEdit version_edit2;
version_edit.AddFile(2, 808, 0, 100U, GetInternalKey("901"),
GetInternalKey("950"), 200, 200, false);
version_edit2.DeleteFile(2, 616);
version_edit2.DeleteFile(2, 636);
version_edit.AddFile(2, 806, 0, 100U, GetInternalKey("801"),
GetInternalKey("850"), 200, 200, false);
version_builder.Apply(&version_edit2);
version_builder.SaveTo(&new_vstorage);
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(2));
UnrefFilesInVersion(&new_vstorage);
}
TEST_F(VersionBuilderTest, EstimatedActiveKeys) {
const uint32_t kTotalSamples = 20;
const uint32_t kNumLevels = 5;
const uint32_t kFilesPerLevel = 8;
const uint32_t kNumFiles = kNumLevels * kFilesPerLevel;
const uint32_t kEntriesPerFile = 1000;
const uint32_t kDeletionsPerFile = 100;
for (uint32_t i = 0; i < kNumFiles; ++i) {
Add(static_cast<int>(i / kFilesPerLevel), i + 1,
ToString((i + 100) * 1000).c_str(),
ToString((i + 100) * 1000 + 999).c_str(),
100U, 0, 100, 100,
kEntriesPerFile, kDeletionsPerFile,
(i < kTotalSamples));
}
// minus 2X for the number of deletion entries because:
// 1x for deletion entry does not count as a data entry.
// 1x for each deletion entry will actually remove one data entry.
ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(),
(kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles);
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
Computing file changes ...
