Revision 5c12a474f30b00b1a14527c194df44ab8cc72deb authored by Yanqin Jin on 31 August 2019, 01:27:43 UTC, committed by Yanqin Jin on 03 September 2019, 20:29:55 UTC
Summary: Before this PR, when the number of column families involved in a file ingestion exceeds 2, a bug in the looping logic prevents correct file number being assigned to each ingestion job. Also skip deleting non-existing hard links during cleanup-after-failure. Test plan (devserver) ``` $COMPILE_WITH_ASAN=1 make all $./external_sst_file_test --gtest_filter=ExternalSSTFileTest/ExternalSSTFileTest.IngestFilesIntoMultipleColumnFamilies_*/* $makke check ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/5760 Differential Revision: D17142982 Pulled By: riversand963 fbshipit-source-id: 06c1847a4e7a402647bcf28d124e70f2a0f9daf6
1 parent 6d113fc
merger_test.cc
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include <vector>
#include <string>
#include "table/merging_iterator.h"
#include "util/testharness.h"
#include "util/testutil.h"
namespace rocksdb {
class MergerTest : public testing::Test {
public:
MergerTest()
: icomp_(BytewiseComparator()),
rnd_(3),
merging_iterator_(nullptr),
single_iterator_(nullptr) {}
~MergerTest() override = default;
std::vector<std::string> GenerateStrings(size_t len, int string_len) {
std::vector<std::string> ret;
for (size_t i = 0; i < len; ++i) {
InternalKey ik(test::RandomHumanReadableString(&rnd_, string_len), 0,
ValueType::kTypeValue);
ret.push_back(ik.Encode().ToString(false));
}
return ret;
}
void AssertEquivalence() {
auto a = merging_iterator_.get();
auto b = single_iterator_.get();
if (!a->Valid()) {
ASSERT_TRUE(!b->Valid());
} else {
ASSERT_TRUE(b->Valid());
ASSERT_EQ(b->key().ToString(), a->key().ToString());
ASSERT_EQ(b->value().ToString(), a->value().ToString());
}
}
void SeekToRandom() {
InternalKey ik(test::RandomHumanReadableString(&rnd_, 5), 0,
ValueType::kTypeValue);
Seek(ik.Encode().ToString(false));
}
void Seek(std::string target) {
merging_iterator_->Seek(target);
single_iterator_->Seek(target);
}
void SeekToFirst() {
merging_iterator_->SeekToFirst();
single_iterator_->SeekToFirst();
}
void SeekToLast() {
merging_iterator_->SeekToLast();
single_iterator_->SeekToLast();
}
void Next(int times) {
for (int i = 0; i < times && merging_iterator_->Valid(); ++i) {
AssertEquivalence();
merging_iterator_->Next();
single_iterator_->Next();
}
AssertEquivalence();
}
void Prev(int times) {
for (int i = 0; i < times && merging_iterator_->Valid(); ++i) {
AssertEquivalence();
merging_iterator_->Prev();
single_iterator_->Prev();
}
AssertEquivalence();
}
void NextAndPrev(int times) {
for (int i = 0; i < times && merging_iterator_->Valid(); ++i) {
AssertEquivalence();
if (rnd_.OneIn(2)) {
merging_iterator_->Prev();
single_iterator_->Prev();
} else {
merging_iterator_->Next();
single_iterator_->Next();
}
}
AssertEquivalence();
}
void Generate(size_t num_iterators, size_t strings_per_iterator,
int letters_per_string) {
std::vector<InternalIterator*> small_iterators;
for (size_t i = 0; i < num_iterators; ++i) {
auto strings = GenerateStrings(strings_per_iterator, letters_per_string);
small_iterators.push_back(new test::VectorIterator(strings));
all_keys_.insert(all_keys_.end(), strings.begin(), strings.end());
}
merging_iterator_.reset(
NewMergingIterator(&icomp_, &small_iterators[0],
static_cast<int>(small_iterators.size())));
single_iterator_.reset(new test::VectorIterator(all_keys_));
}
InternalKeyComparator icomp_;
Random rnd_;
std::unique_ptr<InternalIterator> merging_iterator_;
std::unique_ptr<InternalIterator> single_iterator_;
std::vector<std::string> all_keys_;
};
TEST_F(MergerTest, SeekToRandomNextTest) {
Generate(1000, 50, 50);
for (int i = 0; i < 10; ++i) {
SeekToRandom();
AssertEquivalence();
Next(50000);
}
}
TEST_F(MergerTest, SeekToRandomNextSmallStringsTest) {
Generate(1000, 50, 2);
for (int i = 0; i < 10; ++i) {
SeekToRandom();
AssertEquivalence();
Next(50000);
}
}
TEST_F(MergerTest, SeekToRandomPrevTest) {
Generate(1000, 50, 50);
for (int i = 0; i < 10; ++i) {
SeekToRandom();
AssertEquivalence();
Prev(50000);
}
}
TEST_F(MergerTest, SeekToRandomRandomTest) {
Generate(200, 50, 50);
for (int i = 0; i < 3; ++i) {
SeekToRandom();
AssertEquivalence();
NextAndPrev(5000);
}
}
TEST_F(MergerTest, SeekToFirstTest) {
Generate(1000, 50, 50);
for (int i = 0; i < 10; ++i) {
SeekToFirst();
AssertEquivalence();
Next(50000);
}
}
TEST_F(MergerTest, SeekToLastTest) {
Generate(1000, 50, 50);
for (int i = 0; i < 10; ++i) {
SeekToLast();
AssertEquivalence();
Prev(50000);
}
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...