Revision 85dbdf2586d04d0d833592c26cbede75d8c923d3 authored by Peter Dillinger on 13 March 2020, 20:36:45 UTC, committed by Facebook GitHub Bot on 13 March 2020, 20:39:03 UTC
Summary: After we had a lot of failures with maven.org downloads, we wanted an alternative location for downloading binary dependencies. Hosting them through github would have been good in terms of organizational and network dependencies, but that approach seems to be awkward (fake releases, so would need a 'rocksdb-deps' repo) and strangely complicated for Facebook policy on open source repositories. This commit moves the downloads (that are not officially hosted by others on github) from my personal rocksdb fork to an S3 bucket owned by the Facebook RocksDB AWS account. Facebook employees can access this through an internal tool, and we should be able to grant permission to outside collaborators. Assuming this works out, I will back-port to older branches to stabilize their CI testing as well. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6526 Test Plan: CI Differential Revision: D20430130 Pulled By: pdillinger fbshipit-source-id: df52394a65e0a57942db3039bdaade8a4d520cb2
1 parent 5c30e6c
compaction_filter_example.cc
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include <rocksdb/compaction_filter.h>
#include <rocksdb/db.h>
#include <rocksdb/merge_operator.h>
#include <rocksdb/options.h>
class MyMerge : public ROCKSDB_NAMESPACE::MergeOperator {
public:
virtual bool FullMergeV2(const MergeOperationInput& merge_in,
MergeOperationOutput* merge_out) const override {
merge_out->new_value.clear();
if (merge_in.existing_value != nullptr) {
merge_out->new_value.assign(merge_in.existing_value->data(),
merge_in.existing_value->size());
}
for (const ROCKSDB_NAMESPACE::Slice& m : merge_in.operand_list) {
fprintf(stderr, "Merge(%s)\n", m.ToString().c_str());
// the compaction filter filters out bad values
assert(m.ToString() != "bad");
merge_out->new_value.assign(m.data(), m.size());
}
return true;
}
const char* Name() const override { return "MyMerge"; }
};
class MyFilter : public ROCKSDB_NAMESPACE::CompactionFilter {
public:
bool Filter(int level, const ROCKSDB_NAMESPACE::Slice& key,
const ROCKSDB_NAMESPACE::Slice& existing_value,
std::string* new_value, bool* value_changed) const override {
fprintf(stderr, "Filter(%s)\n", key.ToString().c_str());
++count_;
assert(*value_changed == false);
return false;
}
bool FilterMergeOperand(
int level, const ROCKSDB_NAMESPACE::Slice& key,
const ROCKSDB_NAMESPACE::Slice& existing_value) const override {
fprintf(stderr, "FilterMerge(%s)\n", key.ToString().c_str());
++merge_count_;
return existing_value == "bad";
}
const char* Name() const override { return "MyFilter"; }
mutable int count_ = 0;
mutable int merge_count_ = 0;
};
int main() {
ROCKSDB_NAMESPACE::DB* raw_db;
ROCKSDB_NAMESPACE::Status status;
MyFilter filter;
int ret = system("rm -rf /tmp/rocksmergetest");
if (ret != 0) {
fprintf(stderr, "Error deleting /tmp/rocksmergetest, code: %d\n", ret);
return ret;
}
ROCKSDB_NAMESPACE::Options options;
options.create_if_missing = true;
options.merge_operator.reset(new MyMerge);
options.compaction_filter = &filter;
status = ROCKSDB_NAMESPACE::DB::Open(options, "/tmp/rocksmergetest", &raw_db);
assert(status.ok());
std::unique_ptr<ROCKSDB_NAMESPACE::DB> db(raw_db);
ROCKSDB_NAMESPACE::WriteOptions wopts;
db->Merge(wopts, "0", "bad"); // This is filtered out
db->Merge(wopts, "1", "data1");
db->Merge(wopts, "1", "bad");
db->Merge(wopts, "1", "data2");
db->Merge(wopts, "1", "bad");
db->Merge(wopts, "3", "data3");
db->CompactRange(ROCKSDB_NAMESPACE::CompactRangeOptions(), nullptr, nullptr);
fprintf(stderr, "filter.count_ = %d\n", filter.count_);
assert(filter.count_ == 0);
fprintf(stderr, "filter.merge_count_ = %d\n", filter.merge_count_);
assert(filter.merge_count_ == 6);
}
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...