Revision d3d20dcdca9dc79d893a03dfa611fb1055c28f96 authored by Sagar Vemuri on 11 April 2019, 02:24:25 UTC, committed by Facebook Github Bot on 11 April 2019, 02:31:18 UTC
Summary: Introducing Periodic Compactions. This feature allows all the files in a CF to be periodically compacted. It could help in catching any corruptions that could creep into the DB proactively as every file is constantly getting re-compacted. And also, of course, it helps to cleanup data older than certain threshold. - Introduced a new option `periodic_compaction_time` to control how long a file can live without being compacted in a CF. - This works across all levels. - The files are put in the same level after going through the compaction. (Related files in the same level are picked up as `ExpandInputstoCleanCut` is used). - Compaction filters, if any, are invoked as usual. - A new table property, `file_creation_time`, is introduced to implement this feature. This property is set to the time at which the SST file was created (and that time is given by the underlying Env/OS). This feature can be enabled on its own, or in conjunction with `ttl`. It is possible to set a different time threshold for the bottom level when used in conjunction with ttl. Since `ttl` works only on 0 to last but one levels, you could set `ttl` to, say, 1 day, and `periodic_compaction_time` to, say, 7 days. Since `ttl < periodic_compaction_time` all files in last but one levels keep getting picked up based on ttl, and almost never based on periodic_compaction_time. The files in the bottom level get picked up for compaction based on `periodic_compaction_time`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5166 Differential Revision: D14884441 Pulled By: sagar0 fbshipit-source-id: 408426cbacb409c06386a98632dcf90bfa1bda47
1 parent ef0fc1b
sst_file_writer_collectors.h
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include <string>
#include "db/dbformat.h"
#include "db/table_properties_collector.h"
#include "rocksdb/types.h"
#include "util/string_util.h"
namespace rocksdb {
// Table Properties that are specific to tables created by SstFileWriter.
struct ExternalSstFilePropertyNames {
// value of this property is a fixed uint32 number.
static const std::string kVersion;
// value of this property is a fixed uint64 number.
static const std::string kGlobalSeqno;
};
// PropertiesCollector used to add properties specific to tables
// generated by SstFileWriter
class SstFileWriterPropertiesCollector : public IntTblPropCollector {
public:
explicit SstFileWriterPropertiesCollector(int32_t version,
SequenceNumber global_seqno)
: version_(version), global_seqno_(global_seqno) {}
virtual Status InternalAdd(const Slice& /*key*/, const Slice& /*value*/,
uint64_t /*file_size*/) override {
// Intentionally left blank. Have no interest in collecting stats for
// individual key/value pairs.
return Status::OK();
}
virtual void BlockAdd(uint64_t /* blockRawBytes */,
uint64_t /* blockCompressedBytesFast */,
uint64_t /* blockCompressedBytesSlow */) override {
// Intentionally left blank. No interest in collecting stats for
// blocks.
return;
}
virtual Status Finish(UserCollectedProperties* properties) override {
// File version
std::string version_val;
PutFixed32(&version_val, static_cast<uint32_t>(version_));
properties->insert({ExternalSstFilePropertyNames::kVersion, version_val});
// Global Sequence number
std::string seqno_val;
PutFixed64(&seqno_val, static_cast<uint64_t>(global_seqno_));
properties->insert({ExternalSstFilePropertyNames::kGlobalSeqno, seqno_val});
return Status::OK();
}
virtual const char* Name() const override {
return "SstFileWriterPropertiesCollector";
}
virtual UserCollectedProperties GetReadableProperties() const override {
return {{ExternalSstFilePropertyNames::kVersion, ToString(version_)}};
}
private:
int32_t version_;
SequenceNumber global_seqno_;
};
class SstFileWriterPropertiesCollectorFactory
: public IntTblPropCollectorFactory {
public:
explicit SstFileWriterPropertiesCollectorFactory(int32_t version,
SequenceNumber global_seqno)
: version_(version), global_seqno_(global_seqno) {}
virtual IntTblPropCollector* CreateIntTblPropCollector(
uint32_t /*column_family_id*/) override {
return new SstFileWriterPropertiesCollector(version_, global_seqno_);
}
virtual const char* Name() const override {
return "SstFileWriterPropertiesCollector";
}
private:
int32_t version_;
SequenceNumber global_seqno_;
};
} // namespace rocksdb
Computing file changes ...