Revision 0d4a2b733070a1bd52f981313f9e17f126701407 authored by Yi Wu on 04 August 2017, 20:09:56 UTC, committed by Facebook Github Bot on 04 August 2017, 20:12:07 UTC
Summary: The FsyncFiles background job call Fsync() periodically for blob files. However it can access WritableFileWriter concurrently with a Put() or Write(). And WritableFileWriter does not support concurrent access. It will lead to WritableFileWriter buffer being flush with same content twice, and blob file end up corrupted. Fixing by simply let FsyncFiles hold write_mutex_. Closes https://github.com/facebook/rocksdb/pull/2685 Differential Revision: D5561908 Pulled By: yiwu-arbug fbshipit-source-id: f0bb5bcab0e05694e053b8c49eab43640721e872
1 parent 627c9f1
slice.cc
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <algorithm>
#include "rocksdb/slice_transform.h"
#include "rocksdb/slice.h"
#include "util/string_util.h"
#include <stdio.h>
namespace rocksdb {
namespace {
class FixedPrefixTransform : public SliceTransform {
private:
size_t prefix_len_;
std::string name_;
public:
explicit FixedPrefixTransform(size_t prefix_len)
: prefix_len_(prefix_len),
// Note that if any part of the name format changes, it will require
// changes on options_helper in order to make RocksDBOptionsParser work
// for the new change.
// TODO(yhchiang): move serialization / deserializaion code inside
// the class implementation itself.
name_("rocksdb.FixedPrefix." + ToString(prefix_len_)) {}
virtual const char* Name() const override { return name_.c_str(); }
virtual Slice Transform(const Slice& src) const override {
assert(InDomain(src));
return Slice(src.data(), prefix_len_);
}
virtual bool InDomain(const Slice& src) const override {
return (src.size() >= prefix_len_);
}
virtual bool InRange(const Slice& dst) const override {
return (dst.size() == prefix_len_);
}
virtual bool SameResultWhenAppended(const Slice& prefix) const override {
return InDomain(prefix);
}
};
class CappedPrefixTransform : public SliceTransform {
private:
size_t cap_len_;
std::string name_;
public:
explicit CappedPrefixTransform(size_t cap_len)
: cap_len_(cap_len),
// Note that if any part of the name format changes, it will require
// changes on options_helper in order to make RocksDBOptionsParser work
// for the new change.
// TODO(yhchiang): move serialization / deserializaion code inside
// the class implementation itself.
name_("rocksdb.CappedPrefix." + ToString(cap_len_)) {}
virtual const char* Name() const override { return name_.c_str(); }
virtual Slice Transform(const Slice& src) const override {
assert(InDomain(src));
return Slice(src.data(), std::min(cap_len_, src.size()));
}
virtual bool InDomain(const Slice& src) const override { return true; }
virtual bool InRange(const Slice& dst) const override {
return (dst.size() <= cap_len_);
}
virtual bool SameResultWhenAppended(const Slice& prefix) const override {
return prefix.size() >= cap_len_;
}
};
class NoopTransform : public SliceTransform {
public:
explicit NoopTransform() { }
virtual const char* Name() const override { return "rocksdb.Noop"; }
virtual Slice Transform(const Slice& src) const override { return src; }
virtual bool InDomain(const Slice& src) const override { return true; }
virtual bool InRange(const Slice& dst) const override { return true; }
virtual bool SameResultWhenAppended(const Slice& prefix) const override {
return false;
}
};
}
// 2 small internal utility functions, for efficient hex conversions
// and no need for snprintf, toupper etc...
// Originally from wdt/util/EncryptionUtils.cpp - for ToString(true)/DecodeHex:
char toHex(unsigned char v) {
if (v <= 9) {
return '0' + v;
}
return 'A' + v - 10;
}
// most of the code is for validation/error check
int fromHex(char c) {
// toupper:
if (c >= 'a' && c <= 'f') {
c -= ('a' - 'A'); // aka 0x20
}
// validation
if (c < '0' || (c > '9' && (c < 'A' || c > 'F'))) {
return -1; // invalid not 0-9A-F hex char
}
if (c <= '9') {
return c - '0';
}
return c - 'A' + 10;
}
Slice::Slice(const SliceParts& parts, std::string* buf) {
size_t length = 0;
for (int i = 0; i < parts.num_parts; ++i) {
length += parts.parts[i].size();
}
buf->reserve(length);
for (int i = 0; i < parts.num_parts; ++i) {
buf->append(parts.parts[i].data(), parts.parts[i].size());
}
data_ = buf->data();
size_ = buf->size();
}
// Return a string that contains the copy of the referenced data.
std::string Slice::ToString(bool hex) const {
std::string result; // RVO/NRVO/move
if (hex) {
result.reserve(2 * size_);
for (size_t i = 0; i < size_; ++i) {
unsigned char c = data_[i];
result.push_back(toHex(c >> 4));
result.push_back(toHex(c & 0xf));
}
return result;
} else {
result.assign(data_, size_);
return result;
}
}
// Originally from rocksdb/utilities/ldb_cmd.h
bool Slice::DecodeHex(std::string* result) const {
std::string::size_type len = size_;
if (len % 2) {
// Hex string must be even number of hex digits to get complete bytes back
return false;
}
if (!result) {
return false;
}
result->clear();
result->reserve(len / 2);
for (size_t i = 0; i < len;) {
int h1 = fromHex(data_[i++]);
if (h1 < 0) {
return false;
}
int h2 = fromHex(data_[i++]);
if (h2 < 0) {
return false;
}
result->push_back((h1 << 4) | h2);
}
return true;
}
const SliceTransform* NewFixedPrefixTransform(size_t prefix_len) {
return new FixedPrefixTransform(prefix_len);
}
const SliceTransform* NewCappedPrefixTransform(size_t cap_len) {
return new CappedPrefixTransform(cap_len);
}
const SliceTransform* NewNoopTransform() {
return new NoopTransform;
}
} // namespace rocksdb
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...