Skip to main content

Browse the archive

Revision 33323f21117e5bb8cf72569302f109aabb387bb9 authored by Mark Callaghan on 14 September 2012, 19:35:02 UTC, committed by Mark Callaghan on 14 September 2012, 23:43:50 UTC 
Remove use of mmap for random reads
 
Summary:
Reads via mmap on concurrent workloads are much slower than pread.
For example on a 24-core server with storage that can do 100k IOPS or more
I can get no more than 10k IOPS with mmap reads and 32+ threads.

Test Plan: db_bench benchmarks

Reviewers: dhruba, heyongqiang

Reviewed By: heyongqiang

Differential Revision: https://reviews.facebook.net/D5433
1 parent fa29f82
Raw File
scribe_logger.h 
#ifndef SCRIBE_LOGGER_H_
#define SCRIBE_LOGGER_H_

#include "scribe/if/gen-cpp/scribe.h"
#include "scribe/if/gen-cpp/scribe_types.h"
#include "thrift/lib/cpp/protocol/TProtocol.h"
#include "thrift/lib/cpp/transport/TSocket.h"
#include "thrift/lib/cpp/protocol/TBinaryProtocol.h"
#include "thrift/lib/cpp/transport/TBufferTransports.h"

#include "leveldb/env.h"
#include "port/port.h"
#include "util/stats_logger.h"

#include "boost/lexical_cast.hpp"

using namespace Tleveldb;
using Tleveldb::scribeClient;

using namespace apache::thrift;
using namespace apache::thrift::protocol;
using namespace apache::thrift::transport;
using boost::shared_ptr;


using namespace  ::Tleveldb;

namespace leveldb {

class ScribeLogger : public StatsLogger{
private:
  std::string host_;
  int port_;
  int batch_size_;

  scribeClient* scribe_client_;
  port::Mutex logger_mutex_;

  int retry_times_;
  uint32_t  retry_intervals_;

  void MakeScribeMessage(std::string& output, std::vector<std::string>& cols);

public:

  static const std::string COL_SEPERATOR;
  static const std::string DEPLOY_STATS_CATEGORY;

  ScribeLogger(const std::string& host, int port,
      int retry_times=3, uint32_t retry_intervals=1000000);
  virtual ~ScribeLogger();

  virtual void Log(const std::string& category, const std::string& message);

  virtual void Log_Deploy_Stats(
      const std::string& db_version,
      const std::string& machine_info,
      const std::string& data_dir,
      const uint64_t data_size,
      const uint32_t file_number,
      const std::string& data_size_per_level,
      const std::string& file_number_per_level,
      const int64_t& ts_unix
      );

};
}

#endif /* SCRIBE_LOGGER_H_ */
back to top