Skip to main content

Browse the archive

Revision 2d9efc9ab2475876270741f0592da13eef44096b authored by Cheng Chang on 12 March 2020, 01:36:43 UTC, committed by Facebook GitHub Bot on 12 March 2020, 01:40:05 UTC 
Cache result of GetLogicalBufferSize in Linux (#6457)
 
Summary:
In Linux, when reopening DB with many SST files, profiling shows that 100% system cpu time spent for a couple of seconds for `GetLogicalBufferSize`. This slows down MyRocks' recovery time when site is down.

This PR introduces two new APIs:
1. `Env::RegisterDbPaths` and `Env::UnregisterDbPaths` lets `DB` tell the env when it starts or stops using its database directories . The `PosixFileSystem` takes this opportunity to set up a cache from database directories to the corresponding logical block sizes.
2. `LogicalBlockSizeCache` is defined only for OS_LINUX to cache the logical block sizes.

Other modifications:
1. rename `logical buffer size` to `logical block size` to be consistent with Linux terms.
2. declare `GetLogicalBlockSize` in `PosixHelper` to expose it to `PosixFileSystem`.
3. change the functions `IOError` and `IOStatus` in `env/io_posix.h` to have external linkage since they are used in other translation units too.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6457

Test Plan:
1. A new unit test is added for `LogicalBlockSizeCache` in `env/io_posix_test.cc`.
2. A new integration test is added for `DB` operations related to the cache in `db/db_logical_block_size_cache_test.cc`.

`make check`

Differential Revision: D20131243

Pulled By: cheng-chang

fbshipit-source-id: 3077c50f8065c0bffb544d8f49fb10bba9408d04
1 parent 331e619
Raw File
appveyor.yml 
version: 1.0.{build}

image: Visual Studio 2017

environment:
  JAVA_HOME: C:\Program Files\Java\jdk1.8.0
  THIRDPARTY_HOME: $(APPVEYOR_BUILD_FOLDER)\thirdparty
  SNAPPY_HOME: $(THIRDPARTY_HOME)\snappy-1.1.7
  SNAPPY_INCLUDE: $(SNAPPY_HOME);$(SNAPPY_HOME)\build
  SNAPPY_LIB_DEBUG: $(SNAPPY_HOME)\build\Debug\snappy.lib
  SNAPPY_LIB_RELEASE: $(SNAPPY_HOME)\build\Release\snappy.lib
  LZ4_HOME: $(THIRDPARTY_HOME)\lz4-1.8.3
  LZ4_INCLUDE: $(LZ4_HOME)\lib
  LZ4_LIB_DEBUG: $(LZ4_HOME)\visual\VS2010\bin\x64_Debug\liblz4_static.lib
  LZ4_LIB_RELEASE: $(LZ4_HOME)\visual\VS2010\bin\x64_Release\liblz4_static.lib
  ZSTD_HOME: $(THIRDPARTY_HOME)\zstd-1.4.0
  ZSTD_INCLUDE: $(ZSTD_HOME)\lib;$(ZSTD_HOME)\lib\dictBuilder
  ZSTD_LIB_DEBUG: $(ZSTD_HOME)\build\VS2010\bin\x64_Debug\libzstd_static.lib
  ZSTD_LIB_RELEASE: $(ZSTD_HOME)\build\VS2010\bin\x64_Release\libzstd_static.lib
  matrix:
    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
      CMAKE_GENERATOR: Visual Studio 14 Win64
      DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE\devenv.com
    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
      CMAKE_GENERATOR: Visual Studio 15 Win64
      DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\devenv.com

install:
  - md %THIRDPARTY_HOME%
  - echo "Building Snappy dependency..."
  - cd %THIRDPARTY_HOME%
  - curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
  - unzip snappy-1.1.7.zip
  - cd snappy-1.1.7
  - mkdir build
  - cd build
  - cmake -G "%CMAKE_GENERATOR%" ..
  - msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64
  - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64
  - echo "Building LZ4 dependency..."
  - cd %THIRDPARTY_HOME%
  - curl --fail --silent --show-error --output lz4-1.8.3.zip --location https://github.com/lz4/lz4/archive/v1.8.3.zip
  - unzip lz4-1.8.3.zip
  - cd lz4-1.8.3\visual\VS2010
  - ps: $CMD="$Env:DEV_ENV"; & $CMD lz4.sln /upgrade
  - msbuild lz4.sln /p:Configuration=Debug /p:Platform=x64
  - msbuild lz4.sln /p:Configuration=Release /p:Platform=x64
  - echo "Building ZStd dependency..."
  - cd %THIRDPARTY_HOME%
  - curl --fail --silent --show-error --output zstd-1.4.0.zip --location https://github.com/facebook/zstd/archive/v1.4.0.zip
  - unzip zstd-1.4.0.zip
  - cd zstd-1.4.0\build\VS2010
  - ps: $CMD="$Env:DEV_ENV"; & $CMD zstd.sln /upgrade
  - msbuild zstd.sln /p:Configuration=Debug /p:Platform=x64
  - msbuild zstd.sln /p:Configuration=Release /p:Platform=x64

before_build:
  - md %APPVEYOR_BUILD_FOLDER%\build
  - cd %APPVEYOR_BUILD_FOLDER%\build
  - cmake -G "%CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 ..
  - cd ..

build:
  project: build\rocksdb.sln
  parallel: true
  verbosity: normal

test:

test_script:
  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test,db_merge_operand_test -Concurrency 8

on_failure:
  - cmd: 7z a build-failed.zip %APPVEYOR_BUILD_FOLDER%\build\ && appveyor PushArtifact build-failed.zip
back to top