Revision fb09ef05dcee8fa0ccaebf88ef1992858ea5a083 authored by Yanqin Jin on 21 March 2020, 02:17:54 UTC, committed by Facebook GitHub Bot on 21 March 2020, 02:30:48 UTC
Summary: There are situations when RocksDB tries to recover, but the db is in an inconsistent state due to SST files referenced in the MANIFEST being missing. In this case, previous RocksDB will just fail the recovery and return a non-ok status. This PR enables another possibility. During recovery, RocksDB checks possible MANIFEST files, and try to recover to the most recent state without missing table file. `VersionSet::Recover()` applies version edits incrementally and "materializes" a version only when this version does not reference any missing table file. After processing the entire MANIFEST, the version created last will be the latest version. `DBImpl::Recover()` calls `VersionSet::Recover()`. Afterwards, WAL replay will *not* be performed. To use this capability, set `options.best_efforts_recovery = true` when opening the db. Best-efforts recovery is currently incompatible with atomic flush. Test plan (on devserver): ``` $make check $COMPILE_WITH_ASAN=1 make all && make check ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/6334 Reviewed By: anand1976 Differential Revision: D19778960 Pulled By: riversand963 fbshipit-source-id: c27ea80f29bc952e7d3311ecf5ee9c54393b40a8
1 parent 4fc2166
.travis.yml
dist: xenial
language: cpp
os:
- linux
- osx
arch:
- amd64
- arm64
- ppc64le
compiler:
- clang
- gcc
osx_image: xcode9.4
cache:
- ccache
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- libgflags-dev
- libbz2-dev
- liblz4-dev
- libsnappy-dev
- liblzma-dev # xv
- libzstd-dev
- zlib1g-dev
homebrew:
update: true
packages:
- ccache
- gflags
- lz4
- snappy
- xz
- zstd
env:
- TEST_GROUP=platform_dependent # 16-18 minutes
- TEST_GROUP=1 # 33-35 minutes
- TEST_GROUP=2 # 18-20 minutes
- TEST_GROUP=3 # 20-22 minutes
- TEST_GROUP=4 # 12-14 minutes
# Run java tests
- JOB_NAME=java_test # 4-11 minutes
# Build ROCKSDB_LITE
- JOB_NAME=lite_build # 3-4 minutes
# Build examples
- JOB_NAME=examples # 5-7 minutes
- JOB_NAME=cmake # 3-5 minutes
- JOB_NAME=cmake-gcc8 # 3-5 minutes
- JOB_NAME=cmake-mingw # 3 minutes
matrix:
exclude:
- os: osx
env: JOB_NAME=cmake-gcc8
- os: osx
env: JOB_NAME=cmake-mingw
- os: osx
arch: ppc64le
- os: osx
compiler: gcc
- os : linux
arch: arm64
env: JOB_NAME=cmake-mingw
- os: linux
arch: ppc64le
env: JOB_NAME=cmake-mingw
- os: linux
compiler: clang
# Exclude most osx, arm64 and ppc64le tests for pull requests, but build in branches
# NB: the cmake build is a partial java test
- if: type = pull_request
os: osx
env: TEST_GROUP=1
- if: type = pull_request
os : linux
arch: arm64
env: TEST_GROUP=1
- if: type = pull_request
os: linux
arch: ppc64le
env: TEST_GROUP=1
- if: type = pull_request
os: osx
env: TEST_GROUP=2
- if: type = pull_request
os : linux
arch: arm64
env: TEST_GROUP=2
- if: type = pull_request
os: linux
arch: ppc64le
env: TEST_GROUP=2
- if: type = pull_request
os: osx
env: TEST_GROUP=3
- if: type = pull_request
os : linux
arch: arm64
env: TEST_GROUP=3
- if: type = pull_request
os: linux
arch: ppc64le
env: TEST_GROUP=3
- if: type = pull_request
os: osx
env: TEST_GROUP=4
- if: type = pull_request
os : linux
arch: arm64
env: TEST_GROUP=4
- if: type = pull_request
os: linux
arch: ppc64le
env: TEST_GROUP=4
- if: type = pull_request
os : osx
env: JOB_NAME=java_test
- if: type = pull_request
os : linux
arch: arm64
env: JOB_NAME=java_test
- if: type = pull_request
os: linux
arch: ppc64le
env: JOB_NAME=java_test
- if: type = pull_request
os : osx
env: JOB_NAME=lite_build
- if: type = pull_request
os : linux
arch: arm64
env: JOB_NAME=lite_build
- if: type = pull_request
os: linux
arch: ppc64le
env: JOB_NAME=lite_build
- if: type = pull_request
os : osx
env: JOB_NAME=examples
- if: type = pull_request
os : linux
arch: arm64
env: JOB_NAME=examples
- if: type = pull_request
os: linux
arch: ppc64le
env: JOB_NAME=examples
- if: type = pull_request
os : linux
arch: arm64
env: JOB_NAME=cmake-gcc8
- if: type = pull_request
os: linux
arch: ppc64le
env: JOB_NAME=cmake-gcc8
install:
- if [ "${TRAVIS_OS_NAME}" == osx ]; then
PATH=$PATH:/usr/local/opt/ccache/libexec;
fi
- if [ "${JOB_NAME}" == cmake-gcc8 ]; then
sudo apt-get install -y g++-8;
CC=gcc-8 && CXX=g++-8;
fi
- if [ "${JOB_NAME}" == cmake-mingw ]; then
sudo apt-get install -y mingw-w64 ;
fi
- if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
CMAKE_DIST_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/cmake-3.14.5-Linux-$(uname -m).tar.bz2";
TAR_OPT="--strip-components=1 -xj";
if [ "aarch64" == "$(uname -m)" ]; then
sudo apt-get install -y libuv1 librhash0;
sudo apt-get upgrade -y libstdc++6;
fi;
mkdir cmake-dist && curl --silent --fail --show-error --location "${CMAKE_DIST_URL}" | tar -C cmake-dist ${TAR_OPT} && export PATH=$PWD/cmake-dist/bin:$PATH;
fi
- |
if [[ "${JOB_NAME}" == java_test || "${JOB_NAME}" == cmake* ]]; then
# Ensure JDK 8
if [ "${TRAVIS_OS_NAME}" == osx ]; then
brew tap AdoptOpenJDK/openjdk
brew cask install adoptopenjdk8
export JAVA_HOME=$(/usr/libexec/java_home)
else
sudo apt-get install -y openjdk-8-jdk
export PATH=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin:$PATH
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)
fi
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
fi
before_script:
# Increase the maximum number of open file descriptors, since some tests use
# more FDs than the default limit.
- ulimit -n 8192
script:
- ${CXX} --version
- if [ `command -v ccache` ]; then ccache -C; fi
- case $TEST_GROUP in
platform_dependent)
OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some
;;
1)
OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=db_iter_test make -j4 check_some
;;
2)
OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" V=1 make -j4 tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" V=1 ROCKSDBTESTS_START=db_iter_test ROCKSDBTESTS_END=options_file_test make -j4 check_some
;;
3)
OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=options_file_test ROCKSDBTESTS_END=write_prepared_transaction_test make -j4 check_some
;;
4)
OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_prepared_transaction_test make -j4 check_some
;;
esac
- case $JOB_NAME in
java_test)
OPT=-DTRAVIS V=1 make rocksdbjava jtest
;;
lite_build)
OPT='-DTRAVIS -DROCKSDB_LITE' V=1 make -j4 static_lib tools
;;
examples)
OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4
;;
cmake-mingw)
sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
;;
cmake*)
mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release && make -j4 rocksdb rocksdbjni
;;
esac
notifications:
email:
- leveldb@fb.com
Computing file changes ...