Skip to main content

Browse the archive

Revision 8a72bb14bd0caf0fe0ab6c6efb236832beed1f44 authored by Peter Dillinger on 14 November 2019, 14:18:23 UTC, committed by Peter Dillinger on 15 November 2019, 20:15:12 UTC 
More fixes to auto-GarbageCollect in BackupEngine (#6023)
 
Summary:
Production:
* Fixes GarbageCollect (and auto-GC triggered by PurgeOldBackups, DeleteBackup, or CreateNewBackup) to clean up backup directory independent of current settings (except max_valid_backups_to_open; see issue https://github.com/facebook/rocksdb/issues/4997) and prior settings used with same backup directory.
* Fixes GarbageCollect (and auto-GC) not to attempt to remove "." and ".." entries from directories.
* Clarifies contract with users in modifying BackupEngine operations. In short, leftovers from any incomplete operation are cleaned up by any subsequent call to that same kind of operation (PurgeOldBackups and DeleteBackup considered the same kind of operation). GarbageCollect is available to clean up after all kinds. (NB: right now PurgeOldBackups and DeleteBackup will clean up after incomplete CreateNewBackup, but we aren't promising to continue that behavior.)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6023

Test Plan:
* Refactors open parameters to use an option enum, for readability, etc. (Also fixes an unused parameter bug in the redundant OpenDBAndBackupEngineShareWithChecksum.)
* Fixes an apparent bug in ShareTableFilesWithChecksumsTransition in which old backup data was destroyed in the transition to be tested. That test is now augmented to ensure GarbageCollect (or auto-GC) does not remove shared files when BackupEngine is opened with share_table_files=false.
* Augments DeleteTmpFiles test to ensure that CreateNewBackup does auto-GC when an incompletely created backup is detected.

Differential Revision: D18453559

Pulled By: pdillinger

fbshipit-source-id: 5e54e7b08d711b161bc9c656181012b69a8feac4
1 parent a6d4183
Raw File
appveyor.yml 
version: 1.0.{build}

image: Visual Studio 2017

environment:
  JAVA_HOME: C:\Program Files\Java\jdk1.8.0
  THIRDPARTY_HOME: $(APPVEYOR_BUILD_FOLDER)\thirdparty
  SNAPPY_HOME: $(THIRDPARTY_HOME)\snappy-1.1.7
  SNAPPY_INCLUDE: $(SNAPPY_HOME);$(SNAPPY_HOME)\build
  SNAPPY_LIB_DEBUG: $(SNAPPY_HOME)\build\Debug\snappy.lib
  SNAPPY_LIB_RELEASE: $(SNAPPY_HOME)\build\Release\snappy.lib
  LZ4_HOME: $(THIRDPARTY_HOME)\lz4-1.8.3
  LZ4_INCLUDE: $(LZ4_HOME)\lib
  LZ4_LIB_DEBUG: $(LZ4_HOME)\visual\VS2010\bin\x64_Debug\liblz4_static.lib
  LZ4_LIB_RELEASE: $(LZ4_HOME)\visual\VS2010\bin\x64_Release\liblz4_static.lib
  ZSTD_HOME: $(THIRDPARTY_HOME)\zstd-1.4.0
  ZSTD_INCLUDE: $(ZSTD_HOME)\lib;$(ZSTD_HOME)\lib\dictBuilder
  ZSTD_LIB_DEBUG: $(ZSTD_HOME)\build\VS2010\bin\x64_Debug\libzstd_static.lib
  ZSTD_LIB_RELEASE: $(ZSTD_HOME)\build\VS2010\bin\x64_Release\libzstd_static.lib

install:
  - md %THIRDPARTY_HOME%
  - echo "Building Snappy dependency..."
  - cd %THIRDPARTY_HOME%
  - curl -fsSL -o snappy-1.1.7.zip https://github.com/google/snappy/archive/1.1.7.zip
  - unzip snappy-1.1.7.zip
  - cd snappy-1.1.7
  - mkdir build
  - cd build
  - cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..
  - msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64
  - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64
  - echo "Building LZ4 dependency..."
  - cd %THIRDPARTY_HOME%
  - curl -fsSL -o lz4-1.8.3.zip https://github.com/lz4/lz4/archive/v1.8.3.zip
  - unzip lz4-1.8.3.zip
  - cd lz4-1.8.3\visual\VS2010
  - ps: $CMD="C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\devenv.com"; & $CMD lz4.sln /upgrade
  - msbuild lz4.sln /p:Configuration=Debug /p:Platform=x64
  - msbuild lz4.sln /p:Configuration=Release /p:Platform=x64
  - echo "Building ZStd dependency..."
  - cd %THIRDPARTY_HOME%
  - curl -fsSL -o zstd-1.4.0.zip https://github.com/facebook/zstd/archive/v1.4.0.zip
  - unzip zstd-1.4.0.zip
  - cd zstd-1.4.0\build\VS2010
  - ps: $CMD="C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\devenv.com"; & $CMD zstd.sln /upgrade
  - msbuild zstd.sln /p:Configuration=Debug /p:Platform=x64
  - msbuild zstd.sln /p:Configuration=Release /p:Platform=x64

before_build:
  - md %APPVEYOR_BUILD_FOLDER%\build
  - cd %APPVEYOR_BUILD_FOLDER%\build
  - cmake -G "Visual Studio 15 Win64" -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 ..
  - cd ..
build:
  project: build\rocksdb.sln
  parallel: true
  verbosity: normal

test:

test_script:
  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test,db_merge_operand_test -Concurrency 8

on_failure:
  - cmd: 7z a build-failed.zip %APPVEYOR_BUILD_FOLDER%\build\ && appveyor PushArtifact build-failed.zip
back to top