swh:1:snp:5115096b921df712aeb2a08114fede57fb3331fb
- HEAD
- refs/heads/2.2.fb.branch
- refs/heads/2.3.fb.branch
- refs/heads/2.4.fb.branch
- refs/heads/2.5.fb.branch
- refs/heads/2.6.fb.branch
- refs/heads/2.7
- refs/heads/2.7.fb.branch
- refs/heads/2.8.1.fb
- refs/heads/2.8.fb
- refs/heads/2.8.fb.trunk
- refs/heads/3.0.fb
- refs/heads/3.0.fb.branch
- refs/heads/3.1.fb
- refs/heads/3.10.fb
- refs/heads/3.11.fb
- refs/heads/3.12.fb
- refs/heads/3.13.fb
- refs/heads/3.2.fb
- refs/heads/3.3.fb
- refs/heads/3.4.fb
- refs/heads/3.5.fb
- refs/heads/3.6.fb
- refs/heads/3.7.fb
- refs/heads/3.8.fb
- refs/heads/3.9.fb
- refs/heads/4.0.fb
- refs/heads/4.1.fb
- refs/heads/4.10.fb
- refs/heads/4.11.fb
- refs/heads/4.12.fb
- refs/heads/4.13.fb
- refs/heads/4.2.fb
- refs/heads/4.3.fb
- refs/heads/4.4.fb
- refs/heads/4.5.fb
- refs/heads/4.6.fb
- refs/heads/4.7.fb
- refs/heads/4.8.fb
- refs/heads/4.9.fb
- refs/heads/5.0.fb
- refs/heads/5.1.fb
- refs/heads/5.10.fb
- refs/heads/5.11.fb
- refs/heads/5.12.fb
- refs/heads/5.13.fb
- refs/heads/5.13.fb.myrocks
- refs/heads/5.14.fb
- refs/heads/5.14.fb.myrocks
- refs/heads/5.15.fb
- refs/heads/5.16.fb
- refs/heads/5.17.fb
- refs/heads/5.17.fb.myrocks
- refs/heads/5.18.fb
- refs/heads/5.2.fb
- refs/heads/5.3.fb
- refs/heads/5.4.fb
- refs/heads/5.5.fb
- refs/heads/5.6.fb
- refs/heads/5.7.fb
- refs/heads/5.7.fb.myrocks
- refs/heads/5.8.3
- refs/heads/5.8.fb
- refs/heads/5.9.fb
- refs/heads/5.9.fb.myrocks
- refs/heads/6.0.fb
- refs/heads/6.0.fb.myrocks
- refs/heads/6.1.fb
- refs/heads/6.1.fb.myrocks
- refs/heads/6.1.fb.prod201905
- refs/heads/6.10.fb
- refs/heads/6.11.fb
- refs/heads/6.12.fb
- refs/heads/6.13.fb
- refs/heads/6.13.fb.laser
- refs/heads/6.14.fb
- refs/heads/6.14.fb.laser
- refs/heads/6.15.fb
- refs/heads/6.16.fb
- refs/heads/6.17.fb
- refs/heads/6.2.fb
- refs/heads/6.3.fb
- refs/heads/6.3.fb.myrocks
- refs/heads/6.3.fb.myrocks2
- refs/heads/6.3fb
- refs/heads/6.4.fb
- refs/heads/6.5.fb
- refs/heads/6.6.fb
- refs/heads/6.7.fb
- refs/heads/6.8.fb
- refs/heads/6.9.fb
- refs/heads/adaptive
- refs/heads/ajkr-patch-1
- refs/heads/blob_shadow
- refs/heads/bottom-pri-level
- refs/heads/bugfix-build-detect
- refs/heads/checksum_readahead_mmap_fix
- refs/heads/fb-config
- refs/heads/feature/debug-rocksdbjavastatic
- refs/heads/feature/travis-arm64
- refs/heads/fix-release-notes
- refs/heads/fix-write-batch-comment
- refs/heads/format_compatible_4
- refs/heads/getmergeops
- refs/heads/gh-pages-old
- refs/heads/history-update
- refs/heads/hotfix/lambda-capture
- refs/heads/improve-support
- refs/heads/jay-stress
- refs/heads/katherinez-patch-1
- refs/heads/katherinez-patch-2
- refs/heads/master
- refs/heads/pr-sanity-check-as-GHAction
- refs/heads/pr/6062
- refs/heads/ramvadiv-patch-1
- refs/heads/scaffold
- refs/heads/siying-patch-1
- refs/heads/siying-patch-10
- refs/heads/siying-patch-2
- refs/heads/siying-patch-3
- refs/heads/siying-patch-4
- refs/heads/skip_memtable_flush
- refs/heads/testing_ppc_build
- refs/heads/tests
- refs/heads/unused-var
- refs/heads/v6.6.4
- refs/heads/yiwu_stackable
- refs/heads/yuslepukhin
- refs/remotes/origin/5.13.fb
- refs/tags/2.5.fb
- refs/tags/2.6.fb
- refs/tags/3.0.fb
- refs/tags/do-not-use-me2
- refs/tags/rocksdb-3.1
- refs/tags/rocksdb-3.10.2
- refs/tags/rocksdb-3.11
- refs/tags/rocksdb-3.11.1
- refs/tags/rocksdb-3.11.2
- refs/tags/rocksdb-3.2
- refs/tags/rocksdb-3.3
- refs/tags/rocksdb-3.4
- refs/tags/rocksdb-3.5
- refs/tags/rocksdb-3.5.1
- refs/tags/rocksdb-3.6.1
- refs/tags/rocksdb-3.6.2
- refs/tags/rocksdb-3.7
- refs/tags/rocksdb-3.8
- refs/tags/rocksdb-3.9
- refs/tags/rocksdb-3.9.1
- refs/tags/rocksdb-4.1
- refs/tags/rocksdb-5.10.2
- refs/tags/rocksdb-5.10.3
- refs/tags/rocksdb-5.10.4
- refs/tags/rocksdb-5.11.2
- refs/tags/rocksdb-5.11.3
- refs/tags/rocksdb-5.14.3
- refs/tags/rocksdb-5.2.1
- refs/tags/rocksdb-5.3.3
- refs/tags/rocksdb-5.3.4
- refs/tags/rocksdb-5.3.5
- refs/tags/rocksdb-5.3.6
- refs/tags/rocksdb-5.4.10
- refs/tags/rocksdb-5.4.5
- refs/tags/rocksdb-5.4.6
- refs/tags/rocksdb-5.5.2
- refs/tags/rocksdb-5.5.3
- refs/tags/rocksdb-5.5.4
- refs/tags/rocksdb-5.5.5
- refs/tags/rocksdb-5.5.6
- refs/tags/rocksdb-5.6.1
- refs/tags/rocksdb-5.6.2
- refs/tags/rocksdb-5.7.1
- refs/tags/rocksdb-5.7.2
- refs/tags/rocksdb-5.7.3
- refs/tags/rocksdb-5.7.5
- refs/tags/rocksdb-5.8.6
- refs/tags/rocksdb-5.8.7
- refs/tags/rocksdb-5.8.8
- refs/tags/rocksdb-5.9.2
- refs/tags/v4.0
- refs/tags/v4.1
- refs/tags/v5.10.2
- refs/tags/v5.10.3
- refs/tags/v5.10.4
- refs/tags/v5.11.2
- refs/tags/v5.11.3
- refs/tags/v5.13.3
- refs/tags/v5.14.3
- refs/tags/v5.15.10
- refs/tags/v5.18.3
- refs/tags/v5.2.1
- refs/tags/v5.3.3
- refs/tags/v5.3.4
- refs/tags/v5.3.5
- refs/tags/v5.3.6
- refs/tags/v5.4.10
- refs/tags/v5.4.5
- refs/tags/v5.4.6
- refs/tags/v5.5.2
- refs/tags/v5.5.3
- refs/tags/v5.5.4
- refs/tags/v5.5.5
- refs/tags/v5.5.6
- refs/tags/v5.6.1
- refs/tags/v5.6.2
- refs/tags/v5.7.1
- refs/tags/v5.7.2
- refs/tags/v5.7.3
- refs/tags/v5.7.5
- refs/tags/v5.8.6
- refs/tags/v5.8.7
- refs/tags/v5.8.8
- refs/tags/v5.9.2
- refs/tags/v6.0.1
- refs/tags/v6.0.2
- refs/tags/v6.1.1
- refs/tags/v6.1.2
- refs/tags/v6.10.1
- refs/tags/v6.10.2
- refs/tags/v6.11.4
- refs/tags/v6.11.6
- refs/tags/v6.12.6
- refs/tags/v6.12.7
- refs/tags/v6.13.2
- refs/tags/v6.13.3
- refs/tags/v6.14.5
- refs/tags/v6.14.6
- refs/tags/v6.15.4
- refs/tags/v6.2.2
- refs/tags/v6.2.4
- refs/tags/v6.3.6
- refs/tags/v6.4.6
- refs/tags/v6.5.2
- refs/tags/v6.5.3
- refs/tags/v6.6.3
- refs/tags/v6.6.4
- refs/tags/v6.7.3
- refs/tags/v6.8.1
- v6.15.2
- v5.8
- v5.5.1
- v5.4.7
- v5.18.4
- v5.17.2
- v5.16.6
- v5.14.2
- v5.13.4
- v5.13.2
- v5.13.1
- v5.12.5
- v5.12.4
- v5.12.3
- v5.12.2
- v5.1.4
- v5.1.3
- v5.1.2
- v5.0.2
- v5.0.1
- v4.9
- v4.8
- v4.6.1
- v4.5.1
- v4.4.1
- v4.4
- v4.3.1
- v4.3
- v4.2
- v4.13.5
- v4.13
- v4.11.2
- v3.9
- v3.8
- v3.7
- v3.6.1
- v3.5
- v3.4
- v3.3
- v3.2
- v3.13.1
- v3.13
- v3.12.1
- v3.12
- v3.11
- v3.10
- v3.1
- v3.0
- v2.8
- v2.7
- v2.6
- v2.5
- v2.4
- v2.3
- v2.2
- v2.1
- v2.0
- v1.5.9.1
- v1.5.8.2
- v1.5.8.1
- v1.5.8
- v1.5.7
- rocksdb-5.8
- rocksdb-5.4.7
- rocksdb-5.1.4
- rocksdb-5.1.3
- rocksdb-5.1.2
- rocksdb-5.0.2
- rocksdb-5.0.1
- rocksdb-4.9
- rocksdb-4.8
- rocksdb-4.6.1
- rocksdb-4.5.1
- rocksdb-4.4.1
- rocksdb-4.4
- rocksdb-4.3.1
- rocksdb-4.3
- rocksdb-4.2
- rocksdb-4.13.5
- rocksdb-4.13
- rocksdb-4.11.2
- rocksdb-3.10.1
- 2.8.fb
- 2.7.fb
- 2.4.fb
- 2.3.fb
- 2.2.fb
- 2.1.fb
- 2.0.fb
- 1.5.9.fb
- 1.5.9.2.fb
- 1.5.9.1.fb
- 1.5.8.fb
- 1.5.8.2.fb
- 1.5.8.1.fb
- 1.5.7.fb
Permalinks
To reference or cite the objects present in the Software Heritage archive, permalinks based on SoftWare Hash IDentifiers (SWHIDs) must be used.
Select below a type of object currently browsed in order to display its associated SWHID and permalink.
Revision | Author | Date | Message | Commit Date |
---|---|---|---|---|
8ec8d12 | Yanqin Jin | 17 November 2020, 08:43:20 UTC | Fix the logic of setting read_amp_bytes_per_bit from OPTIONS file (#7680) Summary: Instead of using `EncodeFixed32` which always serialize a integer to little endian, we should use the local machine's endianness when populating a native data structure during options parsing. Without this fix, `read_amp_bytes_per_bit` may be populated incorrectly on big-endian machines. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7680 Test Plan: make check Reviewed By: pdillinger Differential Revision: D24999166 Pulled By: riversand963 fbshipit-source-id: dc603cff6e17f8fa32479ce6df93b93082e6b0c4 | 17 November 2020, 18:27:16 UTC |
a1f37aa | Yanqin Jin | 15 November 2020, 23:13:05 UTC | Bump version and update HISTORY.md | 15 November 2020, 23:13:05 UTC |
98174bc | Yanqin Jin | 13 November 2020, 19:51:24 UTC | Hack to load OPTIONS file for read_amp_bytes_per_bit (#7659) Summary: A temporary hack to work around a bug in 6.10, 6.11, 6.12, 6.13 and 6.14. The bug will write out 8 bytes to OPTIONS file from the starting address of BlockBasedTableOptions.read_amp_bytes_per_bit which is actually a uint32. Consequently, the value of read_amp_bytes_per_bit written in the OPTIONS file is wrong. From 6.15, RocksDB will try to parse the read_amp_bytes_per_bit from OPTIONS file as a uint32. To be able to load OPTIONS file generated by affected releases before the fix, we need to manually parse read_amp_bytes_per_bit with this hack. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7659 Test Plan: Generate a db with current 6.14.fb (head at https://github.com/facebook/rocksdb/commit/b6db05dbb5364c658c5401a8078d73697bb5f31d). Maybe use db_stress. Checkout this PR, run ``` ~/rocksdb/ldb --db=. --try_load_options --ignore_unknown_options idump --count_only ``` Expect success, and should not see ``` Failed: Invalid argument: Error parsing read_amp_bytes_per_bit:17179869184 ``` Also make check Reviewed By: anand1976 Differential Revision: D24954752 Pulled By: riversand963 fbshipit-source-id: c7b802fc3e52acd050a4fc1cd475016122234394 | 15 November 2020, 23:10:54 UTC |
e82e5e0 | Huisheng Liu | 10 November 2020, 19:13:10 UTC | fix read_amp_bytes_per_bit field size (#7651) Summary: The field in BlockBasedTableOptions is 4 bytes: // Default: 0 (disabled) uint32_t read_amp_bytes_per_bit = 0; Pull Request resolved: https://github.com/facebook/rocksdb/pull/7651 Reviewed By: ltamasi Differential Revision: D24844994 Pulled By: riversand963 fbshipit-source-id: e2695e55532256ef8996dd6939cad06987a80293 | 15 November 2020, 23:08:05 UTC |
f3e3354 | Andrew Kryczka | 14 October 2020, 17:55:43 UTC | Update HISTORY.md and version.h for 6.12.7 | 14 October 2020, 17:55:43 UTC |
bfdb0a7 | Amir Sanjar | 14 October 2020, 17:37:55 UTC | Fix a build issue with RocksJava on ppc64le that was introduced in https://github.com/facebook/rocksdb/pull/6660 This was fixed in https://github.com/facebook/rocksdb/pull/7359 and this commit is just a partial back-port of the necessary parts of that PR. | 14 October 2020, 17:53:27 UTC |
7885d8f | Andrew Kryczka | 13 October 2020, 16:30:27 UTC | Update HISTORY.md and version.h for 6.12.6 | 13 October 2020, 16:30:27 UTC |
56fee9f | Andrew Kryczka | 13 October 2020, 15:58:27 UTC | Fix bug when `DeleteRange()` used with `paranoid_file_checks == true` Backports part of 750817555867a43f0e7b73dffa44756a9136c808. The unit test comes from https://github.com/facebook/rocksdb/pull/7521. | 13 October 2020, 16:28:12 UTC |
290e990 | Andrew Kryczka | 12 October 2020, 20:49:18 UTC | Update HISTORY.md and version.h for 6.12.5 | 12 October 2020, 20:49:18 UTC |
24bb466 | Andrew Kryczka | 12 October 2020, 19:42:58 UTC | Fix bug in pinned partitioned indexes with some reads bypassing block cache Backports part of 75d3b6fdf0aa1007c4d26382f65be0adf4519a37. | 12 October 2020, 20:45:33 UTC |
1fdf49c | Andrew Kryczka | 12 October 2020, 20:43:52 UTC | Fix bug in pinned partitioned user key indexes Backports part of 75d3b6fdf0aa1007c4d26382f65be0adf4519a37. | 12 October 2020, 20:43:52 UTC |
ceb7ae1 | Andrew Kryczka | 12 October 2020, 20:43:16 UTC | Add missing release note | 12 October 2020, 20:43:16 UTC |
1b963a9 | Peter Dillinger | 17 September 2020, 21:18:42 UTC | Update HISTORY.md and version.h for 6.12.4 | 18 September 2020, 15:20:39 UTC |
8a6c925 | Peter Dillinger | 16 September 2020, 05:10:18 UTC | Restore file size in backup table file names (and other cleanup) Summary: Prior to 6.12, backup files using share_files_with_checksum had the file size encoded in the file name, after the last '_' and before the last '.'. We considered this an implementation detail subject to change, and indeed removed this information from the file name (with an option to use old behavior) because it was considered ineffective/inefficient for file name uniqueness. However, some downstream RocksDB users were relying on this information since the file size is not explicitly in the backup manifest file. This primary purpose of this change is "retrofitting" the 6.12 release (not yet a public release) to simultaneously support the benefits of the new naming scheme (I/O performance and data correctness at scale) and preserve the file size information, both as default behaviors. With this change, we are essentially making the file size information encoded in the file name an official, though obscure, extension of the backup meta file format. We preserve an option (kLegacyCrc32cAndFileSize) to use the original "legacy" naming scheme, with its caveats, and make it easy to omit the file size information (no kFlagIncludeFileSize), for more compact file names. But note that changing the naming scheme used on an existing db and backup directory can lead to transient space amplification, as some files will be stored under two names in the shared_checksum directory. Because some backups were saved using the original 6.12 naming scheme, we offer two ways of dealing with those files: SST files generated by older 6.12 versions can either use the default naming scheme in effect when the SST files were generated (kFlagMatchInterimNaming, default, no transient space amplification) or can use a new naming scheme (no kFlagMatchInterimNaming, potential space amplification because some already stored files getting a new name). We don't have a natural way to detect which files were generated by previous 6.12 versions, but this change hacks one in by changing DB session ids to now use a more concise encoding, reducing file name length, saving ~dozen bytes from SST files, and making them visually distinct from DB ids so that they are less likely to be mixed up. Finally, recognizing that the backup file names have become a de facto part of the backup meta schema, this change makes them easier to parse and extend by putting a distinct marker, 's', before DB session ids embedded in the name. When we extend this to allow custom checksums in the name, they can get their own marker to ensure safe parsing. For backward compatibility, file size does not get a marker but is assumed for _[0-9]+[.] Test Plan: unit tests included. Sync point callbacks are used to mimic previous version SST files. | 17 September 2020, 07:22:30 UTC |
fad041f | Andrew Kryczka | 16 September 2020, 16:39:59 UTC | update HISTORY.md and version.h for 6.12.3 | 16 September 2020, 16:39:59 UTC |
972d137 | Cheng Chang | 04 September 2020, 06:16:06 UTC | Fix wrong level args (#7346) Summary: The level args should be output level instead of input levels. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7346 Test Plan: make check Reviewed By: ajkr Differential Revision: D23506373 Pulled By: cheng-chang fbshipit-source-id: b2f701d44c13581c5c10c4dbebded4fcd354d641 | 16 September 2020, 16:31:03 UTC |
e6e52a7 | Levi Tamasi | 14 September 2020, 23:15:46 UTC | Update version.h and HISTORY.md for 6.12.2 | 14 September 2020, 23:15:46 UTC |
b7cc96d | Levi Tamasi | 10 September 2020, 18:32:02 UTC | Expose the start of the expiration range for TTL blob files through LiveFileMetaData (#7365) Summary: The patch adds support for exposing the start of the expiration range for TTL blob files through the `GetLiveFilesMetaData` API. This can be used for monitoring purposes, i.e. to make sure TTL blob files are deleted in a timely manner. The patch also fixes a couple of uninitialized variable issues. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7365 Test Plan: `make check` Reviewed By: pdillinger Differential Revision: D23605465 Pulled By: ltamasi fbshipit-source-id: 97a9612bf5f4b058423debdd3f28f576bb23a70f | 14 September 2020, 23:11:01 UTC |
7dc1c55 | Peter Dillinger | 20 August 2020, 22:57:13 UTC | Update for 6.12.1 patch release | 20 August 2020, 23:52:20 UTC |
25bddfa | Peter Dillinger | 20 August 2020, 21:02:35 UTC | Work around a backup bug with DB custom checksums Summary: On a read-write DB configured with DBOptions::file_checksum_gen_factory, BackupEngine::CreateNewBackup can fail intermittently, with non-OK status. This is due to a race between GetLiveFiles and GetLiveFilesChecksumInfo in creating backups. For patching 6.12 release (as this commit is intended for), we can simply treat files for which we falsely failed to get checksum info as legacy files lacking checksum info. Test Plan: unit test reproducer included | 20 August 2020, 22:24:11 UTC |
a38f04a | Yanqin Jin | 29 July 2020, 17:11:47 UTC | Update HISTORY and version for 6.12 release (#7194) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/7194 Reviewed By: gg814 Differential Revision: D22810654 Pulled By: riversand963 fbshipit-source-id: 01f13089fa2b7e31b827da3e30c90e5c62c41380 | 29 July 2020, 17:13:21 UTC |
692f6a3 | sdong | 29 July 2020, 16:43:56 UTC | Implement NextAndGetResult() in memtable and level iterator (#7179) Summary: NextAndGetResult() is not implemented in memtable and is very simply implemented in level iterator. The result is that for a normal leveled iterator, performance regression will be observed for calling PrepareValue() for most iterator Next(). Mitigate the problem by implementing the function for both iterators. In level iterator, the implementation cannot be perfect as when calling file iterator's SeekToFirst() we don't have information about whether the value is prepared. Fortunately, the first key should not cause a big portion of the CPu. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7179 Test Plan: Run normal crash test for a while. Reviewed By: anand1976 Differential Revision: D22783840 fbshipit-source-id: c19f45cdf21b756190adef97a3b66ccde3936e05 | 29 July 2020, 16:45:21 UTC |
d9d1907 | mrambacher | 29 July 2020, 05:58:28 UTC | Make env*_test work with ASSERT_STATUS_CHECKED (#7176) Summary: Make (most of) the env*_test pass when ASSERT_STATUS_CHECKED is enabled. One test that opens a database is currently disabled in this mode, as there are many errors that need revisited for DB tests and status checks. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7176 Reviewed By: cheng-chang Differential Revision: D22799278 Pulled By: ajkr fbshipit-source-id: 16d8a02eaeecd6df1060249b6a5811292801f2ed | 29 July 2020, 05:59:48 UTC |
c0c33a4 | Andrew Kryczka | 28 July 2020, 20:09:12 UTC | Makefile support for link-time optimization (#7181) Summary: `USE_LTO=1` in `make` commands now enables LTO. The archiver (`ar`) needed to change in this PR to use a wrapper that enables the LTO plugin. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7181 Test Plan: build a few ways ``` $ make clean && USE_LTO=1 make -j48 db_bench $ make clean && USE_CLANG=1 USE_LTO=1 make -j48 db_bench $ make clean && ROCKSDB_NO_FBCODE=1 USE_LTO=1 make -j48 db_bench ``` Reviewed By: cheng-chang Differential Revision: D22784994 Pulled By: ajkr fbshipit-source-id: 9c45333bd49bf4615aa04c85b7c6fd3925421152 | 28 July 2020, 20:10:44 UTC |
83ea266 | codingsh | 28 July 2020, 20:04:29 UTC | export stats_persist_period_sec (#7168) Summary: fixed - https://github.com/rust-rocksdb/rust-rocksdb/issues/447 - https://github.com/rust-rocksdb/rust-rocksdb/pull/448 Pull Request resolved: https://github.com/facebook/rocksdb/pull/7168 Reviewed By: cheng-chang Differential Revision: D22736013 Pulled By: ajkr fbshipit-source-id: fdd784aa75d26a367b9108b05ffdd94a2ae117d3 | 28 July 2020, 20:05:34 UTC |
4496719 | zitan | 28 July 2020, 19:08:41 UTC | Fix data race warning of BackupableDBTest.TableFileWithDbChecksumCorruptedDuringBackup (#7177) Summary: Fix the data race warning by removing an unnecessary variable that causes the warning. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7177 Test Plan: `COMPILE_WITH_TSAN=1 make backupable_db_test` `./backupable_db_test --gtest_filter=*TableFileWithDbChecksumCorruptedDuringBackup*` Reviewed By: riversand963 Differential Revision: D22774430 Pulled By: gg814 fbshipit-source-id: 3b0b1ac344d0375c64da564cc97f98745c289959 | 28 July 2020, 19:10:39 UTC |
b0279d3 | Yanqin Jin | 28 July 2020, 16:30:43 UTC | Header file should not be executable (#7182) Summary: As title. Undo file mode change in https://github.com/facebook/rocksdb/issues/6759 . Pull Request resolved: https://github.com/facebook/rocksdb/pull/7182 Reviewed By: ajkr Differential Revision: D22786166 Pulled By: riversand963 fbshipit-source-id: 696903069acda42f26bbbf1f2875f5a08b761b42 | 28 July 2020, 16:39:13 UTC |
69a6d0b | Cheng Chang | 25 July 2020, 07:15:56 UTC | Fix RandomAccessFileReaderTest failures on Travis (#7173) Summary: On Travis, the old `alignment()` returned by `RandomAccessFileReaderTest` is inconsistent with the `GetRequiredBufferAlignment` returned in `RandomAccessFileReader`. This PR removes `alignment()` and consistently use `GetRequiredBufferAlignment` as page size. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7173 Test Plan: make random_access_file_reader_test && ./random_access_file_reader_test Watch Travis Reviewed By: siying Differential Revision: D22741606 Pulled By: cheng-chang fbshipit-source-id: f28f29a7c993bbc3594ae70ecd186fa8bab9c4f2 | 25 July 2020, 07:17:12 UTC |
1adfd72 | Yanqin Jin | 25 July 2020, 00:28:27 UTC | Enable a few jobs in determinator (#7174) Summary: https://github.com/facebook/rocksdb/issues/7170 added a few job specs. This PR enables rocksdb-lego-determinator to support them. Test plan (dev server) ``` $build_tools/rocksdb-lego-determinator blackbox_stress_crash $build_tools/rocksdb-lego-determinator whitebox_stress_crash $build_tools/rocksdb-lego-determinator blackbox_asan_crash $build_tools/rocksdb-lego-determinator whitebox_asan_crash $build_tools/rocksdb-lego-determinator blackbox_ubsan_crash $build_tools/rocksdb-lego-determinator whitebox_ubsan_crash $build_tools/rocksdb-lego-determinator blackbox_tsan_crash $build_tools/rocksdb-lego-determinator whitebox_tsan_crash ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/7174 Reviewed By: siying Differential Revision: D22741153 Pulled By: riversand963 fbshipit-source-id: 39b7d948f04a5b109f009b5499c1dbdc83a13c6e | 25 July 2020, 00:29:36 UTC |
7e37a59 | Akanksha Mahajan | 24 July 2020, 21:45:45 UTC | Fix for flaky test BackupableDBTest.RateLimiting (#7167) Summary: BackupableDBTest.RateLimiting test is failing due to timed out on our test server. It might be because of nested loops run sequentially that test different type of combinations of parameters. This patch converts the test into parameterized test so that all combinations can be tested out. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7167 Test Plan: make check -j64 Reviewed By: zhichao-cao Differential Revision: D22709531 Pulled By: akankshamahajan15 fbshipit-source-id: 95518153e87b3b5311a6c1960a191bca58898786 | 24 July 2020, 21:47:00 UTC |
0c5bb10 | Jay Zhuang | 24 July 2020, 21:12:51 UTC | Remove redundant ROCKSDB_LITE check (#7172) Summary: It's already inside of a `#ifdef ROCKSDB_LITE` block. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7172 Reviewed By: gg814 Differential Revision: D22736057 Pulled By: jay-zhuang fbshipit-source-id: 31f4aa05aba98e2e42fa6f890fa72acf3a0f12f2 | 24 July 2020, 21:14:14 UTC |
cd4592c | Tomas Kolda | 24 July 2020, 20:43:14 UTC | SST Partitioner interface that allows to split SST files (#6957) Summary: SST Partitioner interface that allows to split SST files during compactions. It basically instruct compaction to create a new file when needed. When one is using well defined prefixes and prefixed way of defining tables it is good to define also partitioning so that promotion of some SST file does not cover huge key space on next level (worst case complete space). Pull Request resolved: https://github.com/facebook/rocksdb/pull/6957 Reviewed By: ajkr Differential Revision: D22461239 fbshipit-source-id: 9ce07bba08b3ba89c2d45630520368f704d1316e | 24 July 2020, 20:44:49 UTC |
954ee56 | Yanqin Jin | 24 July 2020, 20:41:19 UTC | Add job specs for blackbox/whitebox stress tests (#7170) Summary: As title. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7170 Test Plan: Manually invoke the commands. Reviewed By: siying Differential Revision: D22732256 Pulled By: riversand963 fbshipit-source-id: d331e5ee84658ac079814292ff1a1eacfd14bfdf | 24 July 2020, 20:42:53 UTC |
d34e015 | Cheng Chang | 23 July 2020, 20:48:17 UTC | Add more tests for RandomAccessFileReader::MultiRead (#7157) Summary: There is a typo in TryMerge which may cause MultiRead to internally read more data than expected, but won't affect MultiRead results' correctness. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7157 Test Plan: make random_access_file_reader_test && ./random_access_file_reader_test Reviewed By: siying Differential Revision: D22670257 Pulled By: cheng-chang fbshipit-source-id: d261289455a65aa496b348c6e5582b48b12963b7 | 23 July 2020, 20:50:00 UTC |
7af1fab | Cheng Chang | 23 July 2020, 15:45:58 UTC | Update HISTORY (#7158) Summary: Mention the MultiRead bug in HISTORY. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7158 Test Plan: N/A Reviewed By: siying Differential Revision: D22670565 Pulled By: cheng-chang fbshipit-source-id: 16abf0192957be66511f6a08e00157bfd37b189f | 23 July 2020, 15:47:13 UTC |
b0c5ecd | Jay Zhuang | 23 July 2020, 01:31:25 UTC | Make max_subcompactions dynamically changeable (#7159) Summary: Make `max-subcompactions` dynamically changeable by passing the `DBOption` to Compaction. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7159 Reviewed By: siying Differential Revision: D22671238 Pulled By: jay-zhuang fbshipit-source-id: 311ca9f6bb606965544d8708616d358cfed5be42 | 23 July 2020, 01:32:52 UTC |
0d04a84 | Levi Tamasi | 23 July 2020, 00:24:07 UTC | Sync blob files before closing them (#7160) Summary: BlobDB currently syncs each blob file periodically after writing a certain amount of data (as specified by the configuration option `BlobDBOptions::bytes_per_sync`) and all open blob files when the base DB's memtables are flushed. With the patch, in addition to the above, blob files are also synced right before being closed, after the footer has been written. This will be beneficial for the new integrated blob file write path as well. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7160 Test Plan: `make check` Reviewed By: riversand963 Differential Revision: D22672646 Pulled By: ltamasi fbshipit-source-id: 62b34263543a7e74abcbb7adf011daa1e699998f | 23 July 2020, 00:25:20 UTC |
4a60cb2 | Jason Volk | 22 July 2020, 22:02:10 UTC | Fix bug in MultiRead() coalescing introduced in 4fc216649d (#6446). (#6979) Summary: TryMerge() overzealously creates one huge file read request in an attempt to merge smaller disjoint requests. For example, ~30 input requests of ~100 bytes output as 1 request of 100 MiB causing alarmingly large read throughputs to be repeatedly observed by the environment. Signed-off-by: Jason Volk <jason@zemos.net> Pull Request resolved: https://github.com/facebook/rocksdb/pull/6979 Reviewed By: siying Differential Revision: D22668892 Pulled By: cheng-chang fbshipit-source-id: 7506fe9621b7f1a747dadf6b8ddb1b1a141c1937 | 22 July 2020, 22:03:22 UTC |
96ce047 | Cheng Chang | 22 July 2020, 20:52:45 UTC | Clean snapshot dir before taking snapshot (#7156) Summary: `DBTest::SnapshotFiles` runs the tests in a `while` loop. Currently, the snapshot directory is not cleaned up in each loop, so previous snapshot files may remain in the next loop's snapshot. When I'm working on https://github.com/facebook/rocksdb/pull/7129, when checking the tracked WALs in MANIFEST, I find that this test always fails because it reads some unknown WAL. It turns out that the unknown WAL is left from previous loops. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7156 Test Plan: make db_test && ./db_test --gtest_filters=*SnapshotFiles Reviewed By: siying Differential Revision: D22668360 Pulled By: cheng-chang fbshipit-source-id: 69d4aa3506038ba30e218e8ae966357935a99c6c | 22 July 2020, 20:54:01 UTC |
d44cbc5 | mrambacher | 22 July 2020, 18:03:29 UTC | Add hash of key/value checks when paranoid_file_checks=true (#7134) Summary: When paraoid_files_checks=true, a rolling key-value hash is generated and compared to what is written to the file. If the values do not match, the SST file is rejected. Code put in place for the check for both flush and compaction jobs. Corresponding test added to corruption_test. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7134 Reviewed By: cheng-chang Differential Revision: D22646149 fbshipit-source-id: 8fde1984a1a11edd3bd82a413acffc5ea7aa683f | 22 July 2020, 18:04:40 UTC |
dbc51ad | Haosen Wen | 22 July 2020, 15:53:21 UTC | Use steady_clock instead of system_clock in FileOperationInfo::TimePoint (#7153) Summary: Issue https://github.com/facebook/rocksdb/issues/7133 reported that using `system_clock` in `FileOperationInfo::TimePoint` causes the duration of file flush operation (which can be a noop on MacOS in some scenarios) appears to be 0 and fail an assertion in listener_test. Using `steady_clock` supposedly fixed the problem. `steady_clock` actually fits better into the use cases of `FileOperationInfo::TimePoint` as all usages care about durations but not wall clock time. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7153 Test Plan: make check. Reviewed By: riversand963 Differential Revision: D22654136 Pulled By: roghnin fbshipit-source-id: 5980b1080734bdae496a18071a2c2b5887c67d85 | 22 July 2020, 15:55:02 UTC |
b923dc7 | Zitan Chen | 21 July 2020, 17:34:30 UTC | BackupEngine computes table checksums only once if db session ids are available (#7110) Summary: BackupEngine requires computing table checksums twice when backing up table files to the `shared_checksum` directory. The repeated computation can be avoided by utilizing the db session id stored as a part of the table properties. Filenames of table files in the `shared_checksum` directory depend on the following conditions: 1. the naming scheme is `kOptionalChecksumAndDbSessionId`, 2. `db_session_id` is not empty, 3. checksum is available in the DB manifest. If 1,2,3 are satisfied, then the filenames will be of the form `<file_number>_<checksum>_<db_session_id>.sst`. If 1,2 are satisfied, then the filenames will be of the form `<file_number>_<db_session_id>.sst`. In all other cases, the filenames are of the form `<file_number>_<checksum>_<size>.sst`. Additionally, if `kOptionalChecksumAndDbSessionId` is used (and not falling back to `kChecksumAndFileSize`), the `<checksum>` appeared in the filenames is hexadecimally encoded, instead of being plain `uint32_t` value. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7110 Test Plan: backupable_db_test and manual tests. Reviewed By: ajkr Differential Revision: D22508992 Pulled By: gg814 fbshipit-source-id: 5669f0ea9ad5a097f69f6d87aca4abba15032389 | 21 July 2020, 17:35:40 UTC |
0f487cc | sdong | 20 July 2020, 21:59:49 UTC | Exclude two tests in CircleCI TSAN tests (#7152) Summary: Two TSAN tests occaionaly fail. Exclude them for now: [ RUN ] DeleteFileTest.BackgroundPurgeCFDropTest db/deletefile_test.cc:122: Failure Expected equality of these values: required_manifest Which is: 1 manifest_cnt Which is: 2 [ RUN ] FormatLatest/ColumnFamilyTest.FlushCloseWALFiles/0 db/column_family_test.cc:3004: Failure Expected equality of these values: 2 env.num_open_wal_file_.load() Which is: 1 Pull Request resolved: https://github.com/facebook/rocksdb/pull/7152 Test Plan: Watch CircleCI restuls Reviewed By: ajkr Differential Revision: D22632285 fbshipit-source-id: 29fa348e8be917be0237c74812a8b0b04978e84e | 20 July 2020, 22:01:17 UTC |
1cf4731 | sdong | 20 July 2020, 21:17:05 UTC | column_family_test: fix a data race related to sleeping task (#7150) Summary: TSAN reports warning in one column_family_test: WARNING: ThreadSanitizer: data race (pid=16352) Write of size 8 at 0x7ffcdf042158 by main thread: #0 pthread_cond_destroy <null> (column_family_test+0x471f65) https://github.com/facebook/rocksdb/issues/1 rocksdb::port::CondVar::~CondVar() /home/circleci/project/port/port_posix.cc:101:49 (column_family_test+0x8a627a) https://github.com/facebook/rocksdb/issues/2 rocksdb::test::SleepingBackgroundTask::~SleepingBackgroundTask() /home/circleci/project/./test_util/testutil.h:397:7 (column_family_test+0x54b6e2) https://github.com/facebook/rocksdb/issues/3 rocksdb::ColumnFamilyTest_FlushCloseWALFiles_Test::TestBody() /home/circleci/project/db/column_family_test.cc:3008:1 (column_family_test+0x54b6e2) ...... Previous read of size 8 at 0x7ffcdf042158 by thread T2 (mutexes: write M0): #0 pthread_cond_broadcast <null> (column_family_test+0x471dd2) https://github.com/facebook/rocksdb/issues/1 rocksdb::port::CondVar::SignalAll() /home/circleci/project/port/port_posix.cc:139:28 (column_family_test+0x8a651a) https://github.com/facebook/rocksdb/issues/2 rocksdb::test::SleepingBackgroundTask::DoSleep() /home/circleci/project/./test_util/testutil.h:412:12 (column_family_test+0x58574b) ...... Likely, SleepingBackgroundTask::DoSleep() started to execute after the main thread has finished everything, cancelled and waited for sleeping tasks to finish. At this time, although DoSlee() will not sleep, but it also accesses the mutex, creating a data race with destructor of the test. Fix this bug by waiting for the sleeping task to start sleeping after it is scheduled. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7150 Test Plan: Run these modified tests and make sure it doesn't break. Reviewed By: riversand963 Differential Revision: D22630716 fbshipit-source-id: cc5781cf69083685de406490438898238bdfc2d3 | 20 July 2020, 21:19:48 UTC |
643c863 | Andrew Kryczka | 20 July 2020, 21:05:21 UTC | minimize BlockIter comparator scope (#7149) Summary: PR https://github.com/facebook/rocksdb/issues/6944 transitioned `BlockIter` from using `Comparator*` to using concrete `UserComparatorWrapper` and `InternalKeyComparator`. However, adding them as instance variables to `BlockIter` was not optimal. Bloating `BlockIter` caused the `ArenaWrappedDBIter`'s arena allocator to do more heap allocations (in certain cases) which harmed performance of `DB::NewIterator()`. This PR pushes down the concrete comparator objects to the point of usage, which forces them to be on the stack. As a result, the `BlockIter` is back to its original size prior to https://github.com/facebook/rocksdb/issues/6944 (actually a bit smaller since there were two `Comparator*` before). Pull Request resolved: https://github.com/facebook/rocksdb/pull/7149 Test Plan: verified our internal `DB::NewIterator()`-heavy regression test no longer reports regression. Reviewed By: riversand963 Differential Revision: D22623189 Pulled By: ajkr fbshipit-source-id: f6d69accfe5de51e0bd9874a480b32b29909bab6 | 20 July 2020, 21:07:04 UTC |
9870704 | sdong | 20 July 2020, 19:09:35 UTC | Fix a minor data race in stats dumping threads initialization (#7151) Summary: https://github.com/facebook/rocksdb/pull/7145 creates a minor data race against the stat creation counter. Turn it to atomic. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7151 Test Plan: Run the test. Reviewed By: ajkr Differential Revision: D22631014 fbshipit-source-id: c6fb69ac5b9df7139795dacea5ce9fb9fd3278d7 | 20 July 2020, 19:12:43 UTC |
77062cf | Jay Zhuang | 20 July 2020, 18:14:51 UTC | Store the test results to CircleCI (#7137) Summary: To have test report. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7137 Reviewed By: siying Differential Revision: D22630798 Pulled By: jay-zhuang fbshipit-source-id: bc07ba673c0bceed5a4829b4af2d9a74435379c7 | 20 July 2020, 18:16:19 UTC |
ed4712f | Zhichao Cao | 18 July 2020, 06:26:07 UTC | Remove time out testing cases in error_handler_fs_test (#7141) Summary: Remove the 3 testing cases that cause the time out in linux build by https://github.com/facebook/rocksdb/issues/6765 . Will fix them later. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7141 Test Plan: make asan_check, buck run Reviewed By: ajkr Differential Revision: D22593831 Pulled By: zhichao-cao fbshipit-source-id: 14956c36476ecc3393f613178c22e13df843126e | 18 July 2020, 06:27:21 UTC |
1cc9b0e | sdong | 18 July 2020, 01:05:51 UTC | Fix parallel test sometimes doesn't fail with failed tests. (#7147) Summary: In CircleCI tests, we failed to fail tests properly if parallel doesn't return an error code. It's probably would happen when unit tests fail with signals, rather than return values. Fix them. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7147 Test Plan: Manually ingest a failure and see it to fail. Reviewed By: jay-zhuang Differential Revision: D22611594 fbshipit-source-id: 88a42425a41d1213d29bd2e7c80731d2bdd5644b | 18 July 2020, 01:07:08 UTC |
961dd62 | Stanislau Hlebik | 18 July 2020, 00:07:23 UTC | remediation of S205607 fbshipit-source-id: 798decc90db4f13770e97cdce3c0df7d5421b2a3 | 18 July 2020, 00:20:49 UTC |
961a496 | Stanislau Hlebik | 18 July 2020, 00:07:23 UTC | remediation of S205607 fbshipit-source-id: 5113fe0c527595e4227ff827253b7414abbdf7ac | 18 July 2020, 00:20:49 UTC |
9a83fd2 | Andrew Kryczka | 17 July 2020, 23:12:09 UTC | stagger first DumpMallocStats after opening DB (#7145) Summary: Previously when running `db_bench` with large value for `num_multi_dbs` and enabled `Options::dump_malloc_stats`, we would see most CPU spent in jemalloc locking. After this PR that no longer shows up at the top of the profile. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7145 Reviewed By: riversand963 Differential Revision: D22593031 Pulled By: ajkr fbshipit-source-id: 3b3fc91f93249c6afee53f59f34c487c3fc5add6 | 17 July 2020, 23:13:26 UTC |
ec711b2 | mrambacher | 17 July 2020, 02:04:44 UTC | Add Support for saving CompressionOptions to Options File (#6817) Summary: This PR does a few things: - The "compression_opts" and "bottom_compression_opts" can now be read/written as name/value pairs of options (instead of only a colon-separated list; - These options can now be read/written to the Options file; - The parallel_threads value can now be set (either in the colon or name-value format). The compression options are now stored and treated as a OptionTypeInfo::Struct by the options system, meaning they can be read and written like the other structs. This change allows them to be read/written easily to the options file. Additionally, the colon-format was extended to allow support for setting parallel threads. Tests were added to test all of the option settings via the optional parameters in the colon format. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6817 Reviewed By: ajkr Differential Revision: D22396004 Pulled By: zhichao-cao fbshipit-source-id: 38bcf74b7e9cd5bc2a84540fac2e9ba4f765b2c8 | 17 July 2020, 02:06:31 UTC |
c5ddece | Levi Tamasi | 16 July 2020, 01:52:12 UTC | Remove some more dead code around syncing blob files (#7138) Summary: Periodic syncing of blob files is handled by a lower layer, namely by `WritableFileWriter`; the `NeedsFsync` method of `BlobFile` and the `last_fsync_` member variable are actually unused and thus can be removed. See also https://github.com/facebook/rocksdb/pull/7125 . Pull Request resolved: https://github.com/facebook/rocksdb/pull/7138 Test Plan: `make check` Reviewed By: zhichao-cao Differential Revision: D22562981 Pulled By: ltamasi fbshipit-source-id: c235aad94a7c27120528c9ec270a7a5b9154e49f | 16 July 2020, 01:53:54 UTC |
a7feebd | Akanksha Mahajan | 16 July 2020, 00:46:18 UTC | Add "build-examples" in CircleCI (#7136) Summary: Add "examples" build (which build examples folder in rocksdb) in TravisCI to CircleCI. This is helpful before pull request. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7136 Test Plan: Watch for CircleCI results to succeed Reviewed By: jay-zhuang Differential Revision: D22555528 Pulled By: akankshamahajan15 fbshipit-source-id: 6bca16647760d5f0131f064765fe9e88e034c578 | 16 July 2020, 00:47:35 UTC |
ca5a069 | sdong | 15 July 2020, 20:24:04 UTC | Suppress a TSAN warning (#7126) Summary: TSAN shows warning with clang with warning similar to this: WARNING: ThreadSanitizer: data race (pid=10159) Atomic write of size 8 at 0x7b5000002890 by thread T33: #0 __tsan_atomic64_store <null> (db_test+0x4ca2b5) https://github.com/facebook/rocksdb/issues/1 std::__atomic_base<unsigned long>::store(unsigned long, std::memory_order) /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/atomic_base.h:374:2 (db_test+0x774fde) https://github.com/facebook/rocksdb/issues/2 rocksdb::VersionSet::SetLastSequence(unsigned long) /home/circleci/project/./db/version_set.h:1057:20 (db_test+0x774fde) https://github.com/facebook/rocksdb/issues/3 rocksdb::DBImpl::WriteImpl(rocksdb::WriteOptions const&, rocksdb::WriteBatch*, rocksdb::WriteCallback*, unsigned long*, unsigned long, bool, unsigned long*, unsigned long, rocksdb::PreReleaseCallback*) /home/circleci/project/db/db_impl/db_impl_write.cc:449:18 (db_test+0x774fde) ...... Previous read of size 8 at 0x7b5000002890 by thread T5 (mutexes: write M1044689462619020832): #0 rocksdb::DBImpl::ReleaseSnapshot(rocksdb::Snapshot const*) /home/circleci/project/db/db_impl/db_impl.cc (db_test+0x6f4ae7) https://github.com/facebook/rocksdb/issues/1 rocksdb::(anonymous namespace)::MTThreadBody(void*) /home/circleci/project/db/db_test.cc:2514:13 (db_test+0x56ac59) https://github.com/facebook/rocksdb/issues/2 rocksdb::(anonymous namespace)::StartThreadWrapper(void*) /home/circleci/project/env/env_posix.cc:443:3 (db_test+0x88c4cd) It is not limited to ReleaseSnapshot() and rocksdb::DBImpl::MultiCFSnapshot(). While we are not 100% sure it doesn't indicate any correctness violation, we suppress them for now to keep TSAN clean with more tests so that we can cover more bugs with CI. In the gcc runs we have been running, this warning rarely shows up. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7126 Test Plan: See the mini-TSAN test to pass with reasonable run time. Reviewed By: ajkr Differential Revision: D22552375 fbshipit-source-id: ebdd3854cb3becec3403970326a1ca961db2ab00 | 15 July 2020, 20:25:14 UTC |
ee8c79d | Levi Tamasi | 15 July 2020, 20:17:48 UTC | Turn the compression_type check in BlobDBImpl::DecompressSlice into an assertion (#7127) Summary: In both cases where `BlobDBImpl::DecompressSlice` is called, `compression_type` is already checked at the call site; thus, the check inside the method is redundant and can be turned into an assertion. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7127 Test Plan: `make check` Reviewed By: zhichao-cao Differential Revision: D22533454 Pulled By: ltamasi fbshipit-source-id: ae524443fc6abe0a5fb12327a3fe761a9cd2c831 | 15 July 2020, 20:19:14 UTC |
afb6bb1 | Jay Zhuang | 15 July 2020, 19:17:04 UTC | CircleCI: Pipe java-build to ignore EAGAIN errors (#7135) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/7135 Reviewed By: siying Differential Revision: D22553594 Pulled By: jay-zhuang fbshipit-source-id: 5b40621ffad05c4fffe3ac536a11d6c87e7ef512 | 15 July 2020, 19:18:32 UTC |
a10f12e | Zhichao Cao | 15 July 2020, 18:02:44 UTC | Auto resume the DB from Retryable IO Error (#6765) Summary: In current codebase, in write path, if Retryable IO Error happens, SetBGError is called. The retryable IO Error is converted to hard error and DB is in read only mode. User or application needs to resume it. In this PR, if Retryable IO Error happens in one DB, SetBGError will create a new thread to call Resume (auto resume). otpions.max_bgerror_resume_count controls if auto resume is enabled or not (if max_bgerror_resume_count<=0, auto resume will not be enabled). options.bgerror_resume_retry_interval controls the time interval to call Resume again if the previous resume fails due to the Retryable IO Error. If non-retryable error happens during resume, auto resume will terminate. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6765 Test Plan: Added the unit test cases in error_handler_fs_test and pass make asan_check Reviewed By: anand1976 Differential Revision: D21916789 Pulled By: zhichao-cao fbshipit-source-id: acb8b5e5dc3167adfa9425a5b7fc104f6b95cb0b | 15 July 2020, 18:03:58 UTC |
e8d5a24 | yxj25245 | 15 July 2020, 16:21:57 UTC | Fix typo in ThreadData comment (#7131) Summary: Fix typo in ThreadData comment Pull Request resolved: https://github.com/facebook/rocksdb/pull/7131 Reviewed By: riversand963 Differential Revision: D22543135 Pulled By: jay-zhuang fbshipit-source-id: 39c9d0e8cd5a364af9a2f05fd3783e8482dea976 | 15 July 2020, 16:23:23 UTC |
27735de | Yanqin Jin | 15 July 2020, 00:16:18 UTC | Report corrupted keys during compaction (#7124) Summary: Currently, RocksDB lets compaction to go through even in case of corrupted keys, the number of which is reported in CompactionJobStats. However, RocksDB does not check this value. We should let compaction run in a stricter mode. Temporarily disable two tests that allow corrupted keys in compaction. With this PR, the two tests will assert(false) and terminate. Still need to investigate what is the recommended google-test way of doing it. Death test (EXPECT_DEATH) in gtest has warnings now. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7124 Test Plan: make check Reviewed By: ajkr Differential Revision: D22530722 Pulled By: riversand963 fbshipit-source-id: 6a5a6a992028c6d4f92cb74693c92db462ae4ad6 | 15 July 2020, 00:18:17 UTC |
687fbd0 | Levi Tamasi | 14 July 2020, 23:51:49 UTC | Update some log messages in BlobDB to account for compaction filters (#7128) Summary: https://github.com/facebook/rocksdb/pull/6850, which added compaction filter support to BlobDB, reused elements of the BlobDB GC mechanism. This patch updates some log messages in this logic to account for this fact; namely, it replaces mentions of "GC" with "compaction/GC" to avoid confusion in cases when GC is not enabled. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7128 Test Plan: `make check` Reviewed By: zhichao-cao Differential Revision: D22535371 Pulled By: ltamasi fbshipit-source-id: 1f14f3b02ab9983728bbca1cf680420208d9a195 | 14 July 2020, 23:53:33 UTC |
de8c92a | Adam Retter | 14 July 2020, 23:00:05 UTC | Only check for python location once (#7123) Summary: This fixes an issue introduced in 0c56fc4 whereby the location of Python is evaluated many times and leads to excessive logging of unknown python locations of CentOS 6. The location is now only checked once. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7123 Reviewed By: zhichao-cao Differential Revision: D22532274 Pulled By: ajkr fbshipit-source-id: cade71b4b46e9a23d63ecb4dd36a4ac8ae217970 | 14 July 2020, 23:01:21 UTC |
e930ba1 | sdong | 14 July 2020, 22:31:57 UTC | Run a subset of TSAN tests in CIrcleCI (#7122) Summary: It is helpful to add some TSAN coverage before a pull request is committed. This diff adds some of them. Some slow tests are excluded for the running speed. Some are blacklisted because they show warnings. Will investigate these warnings and see whether we can fix or suppress them. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7122 Test Plan: Watch CIrcleCI runs Reviewed By: riversand963 Differential Revision: D22532133 fbshipit-source-id: 81ddd02d9df19c513a12811979e8ddabae911354 | 14 July 2020, 22:33:10 UTC |
bdf4de6 | Levi Tamasi | 14 July 2020, 20:50:26 UTC | Remove some dead code from BlobLogWriter (#7125) Summary: Periodic syncing of blob files is performed by `WritableFileWriter`; `bytes_per_sync_` and `next_sync_offset_` in `BlobLogWriter` are actually unused (or more precisely, only used by methods that are themselves unused). The patch removes all this dead code. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7125 Test Plan: `make check` Reviewed By: riversand963 Differential Revision: D22531021 Pulled By: ltamasi fbshipit-source-id: 6b293ad5a79d3e6bf15c5c68f7aedd7ce7a15f10 | 14 July 2020, 20:51:54 UTC |
fc4d5f5 | Jay Zhuang | 14 July 2020, 19:10:56 UTC | Add stress test for GetProperty (#7111) Summary: Add stress test coverage for `DB::GetProperty()`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7111 Test Plan: ``` ./db_stress -get_property_one_in=1 make crash_test ``` Reviewed By: ajkr Differential Revision: D22487906 Pulled By: jay-zhuang fbshipit-source-id: c118d95cc9b4e2fa669a06e6aa531541fa885dc5 | 14 July 2020, 19:12:36 UTC |
c628fae | Yanqin Jin | 14 July 2020, 03:25:37 UTC | Report corruption on unrecognized value type (#7121) Summary: During memtable lookup, an unrecognized value type should be reported as Status::Corruption. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7121 Test Plan: make check Reviewed By: cheng-chang Differential Revision: D22512124 Pulled By: riversand963 fbshipit-source-id: 9b97be7d9b230c5aae9205f96054420e5ea09066 | 14 July 2020, 03:26:58 UTC |
2f32d50 | sdong | 14 July 2020, 01:08:08 UTC | Add VS2017 to CircleCI Runs (#7120) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/7120 Test Plan: Watch CI results. Reviewed By: cheng-chang Differential Revision: D22516995 fbshipit-source-id: ddf7c0482dfac30d9044b833852348eda987cb67 | 14 July 2020, 01:10:20 UTC |
d93bd3c | Akanksha Mahajan | 13 July 2020, 23:35:29 UTC | Add FileSystem wrapper classes for IO tracing. (#7002) Summary: 1. Add the wrapper classes FileSystemTracingWrapper, FSSequentialFileTracingWrapper, FSRandomAccessFileTracingWrapper, FSWritableFileTracingWrapper, FSRandomRWFileTracingWrapper that forward the calls to underlying storage system and then pass the file operation information to IOTracer. IOTracer dumps the record in binary format for tracing. 2. Add the wrapper classes FileSystemPtr, FSSequentialFilePtr, FSRandomAccessFilePtr, FSWritableFilePtr and FSRandomRWFilePtr that overload operator-> and return ptr to underlying storage system or Tracing wrapper class based on enabling/disabling of IO tracing. These classes are added to bypass Tracing Wrapper classes when we disable tracing. 3. Add enums in trace.h that distinguish which options need to be added for different file operations(Read, close, write etc) as part of tracing record. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7002 Test Plan: make check -j64 Reviewed By: anand1976 Differential Revision: D22127897 Pulled By: akankshamahajan15 fbshipit-source-id: 74cff58ce5661c9a3832dfaa52483f3b2d8565e0 | 13 July 2020, 23:36:55 UTC |
0ff752c | Jay Zhuang | 13 July 2020, 22:04:27 UTC | Add circleci java build (#7119) Summary: Add circleci java build workflow. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7119 Reviewed By: siying Differential Revision: D22512426 Pulled By: jay-zhuang fbshipit-source-id: 45a7445c861fee48017bae42cdb7172c8b091475 | 13 July 2020, 22:06:22 UTC |
43cc622 | sdong | 13 July 2020, 19:31:41 UTC | Add CLANG analyze to CircleCI (#7114) Summary: CLANG analyze is useful before pull request. Add it. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7114 Test Plan: Watch the CI results to succeed. Reviewed By: riversand963 Differential Revision: D22491942 fbshipit-source-id: 9ccad91c6142fedc3d3dd491cf55054827908f36 | 13 July 2020, 19:33:16 UTC |
c193529 | sdong | 11 July 2020, 03:02:16 UTC | Add ubsan_check to CircleCI runs (#7112) Summary: It is useful to run UBSAN before merging a PR. This commit adds it. We see warning for stl_tree.h, suppress the warning to make it work. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7112 Test Plan: See the CI to succeed. Manually ingest a UBSAN warning and see it got reported. Reviewed By: pdillinger Differential Revision: D22490519 fbshipit-source-id: e4495a0c78a3e2dae7dbf294da79585e141cbb66 | 11 July 2020, 03:03:41 UTC |
a08f403 | Adam Retter | 10 July 2020, 21:31:25 UTC | Align RocksJava BlockBasedTableOptions with C++ API (#7088) Summary: Closes https://github.com/facebook/rocksdb/issues/6729 Pull Request resolved: https://github.com/facebook/rocksdb/pull/7088 Reviewed By: riversand963 Differential Revision: D22481624 Pulled By: pdillinger fbshipit-source-id: 27c0ebd4168d374ae81f3595e034150c1c97f8b8 | 10 July 2020, 21:32:58 UTC |
393e486 | Stanislav Tkach | 10 July 2020, 21:27:57 UTC | Add getters for options to the C API (#7094) Summary: Along with https://github.com/facebook/rocksdb/issues/6925 and https://github.com/facebook/rocksdb/issues/6998, this should add getters for all Options fields except several ones with non-trivial interface (for example rocksdb_options_set_min_level_to_compress). Pull Request resolved: https://github.com/facebook/rocksdb/pull/7094 Reviewed By: riversand963 Differential Revision: D22479800 Pulled By: pdillinger fbshipit-source-id: d14f305e12cfe268d07e0fe229d55cef299c792a | 10 July 2020, 21:30:04 UTC |
1a8ca66 | Adam Retter | 10 July 2020, 21:25:16 UTC | Make sure directory exists before attempting to write to it (#7090) Summary: Closes https://github.com/facebook/rocksdb/issues/7053 Pull Request resolved: https://github.com/facebook/rocksdb/pull/7090 Reviewed By: riversand963 Differential Revision: D22481199 Pulled By: pdillinger fbshipit-source-id: 287477db94d57b18bee58189135f44936f1c3ca3 | 10 July 2020, 21:27:03 UTC |
4924a50 | wenh | 10 July 2020, 20:39:47 UTC | Reduce `env_->GetChildren()` calls in DBImpl::Recover() (#7044) Summary: There currently exist multiple `GetChildren()` calls in `DBImpl::Recover()`, which can be expensive in cases of distributed file systems. This pull request try to call `DBImpl::Recover()` of each necessary directory only _once_ and reuse the results in the places of repeated calls in current code. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7044 Test Plan: Run `make check` and use the default test suite. The modified code should be semantically identical to the current code. As a proof of this solution, we may optionally deploy the system onto a (real or simulated) distributed system and expect reduced latency caused by manifest fetching. (WIP) Reviewed By: riversand963 Differential Revision: D22419925 Pulled By: roghnin fbshipit-source-id: d3774fbfbc246c5527101bc16747eb5c90919886 | 10 July 2020, 20:41:08 UTC |
a9a9738 | Zhichao Cao | 10 July 2020, 01:11:41 UTC | Fix status message size assert (#7045) Summary: In status.cc, the assert is `assert(sizeof(msgs) > index)`; msgs is a const char* array, sizeof(msgs) is the array size*char* size, which will make the assert pass all the time. Change it to sizeof(msgs)/sizeof(char*) > index. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7045 Test Plan: pass make check Reviewed By: cheng-chang Differential Revision: D22291337 Pulled By: zhichao-cao fbshipit-source-id: 4ba8ebbb8da80ace7ca6adcdb0c66726f993659d | 10 July 2020, 01:12:55 UTC |
c7c7b07 | mrambacher | 09 July 2020, 21:33:42 UTC | More Makefile Cleanup (#7097) Summary: Cleans up some of the dependencies on test code in the Makefile while building tools: - Moves the test::RandomString, DBBaseTest::RandomString into Random - Moves the test::RandomHumanReadableString into Random - Moves the DestroyDir method into file_utils - Moves the SetupSyncPointsToMockDirectIO into sync_point. - Moves the FaultInjection Env and FS classes under env These changes allow all of the tools to build without dependencies on test_util, thereby simplifying the build dependencies. By moving the FaultInjection code, the dependency in db_stress on different libraries for debug vs release was eliminated. Tested both release and debug builds via Make and CMake for both static and shared libraries. More work remains to clean up how the tools are built and remove some unnecessary dependencies. There is also more work that should be done to get the Makefile and CMake to align in their builds -- what is in the libraries and the sizes of the executables are different. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7097 Reviewed By: riversand963 Differential Revision: D22463160 Pulled By: pdillinger fbshipit-source-id: e19462b53324ab3f0b7c72459dbc73165cc382b2 | 09 July 2020, 21:35:17 UTC |
82611ee | Andrew Kryczka | 09 July 2020, 19:25:40 UTC | save key comparisons in BlockIter::BinarySeek (#7068) Summary: This is a followup to https://github.com/facebook/rocksdb/issues/6646. In that PR, for simplicity I just appended a comparison against the 0th restart key in case `BinarySeek()`'s binary search landed at index 0. As a result there were `2/(N+1) + log_2(N)` key comparisons. This PR does it differently. Now we expand the binary search range by one so it also covers the case where target is at or before the restart key at index 0. As a result, it involves `log_2(N+1)` key comparisons. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7068 Test Plan: ran readrandom with mostly default settings and counted key comparisons using `PerfContext`. before: `user_key_comparison_count = 28881965` after: `user_key_comparison_count = 27823245` setup command: ``` $ TEST_TMPDIR=/dev/shm/dbbench ./db_bench -benchmarks=fillrandom,compact -write_buffer_size=1048576 -target_file_size_base=1048576 -max_bytes_for_level_base=4194304 -max_background_jobs=12 -level_compaction_dynamic_level_bytes=true -num=10000000 ``` benchmark command: ``` $ TEST_TMPDIR=/dev/shm/dbbench/ ./db_bench -use_existing_db=true -benchmarks=readrandom -disable_auto_compactions=true -num=10000000 -compression_type=none -reads=1000000 -perf_level=3 ``` Reviewed By: anand1976 Differential Revision: D22357032 Pulled By: ajkr fbshipit-source-id: 8b01e9c1c2a4e9d02fc9dfe16c1cc0327f8bdf24 | 09 July 2020, 19:27:20 UTC |
f70ad03 | Yanqin Jin | 09 July 2020, 18:28:49 UTC | Parameterize a few tests in DBWALTest (#7105) Summary: As title. The goal is to shorten the execution time of several tests when they are combined together in a single TEST_F. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7105 Test Plan: make db_wal_test ./db_wal_test Reviewed By: ltamasi Differential Revision: D22442705 Pulled By: riversand963 fbshipit-source-id: 0ad49b8f21fa86dcd5a4d3c9a06af313735ac217 | 09 July 2020, 18:31:06 UTC |
842bd27 | Yanqin Jin | 09 July 2020, 17:18:52 UTC | Running ./ldb without any extra arg print usage (#7107) Summary: as title. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7107 Test Plan: make ldb && ./ldb Reviewed By: pdillinger Differential Revision: D22451399 Pulled By: riversand963 fbshipit-source-id: 797645e06473bb9cf139c533877e5161281515e8 | 09 July 2020, 17:20:06 UTC |
b35a2f9 | Zitan Chen | 09 July 2020, 15:36:41 UTC | Fix GetFileDbIdentities (#7104) Summary: Although PR https://github.com/facebook/rocksdb/issues/7032 fixes the construction of the `SstFileDumper` in `GetFileDbIdentities` by setting a proper `Env` of the `Options` passed in the constructor, the file path was not corrected accordingly. This actually disables backup engine to use db session ids in the file names since the `db_session_id` is always empty. Now it is fixed by setting the correct path in the construction of `SstFileDumper`. Furthermore, to preserve the Direct IO property that backup engine already has, parameter `EnvOptions` is added to `GetFileDbIdentities` and `SstFileDumper`. The `BackupUsingDirectIO` test is updated accordingly. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7104 Test Plan: backupable_db_test and some manual tests. Reviewed By: ajkr Differential Revision: D22443245 Pulled By: gg814 fbshipit-source-id: 056a9bb8b82947c5e73d7c3fbb62bfe23af5e562 | 09 July 2020, 15:37:59 UTC |
54f171f | Akanksha Mahajan | 09 July 2020, 04:02:06 UTC | Update Flush policy in PartitionedIndexBuilder on switching from user-key to internal-key mode (#7096) Summary: When format_version is high enough to support user-key and there are index entries for same user key that spans multiple data blocks then it changes from user-key mode to internal-key mode. But the flush policy is not reset to point to Block Builder of internal-keys. After this switch, no entries are added to user key index partition result, thus it never triggers flushing the block. Fix: 1. After adding the entry in sub_builder_index_, if there is a switch from user-key to internal-key, then flush policy is updated to point to Block Builder of internal-keys index partition. 2. Set sub_builder_index_->seperator_is_key_plus_seq_ = true if seperator_is_key_plus_seq_ is set to true so that subsequent partitions can also use internal key mode. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7096 Test Plan: make check -j64 Reviewed By: ajkr Differential Revision: D22416598 Pulled By: akankshamahajan15 fbshipit-source-id: 01fc2dc07ea1b32f8fb803995ebe6e9a3fbe67ac | 09 July 2020, 04:03:04 UTC |
7c6f3d8 | 球状闪电 | 09 July 2020, 01:50:58 UTC | fix compile error (#7040) Summary: WITH_TESTS=OFF and WITH_BENCHMARK_TOOLS=ON there has errors: /bin/ld: cannot find -ltestharness Pull Request resolved: https://github.com/facebook/rocksdb/pull/7040 Reviewed By: riversand963 Differential Revision: D22447637 Pulled By: ajkr fbshipit-source-id: f63058376deb4a2e6722d63541c40caa617c331a | 09 July 2020, 01:52:30 UTC |
90fd6b0 | Peter Dillinger | 08 July 2020, 20:03:17 UTC | cf_consistency_stress (crash_test_with_atomic_flush) checkpoint clean (#7103) Summary: Delicious copy-pasta from https://github.com/facebook/rocksdb/issues/7039 Also fixing DestroyDir to allow files to go missing while it is operating. This seems to fix failures I got with test plan reproducer. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7103 Test Plan: make blackbox_crash_test_with_atomic_flush for a while with checkpoint_one_in=100 Reviewed By: siying Differential Revision: D22435315 Pulled By: pdillinger fbshipit-source-id: 0ec0538402493887aeda43ecc03f32979cb84ced | 08 July 2020, 20:04:55 UTC |
cc5c680 | Zitan Chen | 08 July 2020, 19:15:02 UTC | Fix flaky BackupableDBTest.TableFileCorruptedBeforeBackup (#7102) Summary: The fix in PR https://github.com/facebook/rocksdb/issues/7082 is not really successful because there is still a small chance that the test will fail. In addtion to flushing, we close the DB and then reopen before corrupting a table file in the DB. Specifically, we corrupt a table file before backup takes place as follows. * Open DB * Fill DB * Flush DB (optional, no flushing here also works) * Close DB * Reopen DB * Corrupt a table file in the DB This should make the test reliable. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7102 Test Plan: `while ./backupable_db_test --gtest_filter=*TableFileCorruptedBeforeBackup*; do true; done` (kept running for an hour or so :) Reviewed By: pdillinger Differential Revision: D22432417 Pulled By: gg814 fbshipit-source-id: d407eee93ff428bb662f80cde1659fbf0149d0cd | 08 July 2020, 19:16:19 UTC |
712458f | rafael-aero | 08 July 2020, 18:54:30 UTC | Add RestoreDBFromLatestBackup to C API, add new C# package (#7092) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/7092 Reviewed By: riversand963 Differential Revision: D22412323 Pulled By: ajkr fbshipit-source-id: 3fc1c63bb19a8cd2c0ae620800c28f199a7f494b | 08 July 2020, 18:56:41 UTC |
b649d8c | rockeet | 08 July 2020, 18:52:13 UTC | Fixed Factory construct just for calling .Name() (#7080) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/7080 Reviewed By: riversand963 Differential Revision: D22412352 Pulled By: ajkr fbshipit-source-id: 1d7f4c1621040a0130245139b52c3f4d3deac865 | 08 July 2020, 18:54:00 UTC |
226d1f9 | wenh | 08 July 2020, 01:19:32 UTC | extend listener callback functions to more file I/O operations (#7055) Summary: Currently, `EventListener` in listner.h only have callback functions for file read and write. One may favor extended callback functions for more file I/O operations like flush, sync and close. This PR tries to add those interface and have them called when appropriate throughout the code base. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7055 Test Plan: Write an experimental listener with those new callback functions with log output in them; run experiments and check logs to see those functions are actually called. Default test suits `make check` should also be included. Reviewed By: riversand963 Differential Revision: D22380624 Pulled By: roghnin fbshipit-source-id: 4121491d45c2c2aae8c255e7998090559a241c6a | 08 July 2020, 01:21:18 UTC |
dd29ad4 | Andrew Kryczka | 08 July 2020, 00:25:08 UTC | Separate internal and user key comparators in `BlockIter` (#6944) Summary: Replace `BlockIter::comparator_` and `IndexBlockIter::user_comparator_wrapper_` with a concrete `UserComparatorWrapper` and `InternalKeyComparator`. The motivation for this change was the inconvenience of not knowing the concrete type of `BlockIter::comparator_`, which prevented calling specialized internal key comparison functions to optimize comparison of keys with global seqno applied. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6944 Test Plan: benchmark setup -- single file DBs, in-memory, no compression. "normal_db" created by regular flush; "ingestion_db" created by ingesting a file. Both DBs have same contents. ``` $ TEST_TMPDIR=/dev/shm/normal_db/ ./db_bench -benchmarks=fillrandom,compact -write_buffer_size=10485760000 -disable_auto_compactions=true -compression_type=none -num=1000000 $ ./ldb write_extern_sst ./tmp.sst --db=/dev/shm/ingestion_db/dbbench/ --compression_type=no --hex --create_if_missing < <(./sst_dump --command=scan --output_hex --file=/dev/shm/normal_db/dbbench/000007.sst | awk 'began {print "0x" substr($1, 2, length($1) - 2), "==>", "0x" $5} ; /^Sst file format: block-based/ {began=1}') $ ./ldb ingest_extern_sst ./tmp.sst --db=/dev/shm/ingestion_db/dbbench/ ``` benchmark run command: ``` $ TEST_TMPDIR=/dev/shm/$DB/ ./db_bench -benchmarks=seekrandom -seek_nexts=$SEEK_NEXT -use_existing_db=true -cache_index_and_filter_blocks=false -num=1000000 -cache_size=0 -threads=1 -reads=200000000 -mmap_read=1 -verify_checksum=false ``` results: perf improved marginally for ingestion_db and did not change significantly for normal_db: SEEK_NEXT | DB | code | ops/sec | % change -- | -- | -- | -- | -- 0 | normal_db | master | 350880 | 0 | normal_db | PR6944 | 351040 | 0.0 0 | ingestion_db | master | 343255 | 0 | ingestion_db | PR6944 | 349424 | 1.8 10 | normal_db | master | 218711 | 10 | normal_db | PR6944 | 217892 | -0.4 10 | ingestion_db | master | 220334 | 10 | ingestion_db | PR6944 | 226437 | 2.8 Reviewed By: pdillinger Differential Revision: D21924676 Pulled By: ajkr fbshipit-source-id: ea4288a2eefa8112eb6c651a671c1de18c12e538 | 08 July 2020, 00:26:16 UTC |
4202c48 | Peter Dillinger | 07 July 2020, 23:47:20 UTC | Replace large 'rm' with 'find' (#7095) Summary: On some platforms like MacOS, a second 'make check' can lead to /bin/rm: Argument list too long This is fixed by replacing with a 'find'. Also, using '-f' for more rm calls to avoid prompt. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7095 Test Plan: 'make check' on Linux and MacOS Reviewed By: riversand963 Differential Revision: D22415808 Pulled By: pdillinger fbshipit-source-id: 0fd1ebae13739c9d81f9e813e99b062715604d6b | 07 July 2020, 23:48:52 UTC |
787bf79 | Peter Dillinger | 07 July 2020, 23:47:20 UTC | Fix build of db_stress with LIB_MODE=shared (#7098) Summary: by tracking and linking against runtime dependent libraries in Makefile Pull Request resolved: https://github.com/facebook/rocksdb/pull/7098 Test Plan: look for fix in CircleCI Reviewed By: riversand963 Differential Revision: D22420860 Pulled By: pdillinger fbshipit-source-id: d211d709214bf5306db68e43b7a2f18169281022 | 07 July 2020, 23:48:52 UTC |
dbf5c55 | Peter Dillinger | 07 July 2020, 18:24:02 UTC | Exclude c_test from buck build opt mode (#7093) Summary: Fix a Facebook internal build Pull Request resolved: https://github.com/facebook/rocksdb/pull/7093 Test Plan: buck build mode/opt :c_test :c_test_bin (was compilation failure, now "not found") buck build mode/dev :c_test :c_test_bin (still passes) Reviewed By: ajkr Differential Revision: D22412528 Pulled By: pdillinger fbshipit-source-id: 8e55c43dbf95386597e4cc690c41d9cbdcee03aa | 07 July 2020, 18:28:22 UTC |
92731b6 | Peter Dillinger | 07 July 2020, 18:24:00 UTC | Major CircleCI/Linux fixes / tweaks / enhancements (#7078) Summary: Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get make[1]: write error: stdout probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393 Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem. Significant other changes: * Add another linux build that combines * LIB_MODE=shared, to ensure this works with compile and unit test execution * Alternative rocksdb namespace, to ensure this works (not rely on Travis) * ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it * Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!) * Use gflags with unity and LITE build, as typical case. Debugability improvements: * Use V=1 to show commands being executed (thanks to EAGAIN work-around) * Print kernel version and compiler versions as part of V=1 output from Makefile Cosmetic other changes: * Put more commands on one line, for less clutter in CircleCI output pages * Remove redundant "all" in "make all check" and put make command options before targets * Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078 Reviewed By: siying Differential Revision: D22391647 Pulled By: pdillinger fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135 | 07 July 2020, 18:25:46 UTC |
a693341 | Levi Tamasi | 07 July 2020, 00:10:41 UTC | Move the blob file format related classes to the main namespace, rename reader/writer (#7086) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/7086 Test Plan: `make check` Reviewed By: zhichao-cao Differential Revision: D22395420 Pulled By: ltamasi fbshipit-source-id: 088a20097bd6b73b0c433cd79725779f97ec04f2 | 07 July 2020, 00:18:14 UTC |
4b107ce | Peter Dillinger | 06 July 2020, 23:15:16 UTC | Improve code comments in EstimateLiveDataSize (#7072) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/7072 Reviewed By: ajkr Differential Revision: D22391641 Pulled By: pdillinger fbshipit-source-id: 0ef355576454514263ab684eb1a5c06787f3242a | 06 July 2020, 23:17:02 UTC |
899e59e | Adam Retter | 06 July 2020, 18:47:00 UTC | Add DB::OpenAsSecondary to RocksJava (#7047) Summary: Closes https://github.com/facebook/rocksdb/issues/5852 Pull Request resolved: https://github.com/facebook/rocksdb/pull/7047 Reviewed By: cheng-chang Differential Revision: D22335162 Pulled By: pdillinger fbshipit-source-id: 75f3c524deccea7ebc0ad288da41f1ea81406c1c | 06 July 2020, 18:48:57 UTC |
bd77e34 | Peter Dillinger | 06 July 2020, 18:18:11 UTC | More Makefile clean-up (#7066) Summary: (a) use STRESS_LIBRARY for db_stress and make sure STRESS_LIBRARY has other stress test dependencies (as in buck build) (b) fix rpath option to be accepted on MacOS. It still doesn't fully work for me e.g. to run a LIB_MODE=shared unit test binary from another directory, as it does on Linux, but the option is now accepted, and running unit tests from current directory works for me. Also adding LIB_MODE=shared to Travis. (Later TBD where best to fit in in CircleCI.) Pull Request resolved: https://github.com/facebook/rocksdb/pull/7066 Test Plan: manual Reviewed By: cheng-chang Differential Revision: D22364068 Pulled By: pdillinger fbshipit-source-id: 6fa98a222f89f808ee786474de1100d92c1adec3 | 06 July 2020, 18:19:48 UTC |
0117cbf | Adam Retter | 06 July 2020, 18:04:37 UTC | Adds a function to RocksJava for retrieving the version (#7083) Summary: Adds the function `RocksDB#rocksdbVersion()` for retrieving the RocksDB version. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7083 Reviewed By: cheng-chang Differential Revision: D22391628 Pulled By: pdillinger fbshipit-source-id: e1cabcf28aa81f5ee8dcdce5c9eca6b3155a279e | 06 July 2020, 18:06:21 UTC |
147f7b4 | Zitan Chen | 03 July 2020, 22:38:35 UTC | Fix flakiness of BackupableDBTest.TableFileCorruptedBeforeBackup (#7082) Summary: If the corruption of a table file is done before flushing, then db manifest may record the checksum for the corrupted table, which results in "matching checksums" when backup engine tries to verfiy the checksum, and causes a flaky test. Fix the issue by adding `Flush()` before trying to corrupt a table file in *db*. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7082 Test Plan: `buck test` Without the fix, failed 5 of 100 tests. Suspected whether the pseudo randomness causes the issue: doubling `keys_iteration` resulted in 2 of 100 tests failed; deterministically corrupting tables file also caused 2 of 100 tests to fail. With the fix, passed 200 of 200 tests. Reviewed By: pdillinger Differential Revision: D22375421 Pulled By: gg814 fbshipit-source-id: 7304618e7520684b6087e42d0b58329c5ad18329 | 03 July 2020, 22:40:04 UTC |