swh:1:snp:5115096b921df712aeb2a08114fede57fb3331fb
- HEAD
- refs/heads/2.2.fb.branch
- refs/heads/2.3.fb.branch
- refs/heads/2.4.fb.branch
- refs/heads/2.5.fb.branch
- refs/heads/2.6.fb.branch
- refs/heads/2.7
- refs/heads/2.7.fb.branch
- refs/heads/2.8.1.fb
- refs/heads/2.8.fb
- refs/heads/2.8.fb.trunk
- refs/heads/3.0.fb
- refs/heads/3.0.fb.branch
- refs/heads/3.1.fb
- refs/heads/3.10.fb
- refs/heads/3.11.fb
- refs/heads/3.12.fb
- refs/heads/3.13.fb
- refs/heads/3.2.fb
- refs/heads/3.3.fb
- refs/heads/3.4.fb
- refs/heads/3.5.fb
- refs/heads/3.6.fb
- refs/heads/3.7.fb
- refs/heads/3.8.fb
- refs/heads/3.9.fb
- refs/heads/4.0.fb
- refs/heads/4.1.fb
- refs/heads/4.10.fb
- refs/heads/4.11.fb
- refs/heads/4.12.fb
- refs/heads/4.13.fb
- refs/heads/4.2.fb
- refs/heads/4.3.fb
- refs/heads/4.4.fb
- refs/heads/4.5.fb
- refs/heads/4.6.fb
- refs/heads/4.7.fb
- refs/heads/4.8.fb
- refs/heads/4.9.fb
- refs/heads/5.0.fb
- refs/heads/5.1.fb
- refs/heads/5.10.fb
- refs/heads/5.11.fb
- refs/heads/5.12.fb
- refs/heads/5.13.fb
- refs/heads/5.13.fb.myrocks
- refs/heads/5.14.fb
- refs/heads/5.14.fb.myrocks
- refs/heads/5.15.fb
- refs/heads/5.16.fb
- refs/heads/5.17.fb
- refs/heads/5.17.fb.myrocks
- refs/heads/5.18.fb
- refs/heads/5.2.fb
- refs/heads/5.3.fb
- refs/heads/5.4.fb
- refs/heads/5.5.fb
- refs/heads/5.6.fb
- refs/heads/5.7.fb
- refs/heads/5.7.fb.myrocks
- refs/heads/5.8.3
- refs/heads/5.8.fb
- refs/heads/5.9.fb
- refs/heads/5.9.fb.myrocks
- refs/heads/6.0.fb
- refs/heads/6.0.fb.myrocks
- refs/heads/6.1.fb
- refs/heads/6.1.fb.myrocks
- refs/heads/6.1.fb.prod201905
- refs/heads/6.10.fb
- refs/heads/6.11.fb
- refs/heads/6.12.fb
- refs/heads/6.13.fb
- refs/heads/6.13.fb.laser
- refs/heads/6.14.fb
- refs/heads/6.14.fb.laser
- refs/heads/6.15.fb
- refs/heads/6.16.fb
- refs/heads/6.17.fb
- refs/heads/6.2.fb
- refs/heads/6.3.fb
- refs/heads/6.3.fb.myrocks
- refs/heads/6.3.fb.myrocks2
- refs/heads/6.3fb
- refs/heads/6.4.fb
- refs/heads/6.5.fb
- refs/heads/6.6.fb
- refs/heads/6.7.fb
- refs/heads/6.8.fb
- refs/heads/6.9.fb
- refs/heads/adaptive
- refs/heads/ajkr-patch-1
- refs/heads/blob_shadow
- refs/heads/bottom-pri-level
- refs/heads/bugfix-build-detect
- refs/heads/checksum_readahead_mmap_fix
- refs/heads/fb-config
- refs/heads/feature/debug-rocksdbjavastatic
- refs/heads/feature/travis-arm64
- refs/heads/fix-release-notes
- refs/heads/fix-write-batch-comment
- refs/heads/format_compatible_4
- refs/heads/getmergeops
- refs/heads/gh-pages-old
- refs/heads/history-update
- refs/heads/hotfix/lambda-capture
- refs/heads/improve-support
- refs/heads/jay-stress
- refs/heads/katherinez-patch-1
- refs/heads/katherinez-patch-2
- refs/heads/master
- refs/heads/pr-sanity-check-as-GHAction
- refs/heads/pr/6062
- refs/heads/ramvadiv-patch-1
- refs/heads/scaffold
- refs/heads/siying-patch-1
- refs/heads/siying-patch-10
- refs/heads/siying-patch-2
- refs/heads/siying-patch-3
- refs/heads/siying-patch-4
- refs/heads/skip_memtable_flush
- refs/heads/testing_ppc_build
- refs/heads/tests
- refs/heads/unused-var
- refs/heads/v6.6.4
- refs/heads/yiwu_stackable
- refs/heads/yuslepukhin
- refs/remotes/origin/5.13.fb
- refs/tags/2.5.fb
- refs/tags/2.6.fb
- refs/tags/3.0.fb
- refs/tags/do-not-use-me2
- refs/tags/rocksdb-3.1
- refs/tags/rocksdb-3.10.2
- refs/tags/rocksdb-3.11
- refs/tags/rocksdb-3.11.1
- refs/tags/rocksdb-3.11.2
- refs/tags/rocksdb-3.2
- refs/tags/rocksdb-3.3
- refs/tags/rocksdb-3.4
- refs/tags/rocksdb-3.5
- refs/tags/rocksdb-3.5.1
- refs/tags/rocksdb-3.6.1
- refs/tags/rocksdb-3.6.2
- refs/tags/rocksdb-3.7
- refs/tags/rocksdb-3.8
- refs/tags/rocksdb-3.9
- refs/tags/rocksdb-3.9.1
- refs/tags/rocksdb-4.1
- refs/tags/rocksdb-5.10.2
- refs/tags/rocksdb-5.10.3
- refs/tags/rocksdb-5.10.4
- refs/tags/rocksdb-5.11.2
- refs/tags/rocksdb-5.11.3
- refs/tags/rocksdb-5.14.3
- refs/tags/rocksdb-5.2.1
- refs/tags/rocksdb-5.3.3
- refs/tags/rocksdb-5.3.4
- refs/tags/rocksdb-5.3.5
- refs/tags/rocksdb-5.3.6
- refs/tags/rocksdb-5.4.10
- refs/tags/rocksdb-5.4.5
- refs/tags/rocksdb-5.4.6
- refs/tags/rocksdb-5.5.2
- refs/tags/rocksdb-5.5.3
- refs/tags/rocksdb-5.5.4
- refs/tags/rocksdb-5.5.5
- refs/tags/rocksdb-5.5.6
- refs/tags/rocksdb-5.6.1
- refs/tags/rocksdb-5.6.2
- refs/tags/rocksdb-5.7.1
- refs/tags/rocksdb-5.7.2
- refs/tags/rocksdb-5.7.3
- refs/tags/rocksdb-5.7.5
- refs/tags/rocksdb-5.8.6
- refs/tags/rocksdb-5.8.7
- refs/tags/rocksdb-5.8.8
- refs/tags/rocksdb-5.9.2
- refs/tags/v4.0
- refs/tags/v4.1
- refs/tags/v5.10.2
- refs/tags/v5.10.3
- refs/tags/v5.10.4
- refs/tags/v5.11.2
- refs/tags/v5.11.3
- refs/tags/v5.13.3
- refs/tags/v5.14.3
- refs/tags/v5.15.10
- refs/tags/v5.18.3
- refs/tags/v5.2.1
- refs/tags/v5.3.3
- refs/tags/v5.3.4
- refs/tags/v5.3.5
- refs/tags/v5.3.6
- refs/tags/v5.4.10
- refs/tags/v5.4.5
- refs/tags/v5.4.6
- refs/tags/v5.5.2
- refs/tags/v5.5.3
- refs/tags/v5.5.4
- refs/tags/v5.5.5
- refs/tags/v5.5.6
- refs/tags/v5.6.1
- refs/tags/v5.6.2
- refs/tags/v5.7.1
- refs/tags/v5.7.2
- refs/tags/v5.7.3
- refs/tags/v5.7.5
- refs/tags/v5.8.6
- refs/tags/v5.8.7
- refs/tags/v5.8.8
- refs/tags/v5.9.2
- refs/tags/v6.0.1
- refs/tags/v6.0.2
- refs/tags/v6.1.1
- refs/tags/v6.1.2
- refs/tags/v6.10.1
- refs/tags/v6.10.2
- refs/tags/v6.11.4
- refs/tags/v6.11.6
- refs/tags/v6.12.6
- refs/tags/v6.12.7
- refs/tags/v6.13.2
- refs/tags/v6.13.3
- refs/tags/v6.14.5
- refs/tags/v6.14.6
- refs/tags/v6.15.4
- refs/tags/v6.2.2
- refs/tags/v6.2.4
- refs/tags/v6.3.6
- refs/tags/v6.4.6
- refs/tags/v6.5.2
- refs/tags/v6.5.3
- refs/tags/v6.6.3
- refs/tags/v6.6.4
- refs/tags/v6.7.3
- refs/tags/v6.8.1
- v6.15.2
- v5.8
- v5.5.1
- v5.4.7
- v5.18.4
- v5.17.2
- v5.16.6
- v5.14.2
- v5.13.4
- v5.13.2
- v5.13.1
- v5.12.5
- v5.12.4
- v5.12.3
- v5.12.2
- v5.1.4
- v5.1.3
- v5.1.2
- v5.0.2
- v5.0.1
- v4.9
- v4.8
- v4.6.1
- v4.5.1
- v4.4.1
- v4.4
- v4.3.1
- v4.3
- v4.2
- v4.13.5
- v4.13
- v4.11.2
- v3.9
- v3.8
- v3.7
- v3.6.1
- v3.5
- v3.4
- v3.3
- v3.2
- v3.13.1
- v3.13
- v3.12.1
- v3.12
- v3.11
- v3.10
- v3.1
- v3.0
- v2.8
- v2.7
- v2.6
- v2.5
- v2.4
- v2.3
- v2.2
- v2.1
- v2.0
- v1.5.9.1
- v1.5.8.2
- v1.5.8.1
- v1.5.8
- v1.5.7
- rocksdb-5.8
- rocksdb-5.4.7
- rocksdb-5.1.4
- rocksdb-5.1.3
- rocksdb-5.1.2
- rocksdb-5.0.2
- rocksdb-5.0.1
- rocksdb-4.9
- rocksdb-4.8
- rocksdb-4.6.1
- rocksdb-4.5.1
- rocksdb-4.4.1
- rocksdb-4.4
- rocksdb-4.3.1
- rocksdb-4.3
- rocksdb-4.2
- rocksdb-4.13.5
- rocksdb-4.13
- rocksdb-4.11.2
- rocksdb-3.10.1
- 2.8.fb
- 2.7.fb
- 2.4.fb
- 2.3.fb
- 2.2.fb
- 2.1.fb
- 2.0.fb
- 1.5.9.fb
- 1.5.9.2.fb
- 1.5.9.1.fb
- 1.5.8.fb
- 1.5.8.2.fb
- 1.5.8.1.fb
- 1.5.7.fb
Permalinks
To reference or cite the objects present in the Software Heritage archive, permalinks based on SoftWare Hash IDentifiers (SWHIDs) must be used.
Select below a type of object currently browsed in order to display its associated SWHID and permalink.
Revision | Author | Date | Message | Commit Date |
---|---|---|---|---|
0ef9251 | Yanqin Jin | 05 May 2020, 17:44:12 UTC | Do not swallow error returned from SaveTo() (#6801) Summary: With consistency check enabled, VersionBuilder::SaveTo() may return error once corruption is detected while building versions. We should handle these errors. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6801 Test Plan: make check Reviewed By: siying Differential Revision: D21385045 Pulled By: riversand963 fbshipit-source-id: 98f6424e2a4699b62befa21e9fe00e70a771118e | 08 May 2020, 16:52:53 UTC |
ec33e93 | Peter Dillinger | 08 May 2020, 16:20:38 UTC | Fix compilation of compaction_picker_test after cherry-pick | 08 May 2020, 16:20:38 UTC |
70fb77b | anand76 | 08 May 2020, 00:30:41 UTC | Fix race due to delete triggered compaction in Universal compaction mode (#6799) Summary: Delete triggered compaction in universal compaction mode was causing a corruption when scheduled in parallel with other compactions. 1. When num_levels = 1, a file marked for compaction may be picked along with all older files in L0, without checking if any of them are already being compaction. This can cause unpredictable results like resurrection of older versions of keys or deleted keys. 2. When num_levels > 1, a delete triggered compaction would not get scheduled if it overlaps with a running regular compaction. However, the reverse is not true. This is due to the fact that in ```UniversalCompactionBuilder::CalculateSortedRuns```, it assumes that entire sorted runs are picked for compaction and only checks the first file in a sorted run to determine conflicts. This is violated by a delete triggered compaction as it works on a subset of a sorted run. Fix the bug for num_levels > 1, and disable the feature for now when num_levels = 1. After disabling this feature, files would still get marked for compaction, but no compaction would get scheduled. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6799 Reviewed By: pdillinger Differential Revision: D21431286 Pulled By: anand1976 fbshipit-source-id: ae9f0bdb1d6ae2f10284847db731c23f43af164a | 08 May 2020, 15:47:34 UTC |
442bd69 | Peter Dillinger | 07 May 2020, 22:39:49 UTC | Fix false NotFound from batched MultiGet with kHashSearch (#6821) Summary: The error is assigning KeyContext::s to NotFound status in a table reader for a "not found in this table" case, which skips searching in later tables, like only a delete should. (The hash search index iterator is the only one that can return status NotFound even if Valid() == false.) This was detected by intermittent failure in MultiThreadedDBTest.MultiThreaded/5, a kHashSearch configuration. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6821 Test Plan: modified existing unit test to reproduce problem Reviewed By: anand1976 Differential Revision: D21450469 Pulled By: pdillinger fbshipit-source-id: 7478003684d637dbd491cdac81468041a791be2c | 08 May 2020, 15:46:12 UTC |
25e3327 | sdong | 04 May 2020, 21:15:55 UTC | Avoid Swallowing Some File Consistency Checking Bugs (#6793) Summary: We are swallowing some file consistency checking failures. This is not expected. We are fixing two cases: DB reopen and manifest dump. More places are not fixed and need follow-up. Error from CheckConsistencyForDeletes() is also swallowed, which is not fixed in this PR. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6793 Test Plan: Add a unit test to cover the reopen case. Reviewed By: riversand963 Differential Revision: D21366525 fbshipit-source-id: eb438a322237814e8d5125f916a3c6de97f39ded | 08 May 2020, 15:45:41 UTC |
26979ce | Andrew Kryczka | 05 May 2020, 21:52:18 UTC | fixup 33087a2 "fix swallowed error for file deletion consistency check (#6809)" I forgot to amend changes to the above backport commit before pushing it. | 07 May 2020, 21:23:59 UTC |
33087a2 | Andrew Kryczka | 05 May 2020, 21:52:18 UTC | fix swallowed error for file deletion consistency check (#6809) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/6809 Reviewed By: pdillinger Differential Revision: D21411971 Pulled By: ajkr fbshipit-source-id: 900b6b0370b76e9a3e5e03f968e2ac1bbaab73b8 | 07 May 2020, 21:11:06 UTC |
ecf1499 | Zhichao Cao | 01 May 2020, 04:39:38 UTC | Update to RocksDB 6.9.4 | 01 May 2020, 04:39:38 UTC |
e89f9ef | Zhichao Cao | 30 April 2020, 18:45:59 UTC | Fix the status overwrite | 01 May 2020, 01:40:21 UTC |
931b7f6 | Yanqin Jin | 28 April 2020, 22:44:44 UTC | Update HISTORY.md and bump version | 28 April 2020, 22:44:44 UTC |
59530e3 | Yanqin Jin | 28 April 2020, 05:47:44 UTC | Fix timestamp support for MultiGet (#6748) Summary: 1. Avoid nullptr dereference when passing timestamp to KeyContext creation. 2. Construct LookupKey correctly with timestamp when creating MultiGetContext. 3. Compare without timestamp when sorting KeyContexts. Fixes https://github.com/facebook/rocksdb/issues/6745 Test plan (dev server): make check Pull Request resolved: https://github.com/facebook/rocksdb/pull/6748 Reviewed By: pdillinger Differential Revision: D21258691 Pulled By: riversand963 fbshipit-source-id: 44e65b759c18b9986947783edf03be4f890bb004 | 28 April 2020, 21:26:34 UTC |
2d0ed8d | Yanqin Jin | 23 April 2020, 23:18:28 UTC | Update CURRENT file after best-efforts recovery (#6746) Summary: After a successful recovery, the CURRENT file should be updated to point to the valid MANIFEST. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6746 Test Plan: make check Reviewed By: anand1976 Differential Revision: D21189876 Pulled By: riversand963 fbshipit-source-id: 7537b49988c5c425ebe9505a5cc260de351ad79b | 28 April 2020, 21:23:22 UTC |
dbba39e | Peter Dillinger | 28 April 2020, 19:27:23 UTC | HISTORY.md update for bzip upgrade (#6767) Summary: See https://github.com/facebook/rocksdb/issues/6714 and https://github.com/facebook/rocksdb/issues/6703 Pull Request resolved: https://github.com/facebook/rocksdb/pull/6767 Reviewed By: riversand963 Differential Revision: D21283307 Pulled By: pdillinger fbshipit-source-id: 8463bec725669d13846c728ad4b5bde43f9a84f8 | 28 April 2020, 20:51:36 UTC |
295b8bd | Adam Retter | 16 April 2020, 22:32:34 UTC | Update RocksJava static version of bzip2 (#6714) Summary: Updates the version of bzip2 used for RocksJava static builds. Please, can we also get this cherry-picked to: 1. 6.7.fb 2. 6.8.fb 3. 6.9.fb Pull Request resolved: https://github.com/facebook/rocksdb/pull/6714 Reviewed By: cheng-chang Differential Revision: D21067233 Pulled By: pdillinger fbshipit-source-id: 8164b7eb99c5ca7b2021ab8c371ba9ded4cb4f7e | 16 April 2020, 23:02:28 UTC |
bda56d9 | akankshamahajan | 16 April 2020, 22:20:24 UTC | Update to RocksDB 6.9.2 | 16 April 2020, 22:20:24 UTC |
0eda7f4 | Siying Dong | 16 April 2020, 20:55:05 UTC | Fix a backporting error in options_settable_test.cc (#6718) options_settable_test.cc has unit tests backported after compression_opt changed. Fix it. | 16 April 2020, 20:55:05 UTC |
00b4ed2 | akankshamahajan | 15 April 2020, 23:45:01 UTC | Add a feature to HISTORY.md | 15 April 2020, 23:45:01 UTC |
2b7ee8e | Zhichao Cao | 14 April 2020, 02:11:19 UTC | Add NewFileChecksumGenCrc32cFactory to file checksum (#6688) Summary: Add NewFileChecksumGenCrc32cFactory to file checksum public interface such that applications can use the build in crc32 checksum factory. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6688 Test Plan: pass make asan_check Reviewed By: riversand963 Differential Revision: D21006859 Pulled By: zhichao-cao fbshipit-source-id: ea8a45196a8b77c310728ab05f6cc0f49f3baef0 | 15 April 2020, 23:23:21 UTC |
79f39ff | anand76 | 15 April 2020, 22:32:28 UTC | Update HISTORY.md | 15 April 2020, 22:32:28 UTC |
b54712e | anand76 | 14 April 2020, 02:56:02 UTC | Log CompactOnDeletionCollectorFactory parameters on DB open (#6686) Summary: Log it in the info log to help in troubleshooting. It is logged as follows - ``` 2020/04/10-10:51:39.886662 7ffff7fef340 Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 100 Deletion trigger = 90); ``` Tests: make check Pull Request resolved: https://github.com/facebook/rocksdb/pull/6686 Reviewed By: ltamasi Differential Revision: D21002442 Pulled By: anand1976 fbshipit-source-id: 7adf0dbae7f1febcb00ce61fea5097118ede5c6a | 14 April 2020, 23:01:38 UTC |
272d006 | Akanksha Mahajan | 10 April 2020, 22:27:47 UTC | Report kFilesMarkedForCompaction for delete triggered compactions (#6680) Summary: Summary : Set manual_compaction false in case of DeleteTriggeredCompaction object so that kFilesMarkedForComapaction can be reported. Added a DeletionTriggeredUniversalCompactionMarking test case for Deletion Triggered compaction in case of Universal Compaction. Test Plan : make check -j64 Pull Request resolved: https://github.com/facebook/rocksdb/pull/6680 Test Plan: make check -j64 Reviewed By: anand1976 Differential Revision: D20945946 Pulled By: akankshamahajan15 fbshipit-source-id: af84e417bd7127652aaae9143c560d1ab3815d25 | 14 April 2020, 20:21:40 UTC |
51b25da | Zhichao Cao | 09 April 2020, 21:54:33 UTC | Updated to RocksDB 6.9.1 | 09 April 2020, 21:54:33 UTC |
8f2ef73 | Cheng Chang | 09 April 2020, 18:17:39 UTC | Add two more optimization improvements to HISTORY (#6679) Summary: Although these optimizations are not user facing, still feel it's valuable to call out in HISTORY. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6679 Test Plan: no need Reviewed By: zhichao-cao Differential Revision: D20945916 Pulled By: cheng-chang fbshipit-source-id: f3e790c07f3bcc4a8a74246c4fa232800ddd4438 | 09 April 2020, 21:42:56 UTC |
a964d45 | Cheng Chang | 09 April 2020, 04:17:42 UTC | Fix result slice's address for direct io read (#6672) Summary: When aligned_buf is provided, the result slice's starting address should take offset advance into account. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6672 Test Plan: make check Reviewed By: anand1976 Differential Revision: D20934198 Pulled By: cheng-chang fbshipit-source-id: c3475c9c132b92c50d8c7c399fca2e9e76870803 | 09 April 2020, 21:37:57 UTC |
0279d27 | Yi Wu | 09 April 2020, 04:19:49 UTC | Fix wrong key being read on ingested file with global seqno and delta encoding (#6669) Summary: On reading an ingested SST file, `DataBlockIter` will replace seqno encoded in a key with global seqno. However, if the original seqno was part of the prefix used for the next key, the global seqno is by mistake used as part of the prefix to construct the next key, causing wrong result being returned. Although at this point it is only software error while data in the file is not corrupted, the issue can further cause compaction output out of order and corrupted result when the ingested SST participated in compaction. Fixing the issue by save the actual seqno and restore it before the key being used as prefix to construct next key. The unit test is by Little-Wallace from https://github.com/facebook/rocksdb/issues/6666. Fixing https://github.com/facebook/rocksdb/issues/6666. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6669 Test Plan: New unit test Signed-off-by: Yi Wu <yiwu@pingcap.com> Reviewed By: cheng-chang Differential Revision: D20931808 Pulled By: ajkr fbshipit-source-id: f01959c35d6a493954dca981663766c7a5a9e8ab | 09 April 2020, 21:37:23 UTC |
20bc516 | sdong | 09 April 2020, 18:20:33 UTC | Fix memory corruption caused by new test in options_settable_test (#6676) Summary: https://github.com/facebook/rocksdb/pull/6668 added some new test code but it has a risk of memory corruption. Fix it Pull Request resolved: https://github.com/facebook/rocksdb/pull/6676 Test Plan: Run the test under ASAN and see it passes. Reviewed By: ajkr Differential Revision: D20937108 fbshipit-source-id: 22cc96bb02030df0a37a02e67a2cc37ca31ba22d | 09 April 2020, 20:27:21 UTC |
cc0b5a4 | sdong | 08 April 2020, 21:37:01 UTC | compression related options are not copied back from MutableCFOptions… (#6668) Summary: … to CFOptions https://github.com/facebook/rocksdb/pull/6615 made several compression related options dynamically changeable. They are moved to MutableCFOptions. However, they are not copied back to ColumnFamilyOptions, so the changed values are not written to option files and for some other uses. Fix it by copying them back. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6668 Test Plan: Add a unit test to make sure that when a MutableCFOptions is converted to CFOptions and back to MutableCFOptions, they stay the same. This test would fail without the fix. Reviewed By: ajkr Differential Revision: D20923999 fbshipit-source-id: c3bccd6923b00d677764e2269bed6a95ad7ed780 | 09 April 2020, 00:32:33 UTC |
145d48a | Andrew Gallagher | 31 March 2020, 18:36:22 UTC | Fix jemalloc forward declarations (#6613) Summary: Add `nothrow` attribute to match declarations in jemalloc. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6613 Reviewed By: igorsugak Differential Revision: D20749490 fbshipit-source-id: 9ac8df27f7b4268f27b32b130c23ce8a1f772b3a | 07 April 2020, 23:24:40 UTC |
b3f017f | Zhichao Cao | 03 April 2020, 01:06:26 UTC | Fix the multi-thread Manifest write dependency in error_handler_fs_test (#6637) Summary: In CompactionManifestWriteRetryableError in error_handler_fs_test, the manifest write of flush should pass with no fs error. After flush, fs is set to error status and the manifest write of compaction should fail due to the IO Error. Currently, the manifest write of flush is not synced with the compaction in order, which might cause manifest write fails, which will cause test failure. Fixed by adding the LoadDependency of sync-point after flush and before compaction. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6637 Test Plan: pass error_hanlder_fs_tes. Pass make asan_check Reviewed By: anand1976 Differential Revision: D20826969 Pulled By: zhichao-cao fbshipit-source-id: fb2e702caa19bd63c82570320536b7acda870ff1 | 06 April 2020, 23:45:53 UTC |
3eb30d6 | Zhichao Cao | 02 April 2020, 19:09:52 UTC | Remove redundant in HISTORY (#6627) Summary: Remove redundant description in HISTORY no code change Pull Request resolved: https://github.com/facebook/rocksdb/pull/6627 Test Plan: make check Reviewed By: anand1976 Differential Revision: D20797269 Pulled By: zhichao-cao fbshipit-source-id: dee4c9a22f6d241c985f250c0f11bfaa9198f4c1 | 02 April 2020, 23:44:49 UTC |
06c2935 | sdong | 01 April 2020, 18:24:58 UTC | Fix a bug that crashes the service when write buffer manager fails to insert to block cache (#6619) Summary: https://github.com/facebook/rocksdb/issues/6247 reports that when write buffer manager fails to insert the dummy entry to block cache, null pointer is still stored and used to release the handle and cause corruption. Fix the bug by not releasing it with null handle. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6619 Test Plan: Add a unit test that fails without the fix. Reviewed By: ajkr Differential Revision: D20776769 fbshipit-source-id: 4127fbd9f295a0a3e45774746ffcd91f939f6287 | 01 April 2020, 18:38:30 UTC |
b3621aa | Levi Tamasi | 31 March 2020, 23:09:11 UTC | Revert the recent cache deleter change (#6620) Summary: Revert "Use function objects as deleters in the block cache (https://github.com/facebook/rocksdb/issues/6545)" This reverts commit 6301dbe7a71d3663b87f66b3201ff8745bc48742. Revert "Call out the cache deleter related interface change in HISTORY.md (https://github.com/facebook/rocksdb/issues/6606)" This reverts commit 3a35542f8639cc9721db9fd71e964001f7291505. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6620 Test Plan: `make check` Reviewed By: zhichao-cao Differential Revision: D20773311 Pulled By: ltamasi fbshipit-source-id: 7637a761f718f323ef0e7da959462e8fb06e7a2b | 31 March 2020, 23:21:36 UTC |
a8c6000 | sdong | 31 March 2020, 19:08:41 UTC | Make options.bottommost_compression, compression_opts and bottommost_compression_opts dynamically changeable. (#6615) Summary: These three options should be made dynamically changeable. Simply add them to MutableCFOptions and made the change. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6615 Test Plan: Add a unit test to make sure that SetOptions() can change the options. Reviewed By: riversand963 Differential Revision: D20755951 fbshipit-source-id: 8165f4fd7a7a665cc7fb049698935022a5d2e7ff | 31 March 2020, 19:49:12 UTC |
b490f3d | Yanqin Jin | 31 March 2020, 01:57:28 UTC | Use flush time for the props.creation_time for FIFO compaction (#6612) Summary: For FIFO compaction, we use flush time instead of oldest key time as the creation time. This is to prevent FIFO compaction dropping files whose oldest key time is older than TTL but which has newer keys than TTL. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6612 Test Plan: make check Reviewed By: siying Differential Revision: D20748217 Pulled By: riversand963 fbshipit-source-id: 3f7b00a847020760537cdddd12f6fe039e5bc663 | 31 March 2020, 19:48:42 UTC |
eaf95c7 | Zhichao Cao | 30 March 2020, 23:28:46 UTC | Update release version to 6.9.0 (#6610) Summary: Update release version to 6.9.0 Pull Request resolved: https://github.com/facebook/rocksdb/pull/6610 Test Plan: no code change Reviewed By: riversand963 Differential Revision: D20741094 Pulled By: zhichao-cao fbshipit-source-id: 80a9e9ea8d164b6923112352d36fcbc1be85c034 | 30 March 2020, 23:31:02 UTC |
2ae91c6 | Zhichao Cao | 30 March 2020, 21:08:55 UTC | Fix potential memory leak in table_test (#6611) Summary: The checksum generator should be released if file_writer fails to reset the pointer. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6611 Test Plan: pass make asan_check Reviewed By: riversand963 Differential Revision: D20742964 Pulled By: zhichao-cao fbshipit-source-id: cde41be2edb3d1e56083c2b93e1510fb32556146 | 30 March 2020, 21:11:27 UTC |
cb671ea | phantomape | 30 March 2020, 04:54:47 UTC | env: Add clearerr() before repeating an interrupted file read (#6609) Summary: This change updates PosixSequentialFile::Read to call clearerr() before fread()ing again after an EINTR is returned on a previous fread. The original fix is from https://github.com/cockroachdb/rocksdb/commit/bd8f1ebb91bbf0e668d24faef273042cc1fe52de. Fixing https://github.com/facebook/rocksdb/issues/6509 Signed-off-by: phantomape <cxucheng@outlook.com> Pull Request resolved: https://github.com/facebook/rocksdb/pull/6609 Reviewed By: zhichao-cao Differential Revision: D20731482 Pulled By: riversand963 fbshipit-source-id: 7f1f3a1449077d5560f45c465a78d08633740ba0 | 30 March 2020, 04:56:31 UTC |
e8d332d | Zhichao Cao | 29 March 2020, 22:57:02 UTC | Use FileChecksumGenFactory for SST file checksum (#6600) Summary: In the current implementation, sst file checksum is calculated by a shared checksum function object, which may make some checksum function hard to be applied here such as SHA1. In this implementation, each sst file will have its own checksum generator obejct, created by FileChecksumGenFactory. User needs to implement its own FilechecksumGenerator and Factory to plugin the in checksum calculation method. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6600 Test Plan: tested with make asan_check Reviewed By: riversand963 Differential Revision: D20717670 Pulled By: zhichao-cao fbshipit-source-id: 2a74c1c280ac11a07a1980185b43b671acaa71c6 | 29 March 2020, 22:58:46 UTC |
ee50b8d | Cheng Chang | 29 March 2020, 02:05:54 UTC | Be able to decrease background thread's CPU priority when creating database backup (#6602) Summary: When creating a database backup, the background threads will not only consume IO resources by copying files, but also consuming CPU such as by computing checksums. During peak times, the CPU consumption by the background threads might affect online queries. This PR makes it possible to decrease CPU priority of these threads when creating a new backup. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6602 Test Plan: make check Reviewed By: siying, zhichao-cao Differential Revision: D20683216 Pulled By: cheng-chang fbshipit-source-id: 9978b9ed9488e8ce135e90ca083e5b4b7221fd84 | 29 March 2020, 02:07:25 UTC |
3a35542 | Levi Tamasi | 27 March 2020, 23:15:16 UTC | Call out the cache deleter related interface change in HISTORY.md (#6606) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/6606 Reviewed By: riversand963 Differential Revision: D20708411 Pulled By: ltamasi fbshipit-source-id: c15b4ded19a4b5c84e3e4240bdcec15460806c88 | 27 March 2020, 23:18:23 UTC |
3881a67 | Cheng Chang | 27 March 2020, 23:12:40 UTC | Refactor IsLockExpired (#6586) Summary: 1. If expiration_time is non-positive, no need to call NowMicros, save a syscall. 2. expire_time should only be set when expired is false. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6586 Test Plan: make check Reviewed By: lth Differential Revision: D20673730 Pulled By: cheng-chang fbshipit-source-id: a69e8d7b16dc6d0d00487bb1c19f0710d79482e2 | 27 March 2020, 23:14:22 UTC |
4246888 | Zhichao Cao | 27 March 2020, 23:03:05 UTC | Pass IOStatus to write path and set retryable IO Error as hard error in BG jobs (#6487) Summary: In the current code base, we use Status to get and store the returned status from the call. Specifically, for IO related functions, the current Status cannot reflect the IO Error details such as error scope, error retryable attribute, and others. With the implementation of https://github.com/facebook/rocksdb/issues/5761, we have the new Wrapper for IO, which returns IOStatus instead of Status. However, the IOStatus is purged at the lower level of write path and transferred to Status. The first job of this PR is to pass the IOStatus to the write path (flush, WAL write, and Compaction). The second job is to identify the Retryable IO Error as HardError, and set the bg_error_ as HardError. In this case, the DB Instance becomes read only. User is informed of the Status and need to take actions to deal with it (e.g., call db->Resume()). Pull Request resolved: https://github.com/facebook/rocksdb/pull/6487 Test Plan: Added the testing case to error_handler_fs_test. Pass make asan_check Reviewed By: anand1976 Differential Revision: D20685017 Pulled By: zhichao-cao fbshipit-source-id: ff85f042896243abcd6ef37877834e26f36b6eb0 | 27 March 2020, 23:04:43 UTC |
2e27697 | Cheng Chang | 27 March 2020, 22:59:21 UTC | Compute cv_end_time with simpler logic (#6585) Summary: The refactored logic is easier to read. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6585 Test Plan: make check Reviewed By: lth Differential Revision: D20663225 Pulled By: cheng-chang fbshipit-source-id: cfd28955cd03b0a71d9087085170875f6dd0be9e | 27 March 2020, 23:01:23 UTC |
8abd41a | Burton Li | 27 March 2020, 18:10:12 UTC | Fix write_unprepared_transaction_test crash on debug version. (#6574) Summary: The last key may hit index of out bound exception when id = 9. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6574 Reviewed By: riversand963 Differential Revision: D20699791 Pulled By: cheng-chang fbshipit-source-id: 8e2c5be5ff0e53e9857cfd59cea97cff21446819 | 27 March 2020, 18:12:23 UTC |
e91d1a2 | Peter Dillinger | 27 March 2020, 02:34:06 UTC | Streamline persistent_cache_test for testing efficiency (#6601) Summary: This test was written like a stress test, using up to 3x26GB RSS memory during parallel 'make check'. Now, while this code is mostly dormant, I've made the "for Travis" versions of the expensive tests the canonical versions and disabled the expensive versions. This has the side benefit of removing some arbitrary conditional compilation. For unknown reason, the super expensive tests were gated on Snappy_Supported, which appears to be irrelevant, so I removed it. The tests can be fixed / improved / migrated to stress test if/when they are deemed important again. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6601 Test Plan: make check + CI ./persistent_cache_test Before: ... [==========] 10 tests from 2 test cases ran. (114541 ms total) [ PASSED ] 10 tests. YOU HAVE 1 DISABLED TEST After: ... [==========] 3 tests from 2 test cases ran. (1714 ms total) [ PASSED ] 3 tests. YOU HAVE 10 DISABLED TESTS Reviewed By: siying Differential Revision: D20680983 Pulled By: pdillinger fbshipit-source-id: 2be0fde13eeb0a71110ac7f5477cfe63996a509e | 27 March 2020, 02:36:32 UTC |
6f62322 | Levi Tamasi | 27 March 2020, 01:48:55 UTC | Add blob files to VersionStorageInfo/VersionBuilder (#6597) Summary: The patch adds a couple of classes to represent metadata about blob files: `SharedBlobFileMetaData` contains the information elements that are immutable (once the blob file is closed), e.g. blob file number, total number and size of blob files, checksum method/value, while `BlobFileMetaData` contains attributes that can vary across versions like the amount of garbage in the file. There is a single `SharedBlobFileMetaData` for each blob file, which is jointly owned by the `BlobFileMetaData` objects that point to it; `BlobFileMetaData` objects, in turn, are owned by `Version`s and can also be shared if the (immutable _and_ mutable) state of the blob file is the same in two versions. In addition, the patch adds the blob file metadata to `VersionStorageInfo`, and extends `VersionBuilder` so that it can apply blob file related `VersionEdit`s (i.e. those containing `BlobFileAddition`s and/or `BlobFileGarbage`), and save blob file metadata to a new `VersionStorageInfo`. Consistency checks are also extended to ensure that table files point to blob files that are part of the `Version`, and that all blob files that are part of any given `Version` have at least some _non_-garbage data in them. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6597 Test Plan: `make check` Reviewed By: riversand963 Differential Revision: D20656803 Pulled By: ltamasi fbshipit-source-id: f1f74d135045b3b42d0146f03ee576ef0a4bfd80 | 27 March 2020, 01:51:53 UTC |
6301dbe | Levi Tamasi | 26 March 2020, 23:16:43 UTC | Use function objects as deleters in the block cache (#6545) Summary: As the first step of reintroducing eviction statistics for the block cache, the patch switches from using simple function pointers as deleters to function objects implementing an interface. This will enable using deleters that have state, like a smart pointer to the statistics object that is to be updated when an entry is removed from the cache. For now, the patch adds a deleter template class `SimpleDeleter`, which simply casts the `value` pointer to its original type and calls `delete` or `delete[]` on it as appropriate. Note: to prevent object lifecycle issues, deleters must outlive the cache entries referring to them; `SimpleDeleter` ensures this by using the ("leaky") Meyers singleton pattern. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6545 Test Plan: `make asan_check` Reviewed By: siying Differential Revision: D20475823 Pulled By: ltamasi fbshipit-source-id: fe354c33dd96d9bafc094605462352305449a22a | 26 March 2020, 23:19:58 UTC |
963af52 | Mike Kolupaev | 26 March 2020, 22:18:03 UTC | Fix iterator reading filter block despite read_tier == kBlockCacheTier (#6562) Summary: We're seeing iterators with `ReadOptions::read_tier == kBlockCacheTier` sometimes doing file reads. Stack trace: ``` rocksdb::RandomAccessFileReader::Read(unsigned long, unsigned long, rocksdb::Slice*, char*, bool) const rocksdb::BlockFetcher::ReadBlockContents() rocksdb::Status rocksdb::BlockBasedTable::MaybeReadBlockAndLoadToCache<rocksdb::ParsedFullFilterBlock>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::ParsedFullFilterBlock>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::BlockContents*) const rocksdb::Status rocksdb::BlockBasedTable::RetrieveBlock<rocksdb::ParsedFullFilterBlock>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::ParsedFullFilterBlock>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, bool, bool) const rocksdb::FilterBlockReaderCommon<rocksdb::ParsedFullFilterBlock>::ReadFilterBlock(rocksdb::BlockBasedTable const*, rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, bool, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::CachableEntry<rocksdb::ParsedFullFilterBlock>*) rocksdb::FilterBlockReaderCommon<rocksdb::ParsedFullFilterBlock>::GetOrReadFilterBlock(bool, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::CachableEntry<rocksdb::ParsedFullFilterBlock>*) const rocksdb::FullFilterBlockReader::MayMatch(rocksdb::Slice const&, bool, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*) const rocksdb::FullFilterBlockReader::RangeMayExist(rocksdb::Slice const*, rocksdb::Slice const&, rocksdb::SliceTransform const*, rocksdb::Comparator const*, rocksdb::Slice const*, bool*, bool, rocksdb::BlockCacheLookupContext*) rocksdb::BlockBasedTable::PrefixMayMatch(rocksdb::Slice const&, rocksdb::ReadOptions const&, rocksdb::SliceTransform const*, bool, rocksdb::BlockCacheLookupContext*) const rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::SeekImpl(rocksdb::Slice const*) rocksdb::ForwardIterator::SeekInternal(rocksdb::Slice const&, bool) rocksdb::DBIter::Seek(rocksdb::Slice const&) ``` `BlockBasedTableIterator::CheckPrefixMayMatch` was missing a check for `kBlockCacheTier`. This PR adds it. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6562 Test Plan: deployed it to a logdevice test cluster and looked at logdevice's IO tracing. Reviewed By: siying Differential Revision: D20529368 Pulled By: al13n321 fbshipit-source-id: 65bf33964b1951464415c900336635fb20919611 | 26 March 2020, 22:21:26 UTC |
e70629e | Peter Dillinger | 26 March 2020, 17:07:43 UTC | Re-update check_format_compatible.sh for default format_version=4 (#6598) Summary: Forward compatibility with new defaults only starts from 5.16 Pull Request resolved: https://github.com/facebook/rocksdb/pull/6598 Test Plan: facebook automated test (so much easier than running myself) Reviewed By: riversand963 Differential Revision: D20665553 Pulled By: pdillinger fbshipit-source-id: b846bfaccf4d0946f92d323a3b4ee6e3e548df93 | 26 March 2020, 17:11:09 UTC |
8599efa | Peter Dillinger | 25 March 2020, 20:51:27 UTC | Update check_format_compatible.sh for default format_version=4 (#6594) Summary: And add releases that should have been added before (6.6 - 6.8) Pull Request resolved: https://github.com/facebook/rocksdb/pull/6594 Test Plan: facebook automated test (so much easier than running myself) Reviewed By: riversand963 Differential Revision: D20649106 Pulled By: pdillinger fbshipit-source-id: 78832449d9295580282cebf117e3968362fbdc69 | 25 March 2020, 20:54:58 UTC |
93b80ca | Peter Dillinger | 25 March 2020, 04:19:56 UTC | Update default BBTO::format_version from 2 to 4 (#6582) Summary: Version 4 has been around long enough, for compatibility and extensive validation, that it should be default. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6582 Test Plan: CI (w.r.t. changing the default; format_version=4 is well tested and massively in production at Facebook) Reviewed By: siying Differential Revision: D20625233 Pulled By: pdillinger fbshipit-source-id: 2f83ed874cffa4a39bc7a66cdf3833b978fbb948 | 25 March 2020, 04:22:21 UTC |
ccf7676 | Yanqin Jin | 25 March 2020, 03:57:53 UTC | Update a few scripts to be python3 compatible (#6525) Summary: There are a few scripts with python3 compatibility issues that were not detected by automated tool before. Update them now. Test Plan (devserver): python2 tools/ldb_test.py python3 tools/ldb_test.py python2 tools/write_stress_runner.py --runtime_sec=30 python3 tools/write_stress_runner.py --runtime_sec=30 python2 tools/db_crashtest.py --simple --interval=2 --duration=10 blackbox python3 tools/db_crashtest.py --simple --interval=2 --duration=10 blackbox python2 tools/db_crashtest.py --simple --duration=10 --random_kill_odd=1000 --ops_per_thread=1000 whitebox python3 tools/db_crashtest.py --simple --duration=10 --random_kill_odd=1000 --ops_per_thread=1000 whitebox Pull Request resolved: https://github.com/facebook/rocksdb/pull/6525 Reviewed By: cheng-chang Differential Revision: D20627820 Pulled By: riversand963 fbshipit-source-id: 4b25a7bd4d001c7f868be8b640ef876523be6ca3 | 25 March 2020, 04:00:27 UTC |
6fd0ed4 | sdong | 25 March 2020, 03:20:33 UTC | CompactRange() to use bottom pool when goes to bottommost level (#6593) Summary: In automatic compaction, if a compaction is bottommost, it goes to bottom thread pool. We should do the same for manual compaction too. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6593 Test Plan: Add a unit test. See all existing tests pass. Reviewed By: ajkr Differential Revision: D20637408 fbshipit-source-id: cb03031e8f895085f7acf6d2d65e69e84c9ddef3 | 25 March 2020, 03:24:32 UTC |
ceeca75 | akankshamahajan | 24 March 2020, 18:24:53 UTC | Create a thread in DeleteScheduler only when rate limit enabled (#6564) Summary: Create a thread in DeleteScheduler only when delete rate limit is set because when there is no rate limit on deletion, a thread per DeleteScheduler consumes unnecessary resources. Test Plan: make -j64 check Reviewed By: riversand963 Differential Revision: D20538138 Pulled By: akankshamahajan15 fbshipit-source-id: 137499e810e817156345c30d627f8678b9adadf7 | 24 March 2020, 18:29:51 UTC |
a6ce5c8 | Huisheng Liu | 24 March 2020, 18:21:10 UTC | multiget support for timestamps (#6483) Summary: Add timestamp support for MultiGet(). timestamp from readoptions is honored, and timestamps can be returned along with values. MultiReadRandom perf test (10 minutes) on the same development machine ram drive with the same DB data shows no regression (within marge of error). The test is adapted from https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks. base line (commit 17bef7d3a): multireadrandom : 104.173 micros/op 307167 ops/sec; (5462999 of 5462999 found) This PR: multireadrandom : 104.199 micros/op 307095 ops/sec; (5307999 of 5307999 found) .\db_bench --db=r:\rocksdb.github --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --cache_size=2147483648 --cache_numshardbits=6 --compression_type=none --compression_ratio=1 --min_level_to_compress=-1 --disable_seek_compaction=1 --hard_rate_limit=2 --write_buffer_size=134217728 --max_write_buffer_number=2 --level0_file_num_compaction_trigger=8 --target_file_size_base=134217728 --max_bytes_for_level_base=1073741824 --disable_wal=0 --wal_dir=r:\rocksdb.github\WAL_LOG --sync=0 --verify_checksum=1 --statistics=0 --stats_per_interval=0 --stats_interval=1048576 --histogram=0 --use_plain_table=1 --open_files=-1 --memtablerep=prefix_hash --bloom_bits=10 --bloom_locality=1 --duration=600 --benchmarks=multireadrandom --use_existing_db=1 --num=25000000 --threads=32 --allow_concurrent_memtable_write=0 Pull Request resolved: https://github.com/facebook/rocksdb/pull/6483 Reviewed By: anand1976 Differential Revision: D20498373 Pulled By: riversand963 fbshipit-source-id: 8505f22bc40fd791bc7dd05e48d7e67c91edb627 | 24 March 2020, 18:24:09 UTC |
921cdd3 | sdong | 24 March 2020, 17:15:10 UTC | Fix bug that number of table loading threads is set as a boolean (#6576) Summary: When applying a new version in non DB open case, optimize_filters_for_hits is used for max_threads, which is clearly a bug. It is not clear what the indented value in the first place, but it value 1 makes sense here, which would create no extra threads. This bug is not expected to cause user visible problems, assuming C++ implicitly cast bool to 0 or 1. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6576 Test Plan: Run all exsiting test. Reviewed By: ajkr Differential Revision: D20602467 fbshipit-source-id: 40b2cd8619aba09ae9242b36c415464db3c9b737 | 24 March 2020, 17:17:40 UTC |
a9d168c | anand76 | 24 March 2020, 04:50:42 UTC | Simplify migration to FileSystem API (#6552) Summary: The current Env/FileSystem API separation has a couple of issues - 1. It requires the user to specify 2 options - ```Options::env``` and ```Options::file_system``` - which means they have to make code changes to benefit from the new APIs. Furthermore, there is a risk of accessing the same APIs in two different ways, through Env in the old way and through FileSystem in the new way. The two may not always match, for example, if env is ```PosixEnv``` and FileSystem is a custom implementation. Any stray RocksDB calls to env will use the ```PosixEnv``` implementation rather than the file_system implementation. 2. There needs to be a simple way for the FileSystem developer to instantiate an Env for backward compatibility purposes. This PR solves the above issues and simplifies the migration in the following ways - 1. Embed a shared_ptr to the ```FileSystem``` in the ```Env```, and remove ```Options::file_system``` as a configurable option. This way, no code changes will be required in application code to benefit from the new API. The default Env constructor uses a ```LegacyFileSystemWrapper``` as the embedded ```FileSystem```. 1a. - This also makes it more robust by ensuring that even if RocksDB has some stray calls to Env APIs rather than FileSystem, they will go through the same object and thus there is no risk of getting out of sync. 2. Provide a ```NewCompositeEnv()``` API that can be used to construct a PosixEnv with a custom FileSystem implementation. This eliminates an indirection to call Env APIs, and relieves the FileSystem developer of the burden of having to implement wrappers for the Env APIs. 3. Add a couple of missing FileSystem APIs - ```SanitizeEnvOptions()``` and ```NewLogger()``` Tests: 1. New unit tests 2. make check and make asan_check Pull Request resolved: https://github.com/facebook/rocksdb/pull/6552 Reviewed By: riversand963 Differential Revision: D20592038 Pulled By: anand1976 fbshipit-source-id: c3801ad4153f96d21d5a3ae26c92ba454d1bf1f7 | 24 March 2020, 04:54:21 UTC |
43aee93 | Cheng Chang | 24 March 2020, 03:12:38 UTC | Initialize scratch to nullptr explicitly to make clang analyzer happy (#6577) Summary: `scratch` is not initialized in `Align` because it will be set outside of it. But clang analyzer is strict on initializing it before return. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6577 Test Plan: make analyze Reviewed By: siying Differential Revision: D20607303 Pulled By: cheng-chang fbshipit-source-id: 2843d759345a057a8e122178d30b90deff0f9b2a | 24 March 2020, 03:15:27 UTC |
d300d10 | Zhichao Cao | 24 March 2020, 01:47:34 UTC | Fix the MultiGet testing failure in Circleci (#6578) Summary: The MultiGet test in db_basic_test fails in CircleCI vs2019. The reason is that even Snappy compression is enabled, the first compression type is still kNoCompression. This PR checks the list and ensure that only when compression is enable and the compression type is valid, compression will be enabled. Such that, it will not fail the combined read test in MultiGet. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6578 Test Plan: make check, db_basic_test. Reviewed By: anand1976 Differential Revision: D20607529 Pulled By: zhichao-cao fbshipit-source-id: dcead264d5c2da105912c18caad34b8510bb04b0 | 24 March 2020, 01:51:09 UTC |
617f479 | Yanqin Jin | 23 March 2020, 19:59:04 UTC | Fix LITE build (#6575) Summary: Fix LITE build by excluding some unit tests that use features not supported in LITE. ``` db/db_basic_test.cc:1778:8: error: ‘void rocksdb::{anonymous}::TableFileListener::OnTableFileCreated(const rocksdb::TableFileCreationInfo&)’ marked ‘override’, but does not override void OnTableFileCreated(const TableFileCreationInfo& info) override { ^~~~~~~~~~~~~~~~~~ make: *** [db/db_basic_test.o] Error 1 ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/6575 Reviewed By: ltamasi Differential Revision: D20598598 Pulled By: riversand963 fbshipit-source-id: 367f7cb2500360ad57030b138a94c0f731a04339 | 23 March 2020, 20:05:36 UTC |
5c6346c | Zhichao Cao | 23 March 2020, 17:23:42 UTC | Revert "Added the safe-to-ignore tag to version_edit (#6530)" (#6569) Summary: This reverts commit e10553f2a6c5d89ee17a7b44f7b9a6125ae5c735. Pass make asan_check Pull Request resolved: https://github.com/facebook/rocksdb/pull/6569 Reviewed By: riversand963 Differential Revision: D20574319 Pulled By: zhichao-cao fbshipit-source-id: ce36981a21596f5f2e14da6a59a2bb3619509a8b | 23 March 2020, 17:27:47 UTC |
fb09ef0 | Yanqin Jin | 21 March 2020, 02:17:54 UTC | Attempt to recover from db with missing table files (#6334) Summary: There are situations when RocksDB tries to recover, but the db is in an inconsistent state due to SST files referenced in the MANIFEST being missing. In this case, previous RocksDB will just fail the recovery and return a non-ok status. This PR enables another possibility. During recovery, RocksDB checks possible MANIFEST files, and try to recover to the most recent state without missing table file. `VersionSet::Recover()` applies version edits incrementally and "materializes" a version only when this version does not reference any missing table file. After processing the entire MANIFEST, the version created last will be the latest version. `DBImpl::Recover()` calls `VersionSet::Recover()`. Afterwards, WAL replay will *not* be performed. To use this capability, set `options.best_efforts_recovery = true` when opening the db. Best-efforts recovery is currently incompatible with atomic flush. Test plan (on devserver): ``` $make check $COMPILE_WITH_ASAN=1 make all && make check ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/6334 Reviewed By: anand1976 Differential Revision: D19778960 Pulled By: riversand963 fbshipit-source-id: c27ea80f29bc952e7d3311ecf5ee9c54393b40a8 | 21 March 2020, 02:30:48 UTC |
4fc2166 | Cheng Chang | 20 March 2020, 23:15:40 UTC | Support direct IO in RandomAccessFileReader::MultiRead (#6446) Summary: By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined. In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API. For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446 Test Plan: Added a new test `random_access_file_reader_test.cc`. Reviewed By: anand1976 Differential Revision: D20097518 Pulled By: cheng-chang fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1 | 20 March 2020, 23:33:26 UTC |
5fd152b | Cheng Chang | 20 March 2020, 22:19:00 UTC | Get block size only in direct IO mode (#6522) Summary: When `use_direct_reads` and `use_direct_writes` are `false`, `logical_sector_size_` inside various `*File` implementations are not actually used, so `GetLogicalBlockSize` does not necessarily need to be called for `logical_sector_size_`, just set a default page size. This is a follow up PR for https://github.com/facebook/rocksdb/pull/6457. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6522 Test Plan: make check Reviewed By: siying Differential Revision: D20408885 Pulled By: cheng-chang fbshipit-source-id: f2d3808f41265237e7fa2c0be9f084f8fa97fe3d | 20 March 2020, 22:26:10 UTC |
6c50fe1 | sdong | 20 March 2020, 21:42:47 UTC | Change HashMap::Insert()'s value to a const reference (#6567) Summary: When building RocksDB on VS2015, an error shows up with hash_map.h(39): error C2719: 'value': formal parameter with requested alignment of 8 won't be aligned Making the reference a reference can solve the problem, and there isn't a reason we can't do that, at least for the current use of the hash map. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6567 Test Plan: See CI tests pass. Reviewed By: pdillinger Differential Revision: D20548543 fbshipit-source-id: 255b55d74cf68a0b324e6f504c56608a97ea6276 | 20 March 2020, 21:59:54 UTC |
66cd07c | Peter Dillinger | 20 March 2020, 20:14:10 UTC | Exclude more Travis builds for each pull request (#6557) Summary: This commit fixes an incorrect version of this change that was previously landed. On recently adding ARM64 and PPC64LE builds to Travis, we seem to have hit some parallel build limits that dramatically increased queue times. This change majorly limits the configurations for ARM64 and PPC64LE to build on each pull request, but keeps the large matrix for branch builds. In the process, I changed some previously excluded osx build configurations to happen in branch builds. NB: we might want to move master branch Travis build to daily trigger rather than push trigger to further reduce contention. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6557 Test Plan: Travis only Reviewed By: siying Differential Revision: D20563425 Pulled By: pdillinger fbshipit-source-id: d619eb9f196486ed000364aa40de4661f0b1029d | 20 March 2020, 20:20:19 UTC |
d2e3822 | Ben Mehne | 20 March 2020, 17:51:48 UTC | Make testpilot recognize that these tests have coverage instrumentation Summary: TestPilot uses two flags to determine whether coverage is already instrumented: `fbcode_macros` and `coverage`. Normally, these two tags are added automatically to cpp tests, but this is a fake cpp test, so we must manually add them. The first is easy - `fbcode_macros` is added by the `custom_unittest` library, which is in `fbcode_macros`, so it is appropriate. The second is harder - we need to verify that we should add the macro. We do this using the `coverage.bzl` functions. Reviewed By: siying Differential Revision: D20549040 fbshipit-source-id: d2732b3ec26f3dff065efdf398abe3241075bb2f | 20 March 2020, 18:23:23 UTC |
093ff0b | Peter Dillinger | 19 March 2020, 19:49:58 UTC | Exclude more Travis builds for each pull request (#6557) Summary: On recently adding ARM64 and PPC64LE builds to Travis, we seem to have hit some parallel build limits that dramatically increased queue times. This change majorly limits the configurations for ARM64 and PPC64LE to build on each pull request, but keeps the large matrix for branch builds. In the process, I changed some previously excluded osx build configurations to happen in branch builds. NB: we might want to move master branch Travis build to daily trigger rather than push trigger to further reduce contention. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6557 Test Plan: Travis only Reviewed By: siying Differential Revision: D20524575 Pulled By: pdillinger fbshipit-source-id: babcb2c64e195679e472473a1cbdf42de47231ff | 19 March 2020, 19:53:37 UTC |
e10553f | Zhichao Cao | 19 March 2020, 18:25:50 UTC | Added the safe-to-ignore tag to version_edit (#6530) Summary: Each time RocksDB switches to a new MANIFEST file from old one, it calls WriteCurrentStateToManifest() which writes a 'snapshot' of the current in-memory state of versions to the beginning of the new manifest as a bunch of version edits. We can distinguish these version edits from other version edits written during normal operations with a custom, safe-to-ignore tag. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6530 Test Plan: added test to version_edit_test, pass make asan_check Reviewed By: riversand963 Differential Revision: D20524516 Pulled By: zhichao-cao fbshipit-source-id: f1de102f5499bfa88dae3caa2f32c7f42cf904db | 19 March 2020, 18:30:26 UTC |
4424045 | Levi Tamasi | 19 March 2020, 17:40:31 UTC | Clean up VersionBuilder a bit (#6556) Summary: The whole point of the pimpl idiom is to hide implementation details. Internal helper methods like `CheckConsistency`, `CheckConsistencyForDeletes`, and `MaybeAddFile` do not belong in the public interface of the class. In addition, the patch switches to `unique_ptr` for the implementation object instead of using a raw `delete`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6556 Test Plan: `make check` Reviewed By: riversand963 Differential Revision: D20523568 Pulled By: ltamasi fbshipit-source-id: 5bbb0ccebd0c47a33b815398c7f9cfe13bd775ac | 19 March 2020, 17:44:16 UTC |
217ce20 | Levi Tamasi | 19 March 2020, 00:11:06 UTC | Remove GetSortedWalFiles/GetCurrentWalFile from the crash test (#6491) Summary: Currently, `db_stress` tests a randomly picked one of `GetLiveFiles`, `GetSortedWalFiles`, and `GetCurrentWalFile` with a 1/N chance when the command line parameter `get_live_files_and_wal_files_one_in` is specified. The problem is that `GetSortedWalFiles` and `GetCurrentWalFile` are unreliable in the sense that they can return errors if another thread removes a WAL file while they are executing (which is a perfectly plausible and legitimate scenario). The patch splits this command line parameter into three (one for each API), and changes the crash test script so that only `GetLiveFiles` is tested during our continuous crash test runs. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6491 Test Plan: ``` make check python tools/db_crashtest.py whitebox ``` Reviewed By: siying Differential Revision: D20312200 Pulled By: ltamasi fbshipit-source-id: e7c3481eddfe3bd3d5349476e34abc9eee5b7dc8 | 19 March 2020, 00:14:15 UTC |
8ad4b32 | sdong | 18 March 2020, 17:57:46 UTC | cmake: add option WITH_CORE_TOOLS to exclude tools except ldb and sst_dump (#6506) Summary: ldb and sst_dump are most important tools and they don't dependend on gflags. In cmake, we don't have an way to only build these two tools and exclude other tools. This is inconvenient if the environment has a problem with gflags. Add such an option WITH_CORE_TOOLS. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6506 Test Plan: cmake and build with WITH_TOOLS and without. Differential Revision: D20473029 fbshipit-source-id: 3d730fd14bbae6eeeae7f9cc9aec50a4e488ad72 | 18 March 2020, 18:01:38 UTC |
1df9b01 | Levi Tamasi | 18 March 2020, 16:21:00 UTC | Disable distributed mutex test for valgrind_test (#6553) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/6553 Test Plan: ``` $ make valgrind_test -j24 $ ./folly_synchronization_distributed_mutex_test DistributedMutex is not supported in ROCKSDB_LITE, on ARM, or in valgrind_test runs ``` Reviewed By: pdillinger Differential Revision: D20501966 Pulled By: ltamasi fbshipit-source-id: 386ec5f258f89d0781a36c5b390c665787093a74 | 18 March 2020, 16:24:31 UTC |
712bc4b | sdong | 17 March 2020, 19:28:46 UTC | Fix regression bug in partitioned index reseek caused by #6531 (#6551) Summary: https://github.com/facebook/rocksdb/pull/6531 removed some code in partitioned index seek logic. By mistake the logic of storing previous index offset is removed, while the logic of using it is preserved, so that the code might use wrong value to determine reseeking condition. This will trigger a bug, if following a Seek() not going to the last block, SeekToLast() is called, and then Seek() is called which should position the cursor to the block before SeekToLast(). Pull Request resolved: https://github.com/facebook/rocksdb/pull/6551 Test Plan: Add a unit test that reproduces the bug. In the same unit test, also some reseek cases are covered to avoid regression. Reviewed By: pdillinger Differential Revision: D20493990 fbshipit-source-id: 3919aa4861c0481ec96844e053048da1a934b91d | 17 March 2020, 19:33:10 UTC |
a8149ae | akankshamahajan | 17 March 2020, 17:59:18 UTC | Allow table/sst_file_reader_test.cc to use custom Env (#6536) Summary: Allowing table/sst_file_reader_test.cc to use custom Env specified by TEST_ENV_URI. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6536 Reviewed By: riversand963 Differential Revision: D20448525 Pulled By: akankshamahajan15 fbshipit-source-id: 74e4d34c8ac4c2743741e78bf599571a4a465459 | 17 March 2020, 18:02:13 UTC |
66ed580 | Yanqin Jin | 17 March 2020, 17:47:36 UTC | Reduce runtime of db_with_timestamp_basic_test (#6546) Summary: Reduce runtime by reducing test scale to avoid test time-outs. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6546 Test Plan: time ./db_with_timestamp_basic_test and watch internal tests. Reviewed By: zhichao-cao Differential Revision: D20479292 Pulled By: riversand963 fbshipit-source-id: c9e4a155be7699dd4de60fa531de86d442a3ba0a | 17 March 2020, 17:50:48 UTC |
098dce2 | Yanqin Jin | 17 March 2020, 16:56:15 UTC | Fix compiler warning treated as error (#6547) Summary: Define a private member variable only in debug mode. Without fix, build will fail ``` In file included from table/block_based/partitioned_index_iterator.cc:9: ./table/block_based/partitioned_index_iterator.h:125:32: error: private field 'icomp_' is not used [-Werror,-Wunused-private-field] const InternalKeyComparator& icomp_; ``` Test plan (dev server) 1. make check 2. Make sure fixed in Travis Pull Request resolved: https://github.com/facebook/rocksdb/pull/6547 Reviewed By: siying Differential Revision: D20480027 Pulled By: pdillinger fbshipit-source-id: 288bc94280e240c3136335b6c73eb1ccb0db459d | 17 March 2020, 16:59:28 UTC |
6c595f0 | Peter Dillinger | 17 March 2020, 02:04:23 UTC | Update folly/lang/Align.h (backport to C++11) (#6534) Summary: For s390x support, some updates in newer version of Align.h are needed. Upgrading just that file as best we can, with one addition to Portability.h and tweaking new code in Align.h to use C++11 only (no non-trivial constexpr functions). Pull Request resolved: https://github.com/facebook/rocksdb/pull/6534 Test Plan: CI, further work in PR https://github.com/facebook/rocksdb/issues/6168 Differential Revision: D20445942 Pulled By: pdillinger fbshipit-source-id: 0cef3c367463c71f3123d12cdf287c573af5e342 | 17 March 2020, 02:07:31 UTC |
db02664 | Peter Dillinger | 16 March 2020, 23:57:03 UTC | Remove XXH3(preview) streaming APIs (#6540) Summary: There was an alignment bug in our copy of the streaming APIs for XXH3 (which we dubbed "XXH3p" for "preview" release). Since those APIs are unused and some values for XXH3 have changed since XXH3p, I'm simply removing those APIs, expecting it's better to use finalized XXH3 function if/when we decide to use those APIs (e.g. for checksums). Fixes https://github.com/facebook/rocksdb/issues/6508 Pull Request resolved: https://github.com/facebook/rocksdb/pull/6540 Test Plan: make check Differential Revision: D20479271 Pulled By: pdillinger fbshipit-source-id: 246cf1690d614d3b31042b563d249de32dec1e0d | 17 March 2020, 00:02:00 UTC |
58918d4 | Yanqin Jin | 16 March 2020, 23:54:29 UTC | Use correct Env for DestroyDB in stress test (#6539) Summary: When using custom Env, trying to call DestroyDB() with default Options will fail. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6539 Test Plan: ./db_stress Differential Revision: D20476204 Pulled By: riversand963 fbshipit-source-id: 612c6754660cc9b5bb3e9c2dbb2f6ecd7f648797 | 16 March 2020, 23:57:48 UTC |
488b1e6 | sdong | 16 March 2020, 20:47:28 UTC | Fix an error in db_bench with gcc 4.8 (#6537) Summary: I start to see following failures: tools/db_bench_tool.cc: In constructor ‘rocksdb::NormalDistribution::NormalDistribution(unsigned int, unsigned int)’: tools/db_bench_tool.cc:1528:58: error: declaration of ‘max’ shadows a member of 'this' [-Werror=shadow] NormalDistribution(unsigned int min, unsigned int max) : ^ tools/db_bench_tool.cc:1528:58: error: declaration of ‘min’ shadows a member of 'this' [-Werror=shadow] tools/db_bench_tool.cc: In constructor ‘rocksdb::UniformDistribution::UniformDistribution(unsigned int, unsigned int)’: tools/db_bench_tool.cc:1546:59: error: declaration of ‘max’ shadows a member of 'this' [-Werror=shadow] UniformDistribution(unsigned int min, unsigned int max) : ^ tools/db_bench_tool.cc:1546:59: error: declaration of ‘min’ shadows a member of 'this' [-Werror=shadow] when I build from GCC 4.8. Rename those variables to fix the problem. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6537 Test Plan: make all with the compiler that used to show the failure. Differential Revision: D20448741 fbshipit-source-id: 18bcf012dbe020f22f79038a9b08f447befa2574 | 16 March 2020, 20:50:40 UTC |
d669080 | sdong | 16 March 2020, 19:17:34 UTC | De-template block based table iterator (#6531) Summary: Right now block based table iterator is used as both of iterating data for block based table, and for the index iterator for partitioend index. This was initially convenient for introducing a new iterator and block type for new index format, while reducing code change. However, these two usage doesn't go with each other very well. For example, Prev() is never called for partitioned index iterator, and some other complexity is maintained in block based iterators, which is not needed for index iterator but maintainers will always need to reason about it. Furthermore, the template usage is not following Google C++ Style which we are following, and makes a large chunk of code tangled together. This commit separate the two iterators. Right now, here is what it is done: 1. Copy the block based iterator code into partitioned index iterator, and de-template them. 2. Remove some code not needed for partitioned index. The upper bound check and tricks are removed. We never tested performance for those tricks when partitioned index is enabled in the first place. It's unlikelyl to generate performance regression, as creating new partitioned index block is much rarer than data blocks. 3. Separate out the prefetch logic to a helper class and both classes call them. This commit will enable future follow-ups. One direction is that we might separate index iterator interface for data blocks and index blocks, as they are quite different. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6531 Test Plan: build using make and cmake. And build release Differential Revision: D20473108 fbshipit-source-id: e48011783b339a4257c204cc07507b171b834b0f | 16 March 2020, 19:20:50 UTC |
402da45 | Cheng Chang | 14 March 2020, 04:51:26 UTC | Migrate AppVeyor to CircleCI (#6518) Summary: CircleCI is the new recommended CI system internally. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6518 Test Plan: Watch https://app.circleci.com/pipelines/github/facebook/rocksdb Differential Revision: D20454743 Pulled By: cheng-chang fbshipit-source-id: 39031568d6c1d3d25b7fbd78fa9a0e6067ddc47c | 14 March 2020, 04:58:51 UTC |
23eae14 | Cheng Chang | 14 March 2020, 04:51:16 UTC | Destroy DB at the end of each test in db_logical_block_size_cache_test (#6532) Summary: If DB is not deleted, in concurrent test, the tests might fail because of the previously existing DB. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6532 Test Plan: make clean && make -j24 LITE=1 db_logical_block_size_cache_test && ./db_logical_block_size_cache_test make clean && make -j24 db_logical_block_size_cache_test && ./db_logical_block_size_cache_test Differential Revision: D20454734 Pulled By: cheng-chang fbshipit-source-id: 8abede2ec1d79c1a4fe1bc95fbda489f8f7ee052 | 14 March 2020, 04:53:38 UTC |
a824727 | Zhichao Cao | 13 March 2020, 23:45:28 UTC | Fix build bug caused by PR 6516 (#6535) Summary: Fix the build corruption caused by PR https://github.com/facebook/rocksdb/issues/6516 Testing plan: make make asan_check Pull Request resolved: https://github.com/facebook/rocksdb/pull/6535 Differential Revision: D20448614 Pulled By: zhichao-cao fbshipit-source-id: 4d2a3dae6cdd781fcfe8e28a84ac3f536db1b067 | 13 March 2020, 23:48:03 UTC |
85dbdf2 | Peter Dillinger | 13 March 2020, 20:36:45 UTC | Use an Amazon S3 bucket for downloading deps (#6526) Summary: After we had a lot of failures with maven.org downloads, we wanted an alternative location for downloading binary dependencies. Hosting them through github would have been good in terms of organizational and network dependencies, but that approach seems to be awkward (fake releases, so would need a 'rocksdb-deps' repo) and strangely complicated for Facebook policy on open source repositories. This commit moves the downloads (that are not officially hosted by others on github) from my personal rocksdb fork to an S3 bucket owned by the Facebook RocksDB AWS account. Facebook employees can access this through an internal tool, and we should be able to grant permission to outside collaborators. Assuming this works out, I will back-port to older branches to stabilize their CI testing as well. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6526 Test Plan: CI Differential Revision: D20430130 Pulled By: pdillinger fbshipit-source-id: df52394a65e0a57942db3039bdaade8a4d520cb2 | 13 March 2020, 20:39:03 UTC |
5c30e6c | Zhichao Cao | 13 March 2020, 18:33:04 UTC | Separate timestamp related test from db_basic_test (#6516) Summary: In some of the test, db_basic_test may cause time out due to its long running time. Separate the timestamp related test from db_basic_test to avoid the potential issue. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6516 Test Plan: pass make asan_check Differential Revision: D20423922 Pulled By: zhichao-cao fbshipit-source-id: d6306f89a8de55b07bf57233e4554c09ef1fe23a | 13 March 2020, 18:37:15 UTC |
674cf41 | sdong | 13 March 2020, 04:39:36 UTC | Divide block_based_table_reader.cc (#6527) Summary: block_based_table_reader.cc is a giant file, which makes it hard for users to navigate the code. Divide the files to multiple files. Some class templates cannot be moved to .cc file. They are moved to .h files. It is still better than including them all in block_based_table_reader.cc. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6527 Test Plan: "make all check" and "make release". Also build using cmake. Differential Revision: D20428455 fbshipit-source-id: ca713c698469f07f35bc0c271358c0874ed4eb28 | 13 March 2020, 04:41:50 UTC |
dd7a4a8 | Yuqi Gu | 13 March 2020, 03:58:26 UTC | CI: add Arm support to travis CI matrix (#6436) Summary: This patch based on https://github.com/facebook/rocksdb/issues/5932 offers a better solution to add arm64 to TravisCI matrix. Really thank adamretter for initiating Arm CI setup. Difference comparing to amd64: 1. For CMake, as no official arm64 release ready on Kitware page, a third party (conda-forge) released one is used instead of building from source. The main reason is to save CI time. 2. Explicit export JAVA_HOME on arm64 3. Disable mingw test Signed-off-by: Yuqi Gu <yuqi.gu@arm.com> Pull Request resolved: https://github.com/facebook/rocksdb/pull/6436 Differential Revision: D20428505 Pulled By: pdillinger fbshipit-source-id: 81ef02435e41480bb71710b783d85ebf452ce926 | 13 March 2020, 04:01:20 UTC |
a8851f2 | Ben Mehne | 13 March 2020, 00:45:04 UTC | Fix coverage for internal_repo_rocksdb Summary: tcc gtest runner need to know the location of the binary in order to collect coverage. We can give them the location in an environment variable. Note that all these tests will break in tpx currently, though this is a bug in rocksdb's wrapper script, not tpx. Reviewed By: siying Differential Revision: D20430043 fbshipit-source-id: c77d5f70bbc28f6011c6f91906bce2ceecc2f167 | 13 March 2020, 00:48:16 UTC |
2ccb794 | Cheng Chang | 12 March 2020, 21:27:51 UTC | Use DestroyColumnFamilyHandle instead of directly deleting column family handle (#6505) Summary: Update example usage of closing column family. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6505 Test Plan: cd examples && make column_families_example && ./column_families_example Differential Revision: D20362100 Pulled By: cheng-chang fbshipit-source-id: 493c5e0068a40b4f237f8f8511cddd22dc15ea5c | 12 March 2020, 21:30:46 UTC |
0d2c8e4 | Cheng Chang | 12 March 2020, 21:10:47 UTC | OpenForReadOnly is not supported in LITE mode (#6523) Summary: In DBLogicalBlockSizeCacheTest, do not test OpenForReadOnly in LITE mode. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6523 Test Plan: watch test for LITE mode Differential Revision: D20420321 Pulled By: cheng-chang fbshipit-source-id: e45bf6f2800206d6f8ce9af7308e76a08de80643 | 12 March 2020, 21:13:59 UTC |
0772768 | Adam Retter | 12 March 2020, 19:22:03 UTC | Force Java version on Travis CI (#6512) Summary: In the `.travis.yml` file the `jdk: openjdk7` element is ignored when `language: cpp`. So whatever version of the JDK that was installed in the Travis container was used - typically JDK 11. To ensure our RocksJava builds are working, we now instead install and use OpenJDK 8. Ideally we would use OpenJDK 7, as RocksJava supports Java 7, but many of the newer Travis containers don't support Java 7, so Java 8 is the next best thing. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6512 Differential Revision: D20388296 Pulled By: pdillinger fbshipit-source-id: 8bbe6b59b70cfab7fe81ff63867d907fefdd2df1 | 12 March 2020, 19:24:51 UTC |
c15e85b | Levi Tamasi | 12 March 2020, 17:58:27 UTC | Move BlobDB related files under db/ to db/blob/ (#6519) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/6519 Test Plan: ``` make all make check ``` Differential Revision: D20400691 Pulled By: ltamasi fbshipit-source-id: 20ef911cf1c2c92c7f71ef0b493f9be64f2eef94 | 12 March 2020, 18:00:56 UTC |
07a3f7f | Huisheng Liu | 12 March 2020, 15:39:46 UTC | fix MSVC build failures (#6517) Summary: fix a few build warnings that are treated as failures with more strict MSVC warning settings Pull Request resolved: https://github.com/facebook/rocksdb/pull/6517 Differential Revision: D20401325 Pulled By: pdillinger fbshipit-source-id: b44979dfaafdc7b3b8cb44a565400a99b331dd30 | 12 March 2020, 15:42:39 UTC |
6dea753 | Cheng Chang | 12 March 2020, 04:35:41 UTC | Remove copy of pairs from the for range loop (#6514) Summary: Remove copy of pairs from the for range loop Pull Request resolved: https://github.com/facebook/rocksdb/pull/6514 Test Plan: make check Differential Revision: D20389688 Pulled By: cheng-chang fbshipit-source-id: 1c772091f955be33267514010f3596c61a6f46b5 | 12 March 2020, 04:38:09 UTC |
2d9efc9 | Cheng Chang | 12 March 2020, 01:36:43 UTC | Cache result of GetLogicalBufferSize in Linux (#6457) Summary: In Linux, when reopening DB with many SST files, profiling shows that 100% system cpu time spent for a couple of seconds for `GetLogicalBufferSize`. This slows down MyRocks' recovery time when site is down. This PR introduces two new APIs: 1. `Env::RegisterDbPaths` and `Env::UnregisterDbPaths` lets `DB` tell the env when it starts or stops using its database directories . The `PosixFileSystem` takes this opportunity to set up a cache from database directories to the corresponding logical block sizes. 2. `LogicalBlockSizeCache` is defined only for OS_LINUX to cache the logical block sizes. Other modifications: 1. rename `logical buffer size` to `logical block size` to be consistent with Linux terms. 2. declare `GetLogicalBlockSize` in `PosixHelper` to expose it to `PosixFileSystem`. 3. change the functions `IOError` and `IOStatus` in `env/io_posix.h` to have external linkage since they are used in other translation units too. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6457 Test Plan: 1. A new unit test is added for `LogicalBlockSizeCache` in `env/io_posix_test.cc`. 2. A new integration test is added for `DB` operations related to the cache in `db/db_logical_block_size_cache_test.cc`. `make check` Differential Revision: D20131243 Pulled By: cheng-chang fbshipit-source-id: 3077c50f8065c0bffb544d8f49fb10bba9408d04 | 12 March 2020, 01:40:05 UTC |
331e619 | sdong | 11 March 2020, 23:20:13 UTC | Include more information in file lock failure (#6507) Summary: When users fail to open a DB with file lock failure, it is sometimes hard for users to debug. We now include the time the lock is acquired and the thread ID that acquired the lock, to help users debug problems like this. Default Env's thread ID is used. Since type of lockedFiles is changed, rename it to follow naming convention too. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6507 Test Plan: Add a unit test and improve an existing test to validate the case. Differential Revision: D20378333 fbshipit-source-id: 312fe0e9733fd1d1e9969c321b90ce523cf4708a | 11 March 2020, 23:23:08 UTC |
37a635c | Levi Tamasi | 11 March 2020, 21:41:34 UTC | Disambiguate CustomFieldTags for the unity build (#6513) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/6513 Test Plan: `make unity_test` Differential Revision: D20388919 Pulled By: ltamasi fbshipit-source-id: 88dbceab0723a54ee3939e1644e13dc9a4c70420 | 11 March 2020, 21:45:12 UTC |
8fc20ac | Adam Retter | 11 March 2020, 19:31:06 UTC | Add ppc64le builds to Travis (#6144) Summary: Let's see how this goes... Pull Request resolved: https://github.com/facebook/rocksdb/pull/6144 Differential Revision: D20387515 Pulled By: pdillinger fbshipit-source-id: ba2669348c267141dfddff910b4c2224a22cbb38 | 11 March 2020, 19:33:45 UTC |