swh:1:snp:5115096b921df712aeb2a08114fede57fb3331fb
- HEAD
- refs/heads/2.2.fb.branch
- refs/heads/2.3.fb.branch
- refs/heads/2.4.fb.branch
- refs/heads/2.5.fb.branch
- refs/heads/2.6.fb.branch
- refs/heads/2.7
- refs/heads/2.7.fb.branch
- refs/heads/2.8.1.fb
- refs/heads/2.8.fb
- refs/heads/2.8.fb.trunk
- refs/heads/3.0.fb
- refs/heads/3.0.fb.branch
- refs/heads/3.1.fb
- refs/heads/3.10.fb
- refs/heads/3.11.fb
- refs/heads/3.12.fb
- refs/heads/3.13.fb
- refs/heads/3.2.fb
- refs/heads/3.3.fb
- refs/heads/3.4.fb
- refs/heads/3.5.fb
- refs/heads/3.6.fb
- refs/heads/3.7.fb
- refs/heads/3.8.fb
- refs/heads/3.9.fb
- refs/heads/4.0.fb
- refs/heads/4.1.fb
- refs/heads/4.10.fb
- refs/heads/4.11.fb
- refs/heads/4.12.fb
- refs/heads/4.13.fb
- refs/heads/4.2.fb
- refs/heads/4.3.fb
- refs/heads/4.4.fb
- refs/heads/4.5.fb
- refs/heads/4.6.fb
- refs/heads/4.7.fb
- refs/heads/4.8.fb
- refs/heads/4.9.fb
- refs/heads/5.0.fb
- refs/heads/5.1.fb
- refs/heads/5.10.fb
- refs/heads/5.11.fb
- refs/heads/5.12.fb
- refs/heads/5.13.fb
- refs/heads/5.13.fb.myrocks
- refs/heads/5.14.fb
- refs/heads/5.14.fb.myrocks
- refs/heads/5.15.fb
- refs/heads/5.16.fb
- refs/heads/5.17.fb
- refs/heads/5.17.fb.myrocks
- refs/heads/5.18.fb
- refs/heads/5.2.fb
- refs/heads/5.3.fb
- refs/heads/5.4.fb
- refs/heads/5.5.fb
- refs/heads/5.6.fb
- refs/heads/5.7.fb
- refs/heads/5.7.fb.myrocks
- refs/heads/5.8.3
- refs/heads/5.8.fb
- refs/heads/5.9.fb
- refs/heads/5.9.fb.myrocks
- refs/heads/6.0.fb
- refs/heads/6.0.fb.myrocks
- refs/heads/6.1.fb
- refs/heads/6.1.fb.myrocks
- refs/heads/6.1.fb.prod201905
- refs/heads/6.10.fb
- refs/heads/6.11.fb
- refs/heads/6.12.fb
- refs/heads/6.13.fb
- refs/heads/6.13.fb.laser
- refs/heads/6.14.fb
- refs/heads/6.14.fb.laser
- refs/heads/6.15.fb
- refs/heads/6.16.fb
- refs/heads/6.17.fb
- refs/heads/6.2.fb
- refs/heads/6.3.fb
- refs/heads/6.3.fb.myrocks
- refs/heads/6.3.fb.myrocks2
- refs/heads/6.3fb
- refs/heads/6.4.fb
- refs/heads/6.5.fb
- refs/heads/6.6.fb
- refs/heads/6.7.fb
- refs/heads/6.8.fb
- refs/heads/6.9.fb
- refs/heads/adaptive
- refs/heads/ajkr-patch-1
- refs/heads/blob_shadow
- refs/heads/bottom-pri-level
- refs/heads/bugfix-build-detect
- refs/heads/checksum_readahead_mmap_fix
- refs/heads/fb-config
- refs/heads/feature/debug-rocksdbjavastatic
- refs/heads/feature/travis-arm64
- refs/heads/fix-release-notes
- refs/heads/fix-write-batch-comment
- refs/heads/format_compatible_4
- refs/heads/getmergeops
- refs/heads/gh-pages-old
- refs/heads/history-update
- refs/heads/hotfix/lambda-capture
- refs/heads/improve-support
- refs/heads/jay-stress
- refs/heads/katherinez-patch-1
- refs/heads/katherinez-patch-2
- refs/heads/master
- refs/heads/pr-sanity-check-as-GHAction
- refs/heads/pr/6062
- refs/heads/ramvadiv-patch-1
- refs/heads/scaffold
- refs/heads/siying-patch-1
- refs/heads/siying-patch-10
- refs/heads/siying-patch-2
- refs/heads/siying-patch-3
- refs/heads/siying-patch-4
- refs/heads/skip_memtable_flush
- refs/heads/testing_ppc_build
- refs/heads/tests
- refs/heads/unused-var
- refs/heads/v6.6.4
- refs/heads/yiwu_stackable
- refs/heads/yuslepukhin
- refs/remotes/origin/5.13.fb
- refs/tags/2.5.fb
- refs/tags/2.6.fb
- refs/tags/3.0.fb
- refs/tags/do-not-use-me2
- refs/tags/rocksdb-3.1
- refs/tags/rocksdb-3.10.2
- refs/tags/rocksdb-3.11
- refs/tags/rocksdb-3.11.1
- refs/tags/rocksdb-3.11.2
- refs/tags/rocksdb-3.2
- refs/tags/rocksdb-3.3
- refs/tags/rocksdb-3.4
- refs/tags/rocksdb-3.5
- refs/tags/rocksdb-3.5.1
- refs/tags/rocksdb-3.6.1
- refs/tags/rocksdb-3.6.2
- refs/tags/rocksdb-3.7
- refs/tags/rocksdb-3.8
- refs/tags/rocksdb-3.9
- refs/tags/rocksdb-3.9.1
- refs/tags/rocksdb-4.1
- refs/tags/rocksdb-5.10.2
- refs/tags/rocksdb-5.10.3
- refs/tags/rocksdb-5.10.4
- refs/tags/rocksdb-5.11.2
- refs/tags/rocksdb-5.11.3
- refs/tags/rocksdb-5.14.3
- refs/tags/rocksdb-5.2.1
- refs/tags/rocksdb-5.3.3
- refs/tags/rocksdb-5.3.4
- refs/tags/rocksdb-5.3.5
- refs/tags/rocksdb-5.3.6
- refs/tags/rocksdb-5.4.10
- refs/tags/rocksdb-5.4.5
- refs/tags/rocksdb-5.4.6
- refs/tags/rocksdb-5.5.2
- refs/tags/rocksdb-5.5.3
- refs/tags/rocksdb-5.5.4
- refs/tags/rocksdb-5.5.5
- refs/tags/rocksdb-5.5.6
- refs/tags/rocksdb-5.6.1
- refs/tags/rocksdb-5.6.2
- refs/tags/rocksdb-5.7.1
- refs/tags/rocksdb-5.7.2
- refs/tags/rocksdb-5.7.3
- refs/tags/rocksdb-5.7.5
- refs/tags/rocksdb-5.8.6
- refs/tags/rocksdb-5.8.7
- refs/tags/rocksdb-5.8.8
- refs/tags/rocksdb-5.9.2
- refs/tags/v4.0
- refs/tags/v4.1
- refs/tags/v5.10.2
- refs/tags/v5.10.3
- refs/tags/v5.10.4
- refs/tags/v5.11.2
- refs/tags/v5.11.3
- refs/tags/v5.13.3
- refs/tags/v5.14.3
- refs/tags/v5.15.10
- refs/tags/v5.18.3
- refs/tags/v5.2.1
- refs/tags/v5.3.3
- refs/tags/v5.3.4
- refs/tags/v5.3.5
- refs/tags/v5.3.6
- refs/tags/v5.4.10
- refs/tags/v5.4.5
- refs/tags/v5.4.6
- refs/tags/v5.5.2
- refs/tags/v5.5.3
- refs/tags/v5.5.4
- refs/tags/v5.5.5
- refs/tags/v5.5.6
- refs/tags/v5.6.1
- refs/tags/v5.6.2
- refs/tags/v5.7.1
- refs/tags/v5.7.2
- refs/tags/v5.7.3
- refs/tags/v5.7.5
- refs/tags/v5.8.6
- refs/tags/v5.8.7
- refs/tags/v5.8.8
- refs/tags/v5.9.2
- refs/tags/v6.0.1
- refs/tags/v6.0.2
- refs/tags/v6.1.1
- refs/tags/v6.1.2
- refs/tags/v6.10.1
- refs/tags/v6.10.2
- refs/tags/v6.11.4
- refs/tags/v6.11.6
- refs/tags/v6.12.6
- refs/tags/v6.12.7
- refs/tags/v6.13.2
- refs/tags/v6.13.3
- refs/tags/v6.14.5
- refs/tags/v6.14.6
- refs/tags/v6.15.4
- refs/tags/v6.2.2
- refs/tags/v6.2.4
- refs/tags/v6.3.6
- refs/tags/v6.4.6
- refs/tags/v6.5.2
- refs/tags/v6.5.3
- refs/tags/v6.6.3
- refs/tags/v6.6.4
- refs/tags/v6.7.3
- refs/tags/v6.8.1
- d4ad32d7bd908d0b065c04ed697c5c157cd4512d
- v6.15.2
- v5.8
- v5.5.1
- v5.4.7
- v5.18.4
- v5.17.2
- v5.16.6
- v5.14.2
- v5.13.4
- v5.13.2
- v5.13.1
- v5.12.5
- v5.12.4
- v5.12.3
- v5.12.2
- v5.1.4
- v5.1.3
- v5.1.2
- v5.0.2
- v5.0.1
- v4.9
- v4.8
- v4.6.1
- v4.5.1
- v4.4.1
- v4.4
- v4.3.1
- v4.3
- v4.2
- v4.13.5
- v4.13
- v4.11.2
- v3.9
- v3.8
- v3.7
- v3.6.1
- v3.5
- v3.4
- v3.3
- v3.2
- v3.13.1
- v3.13
- v3.12.1
- v3.12
- v3.11
- v3.10
- v3.1
- v3.0
- v2.8
- v2.7
- v2.6
- v2.5
- v2.4
- v2.3
- v2.2
- v2.1
- v2.0
- v1.5.9.1
- v1.5.8.2
- v1.5.8.1
- v1.5.8
- v1.5.7
- rocksdb-5.8
- rocksdb-5.4.7
- rocksdb-5.1.4
- rocksdb-5.1.3
- rocksdb-5.1.2
- rocksdb-5.0.2
- rocksdb-5.0.1
- rocksdb-4.9
- rocksdb-4.8
- rocksdb-4.6.1
- rocksdb-4.5.1
- rocksdb-4.4.1
- rocksdb-4.4
- rocksdb-4.3.1
- rocksdb-4.3
- rocksdb-4.2
- rocksdb-4.13.5
- rocksdb-4.13
- rocksdb-4.11.2
- rocksdb-3.10.1
- 2.8.fb
- 2.7.fb
- 2.4.fb
- 2.3.fb
- 2.2.fb
- 2.1.fb
- 2.0.fb
- 1.5.9.fb
- 1.5.9.2.fb
- 1.5.9.1.fb
- 1.5.8.fb
- 1.5.8.2.fb
- 1.5.8.1.fb
- 1.5.7.fb
Permalinks
To reference or cite the objects present in the Software Heritage archive, permalinks based on SoftWare Hash IDentifiers (SWHIDs) must be used.
Select below a type of object currently browsed in order to display its associated SWHID and permalink.
Revision | Author | Date | Message | Commit Date |
---|---|---|---|---|
d4ad32d | Maysam Yabandeh | 13 July 2018, 00:19:57 UTC | Refactor BlockIter (#4121) Summary: BlockIter is getting crowded including details that specific only to either index or data blocks. The patch moves down such details to DataBlockIter and IndexBlockIter, both inheriting from BlockIter. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4121 Differential Revision: D8816832 Pulled By: maysamyabandeh fbshipit-source-id: d492e74155c11d8a0c1c85cd7ee33d24c7456197 | 13 July 2018, 00:27:31 UTC |
6390443 | Andrew Kryczka | 12 July 2018, 22:55:07 UTC | db_bench periodically dump stats to info log (#4109) Summary: give control of how often stats are printed, including jemalloc stats if enabled. Previously the default was 10 minutes so we'd only see updated stats for very long benchmark runs. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4109 Differential Revision: D8796444 Pulled By: ajkr fbshipit-source-id: fd7902fe3f105fae89322c4ab63316bba4a2b15e | 12 July 2018, 22:57:42 UTC |
dbeaa0d | Yanqin Jin | 12 July 2018, 21:39:07 UTC | Reduce #iterations to shorten execution time. (#4123) Summary: Reduce #iterations from 5000 to 1000 so that `ExternalSSTFileTest.CompactDuringAddFileRandom` can finish faster. On the one hand, 5000 iterations does not seem to improve the quality of unit test in comparison with 1000. On the other hand, long running tests should belong to stress tests. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4123 Differential Revision: D8822514 Pulled By: riversand963 fbshipit-source-id: 0f439b8d5ccd9a4aed84638f8bac16382de17245 | 12 July 2018, 21:42:39 UTC |
5f3088d | Nikhil Benesch | 12 July 2018, 21:28:10 UTC | Range deletion performance improvements + cleanup (#4014) Summary: This fixes the same performance issue that #3992 fixes but with much more invasive cleanup. I'm more excited about this PR because it paves the way for fixing another problem we uncovered at Cockroach where range deletion tombstones can cause massive compactions. For example, suppose L4 contains deletions from [a, c) and [x, z) and no other keys, and L5 is entirely empty. L6, however, is full of data. When compacting L4 -> L5, we'll end up with one file that spans, massively, from [a, z). When we go to compact L5 -> L6, we'll have to rewrite all of L6! If, instead of range deletions in L4, we had keys a, b, x, y, and z, RocksDB would have been smart enough to create two files in L5: one for a and b and another for x, y, and z. With the changes in this PR, it will be possible to adjust the compaction logic to split tombstones/start new output files when they would span too many files in the grandparent level. ajkr please take a look when you have a minute! Pull Request resolved: https://github.com/facebook/rocksdb/pull/4014 Differential Revision: D8773253 Pulled By: ajkr fbshipit-source-id: ec62fa85f648fdebe1380b83ed997f9baec35677 | 12 July 2018, 21:42:39 UTC |
121e321 | Fosco Marotto | 12 July 2018, 18:44:46 UTC | Update docs/Gemfile.lock for nokogiri cve (#4116) Summary: Per GitHub warning Pull Request resolved: https://github.com/facebook/rocksdb/pull/4116 Differential Revision: D8812291 Pulled By: gfosco fbshipit-source-id: 3c55adc4ac737e4be077ddf29322c8961018d67c | 12 July 2018, 18:57:40 UTC |
a61ff87 | Siying Dong | 12 July 2018, 18:27:28 UTC | Remove two CI tests (#4110) Summary: Two CI tests never pass because of the environment problem. Delete them. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4110 Differential Revision: D8805713 Pulled By: siying fbshipit-source-id: 6eb4813dc2094ee2045ec8ede7fe8967d546d6e8 | 12 July 2018, 18:43:25 UTC |
1ea83c5 | Anand Ananthabhotla | 12 July 2018, 06:36:36 UTC | Reduce runtime of compact_on_deletion_collector_test (#4117) Summary: This test routinely exceeds the FB contbuild test timeout of 10 minutes, due to the large number of iterations. The large number (mainly due to 100 randomly selected window sizes) does not seem to add any value. Reduce it to allow the test to finish in < 10 mins. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4117 Differential Revision: D8815646 Pulled By: anand1976 fbshipit-source-id: 260690d24f444767ad93b039dec3ae8b9cdd1843 | 12 July 2018, 06:41:58 UTC |
35b38a2 | Siying Dong | 12 July 2018, 00:31:55 UTC | Update comments of WriteBatchWithIndex Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/4113 Differential Revision: D8814172 Pulled By: siying fbshipit-source-id: cabc31db2c74803af9b2f99329155a1086eb1b22 | 12 July 2018, 00:42:50 UTC |
5cd8240 | Nikhil Benesch | 11 July 2018, 22:45:21 UTC | Test range deletions with more configurations (#4021) Summary: Run the basic range deletion tests against the standard set of configurations. This testing exposed that files with hash indexes and partitioned indexes were not handling the case where the file contained only range deletions--i.e., where the index was empty. Additionally file a TODO about the fact that range deletions are broken when allow_mmap_reads = true is set. /cc ajkr nvanbenschoten Best viewed with ?w=1: https://github.com/facebook/rocksdb/pull/4021/files?w=1 Pull Request resolved: https://github.com/facebook/rocksdb/pull/4021 Differential Revision: D8811860 Pulled By: ajkr fbshipit-source-id: 3cc07e6d6210a2a00b932866481b3d5c59775343 | 11 July 2018, 22:57:49 UTC |
cfee7fb | Nicolas Pépin-Perreault | 11 July 2018, 22:44:52 UTC | Allow storing metadata with backups for Java API (#4111) Summary: Exposes BackupEngine::CreateNewBackupWithMetadata and BackupInfo metadata to the Java API. Full disclaimer, I'm not familiar with JNI stuff, so I might have forgotten something (hopefully no memory leaks!). I also tried to find contributing guidelines but didn't see any, but I hope the PR style is consistent with the rest of the code base. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4111 Differential Revision: D8811180 Pulled By: ajkr fbshipit-source-id: e38b3e396c7574328c2a1a0e55acc8d092b6a569 | 11 July 2018, 22:57:49 UTC |
1c91219 | Sagar Vemuri | 11 July 2018, 22:42:49 UTC | Remove external tracking of AlignedBuffer's size (#4105) Summary: Remove external tracking of AlignedBuffer's size in `ReadaheadRandomAccessFile` and `FilePrefetchBuffer`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4105 Differential Revision: D8805724 Pulled By: sagar0 fbshipit-source-id: d61d8c203c7c500e3f36e912132d7852026ed023 | 11 July 2018, 22:57:49 UTC |
331cb63 | Yanqin Jin | 11 July 2018, 21:49:31 UTC | SetOptions Backup Race Condition (#4108) Summary: Prior to this PR, there was a race condition between `DBImpl::SetOptions` and `BackupEngine::CreateNewBackup`, as illustrated below. ``` Time thread 1 thread 2 | CreateNewBackup -> GetLiveFiles | SetOptions -> RenameTempFileToOptionsFile | SetOptions -> RenameTempFileToOptionsFile | SetOptions -> RenameTempFileToOptionsFile // unlink oldest OPTIONS file | copy the oldest OPTIONS // IO error! V ``` Proposed fix is to check the value of `DBImpl::disable_obsolete_files_deletion_` before calling `DeleteObsoleteOptionsFiles`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4108 Differential Revision: D8796360 Pulled By: riversand963 fbshipit-source-id: 02045317f793ea4c7d4400a5bf333b8502fa3e82 | 11 July 2018, 21:57:46 UTC |
440621a | Sagar Vemuri | 11 July 2018, 19:24:07 UTC | Fix Copying of data between buffers in FilePrefetchBuffer (#4100) Summary: Copy data between buffers inside FilePrefetchBuffer only when chunk length is greater than 0. Otherwise AlignedBuffer was accessing memory out of its range causing crashes. Removing the tracking of buffer length outside of `AlignedBuffer`, i.e. in `FilePrefetchBuffer` and `ReadaheadRandomAccessFile`, will follow in a separate PR, as it is not the root cause of the crash reported in #4051. (`FilePrefetchBuffer` itself has been this way from its inception, and `ReadaheadRandomAccessFile` was updated to add the buffer length at some point). Comprehensive tests for `FilePrefetchBuffer` also to follow in a separate PR. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4100 Differential Revision: D8792590 Pulled By: sagar0 fbshipit-source-id: 3578f45761cf6884243e767f749db4016ccc93e1 | 11 July 2018, 19:28:13 UTC |
926f3a7 | Siying Dong | 09 July 2018, 22:17:38 UTC | In delete scheduler, before ftruncate file for slow delete, check whether there is other hard links (#4093) Summary: Right now slow deletion with ftruncate doesn't work well with checkpoints because it ruin hard linked files in checkpoints. To fix it, check the file has no other hard link before ftruncate it. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4093 Differential Revision: D8730360 Pulled By: siying fbshipit-source-id: 756eea5bce8a87b9a2ea3a5bfa190b2cab6f75df | 09 July 2018, 22:28:12 UTC |
b984637 | Manuel Ung | 07 July 2018, 00:17:36 UTC | WriteUnPrepared: Add support for recovering WriteUnprepared transactions (#4078) Summary: This adds support for recovering WriteUnprepared transactions through the following changes: - The information in `RecoveredTransaction` is extended so that it can reference multiple batches. - `MarkBeginPrepare` is extended with a bool indicating whether it is an unprepared begin, and this is passed down to `InsertRecoveredTransaction` to indicate whether the current transaction is prepared or not. - `WriteUnpreparedTxnDB::Initialize` is overridden so that it will rollback unprepared transactions from the recovered transactions. This can be done without updating the prepare heap/commit map, because this is before the DB has finished initializing, and after writing the rollback batch, those data structures should not contain information about the rolled back transaction anyway. Commit/Rollback of live transactions is still unimplemented and will come later. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4078 Differential Revision: D8703382 Pulled By: lth fbshipit-source-id: 7e0aada6c23bd39299f1f20d6c060492e0e6b60a | 07 July 2018, 00:59:13 UTC |
db7ae0a | Yanqin Jin | 06 July 2018, 23:07:29 UTC | Fix a map lookup that may throw exception. (#4098) Summary: `std::map::at(key)` throws std::out_of_range if key does not exist. Current code does not handle this. Although this case is unlikely, I feel it's safe to use `std::map::find`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4098 Differential Revision: D8753865 Pulled By: riversand963 fbshipit-source-id: 9a9ba43badb0fb5e0d24cd87903931fd12f3f8ec | 06 July 2018, 23:12:49 UTC |
d4d9fe8 | Yanqin Jin | 06 July 2018, 20:09:57 UTC | Fix a bug caused by not copying the block trailer. (#4096) Summary: This was caught by crash test, and the following is a simple way to reproduce it and verify the fix. One way to trigger this code path is to use the following configuration: - Compress SST file - Enable direct IO and prefetch buffer - Do NOT use compressed block cache Closes https://github.com/facebook/rocksdb/pull/4096 Differential Revision: D8742009 Pulled By: riversand963 fbshipit-source-id: f13381078bbb0dce92f60bd313a78ab602bcacd2 | 06 July 2018, 20:12:39 UTC |
35b8332 | Huachao Huang | 06 July 2018, 03:10:37 UTC | compaction: fix max_subcompactions option for CompactRange (#4082) Summary: The max_subcompactions option was introduced in https://github.com/facebook/rocksdb/pull/3775. Closes https://github.com/facebook/rocksdb/pull/4082 Differential Revision: D8743258 Pulled By: ajkr fbshipit-source-id: d60ee75769dfc19ab6f8754e4ff3a267848f1ed9 | 06 July 2018, 03:12:56 UTC |
39218a7 | Yanqin Jin | 05 July 2018, 18:32:42 UTC | Increase the size of LRU cache. (#4090) Summary: Increase the size of each shard so that the number of cache hit/miss match expectation. Otherwise FilterBlockInBlockCache test will fail. Closes https://github.com/facebook/rocksdb/pull/4090 Differential Revision: D8736158 Pulled By: riversand963 fbshipit-source-id: 5cdbc06b02390389fd5b72a6d251d88949ad3d91 | 05 July 2018, 18:45:11 UTC |
17027ae | Siying Dong | 04 July 2018, 00:54:41 UTC | Change default value of `bytes_max_delete_chunk` to 0 in NewSstFileManager() (#4092) Summary: Now by default, with NewSstFileManager, checkpoints may be corrupted. Disable this feature to avoid this issue. Closes https://github.com/facebook/rocksdb/pull/4092 Differential Revision: D8729856 Pulled By: siying fbshipit-source-id: 914c321d6eaf52d8c5981171322d85dd29088307 | 04 July 2018, 00:57:37 UTC |
0d234df | Adam Retter | 02 July 2018, 17:21:26 UTC | Remove unused arg which causes compilation failure (#4080) Summary: It seems that compilation has been made stricter about unused args. Closes https://github.com/facebook/rocksdb/pull/4080 Differential Revision: D8712049 Pulled By: sagar0 fbshipit-source-id: 984af1982638af3568aac1a167f565f4741badee | 02 July 2018, 17:27:36 UTC |
e099c2d | Andrey Zagrebin | 29 June 2018, 23:03:28 UTC | check if data size exceeds java array vm limit when it is copied in jni (#3850) Summary: to address issue #3849 Closes https://github.com/facebook/rocksdb/pull/3850 Differential Revision: D8695487 Pulled By: sagar0 fbshipit-source-id: 04baeb2127663934ed1321fe6d9a9ec23c86e16b | 29 June 2018, 23:13:06 UTC |
36fa49c | Daniel Black | 29 June 2018, 18:42:08 UTC | transaction_test: -Wunused-variable with clang-7 (#4074) Summary: clang version 7.0.0- (trunk) Target: x86_64-unknown-linux-gnu Thread model: posix InstalledDir: /usr/bin clang++-7 -DROCKSDB_USE_RTTI -g -W -Wextra -Wall -Wsign-compare -Wshadow -Wno-unused-parameter -Werror -I. -I./include -std=c++11 -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -fno-builtin-memcmp -DROCKSDB_FALLOCATE_PRESENT -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX -DROCKSDB_BACKTRACE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -Wshorten-64-to-32 -march=native -DHAVE_SSE42 -DROCKSDB_SUPPORT_THREAD_LOCAL -isystem ./third-party/gtest-1.7.0/fused-src -DTRAVIS -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -c utilities/transactions/transaction_test.cc -o utilities/transactions/transaction_test.o utilities/transactions/transaction_test.cc:2282:22: error: unused variable 'txn_options' [-Werror,-Wunused-variable] TransactionOptions txn_options; ^ utilities/transactions/transaction_test.cc:2822:22: error: unused variable 'txn_options' [-Werror,-Wunused-variable] TransactionOptions txn_options; ^ utilities/transactions/transaction_test.cc:2928:22: error: unused variable 'txn_options' [-Werror,-Wunused-variable] TransactionOptions txn_options; ^ utilities/transactions/transaction_test.cc:3109:22: error: unused variable 'txn_options' [-Werror,-Wunused-variable] TransactionOptions txn_options; ^ utilities/transactions/transaction_test.cc:4364:22: error: unused variable 'txn_options' [-Werror,-Wunused-variable] TransactionOptions txn_options; ^ Closes https://github.com/facebook/rocksdb/pull/4074 Differential Revision: D8698051 Pulled By: ajkr fbshipit-source-id: 6255618eefdd189962fbea1b02cf1eb5ae501274 | 29 June 2018, 18:43:36 UTC |
2462763 | Maysam Yabandeh | 29 June 2018, 16:13:57 UTC | Fix mis-spoken assert on prefetch_filter and prefetch_index (#4077) Summary: We can have prefetch_index without prefetch_filter but not the other way around. The assert statement is fixed. Closes https://github.com/facebook/rocksdb/pull/4077 Differential Revision: D8694472 Pulled By: maysamyabandeh fbshipit-source-id: ccd2804d9d9cdafb1c3e65062c7bc38603e69004 | 29 June 2018, 16:28:12 UTC |
29ffbb8 | Maysam Yabandeh | 29 June 2018, 15:55:33 UTC | Charging block cache more accurately (#4073) Summary: Currently the block cache is charged only by the size of the raw data block and excludes the overhead of the c++ objects that contain the raw data block. The patch improves the accuracy of the charge by including the c++ object overhead into it. Closes https://github.com/facebook/rocksdb/pull/4073 Differential Revision: D8686552 Pulled By: maysamyabandeh fbshipit-source-id: 8472f7fc163c0644533bc6942e20cdd5725f520f | 29 June 2018, 15:57:20 UTC |
b3efb1c | Zhongyi Xie | 29 June 2018, 01:59:18 UTC | fix clang analyzer warnings (#4072) Summary: clang analyze is giving the following warnings: > db/compaction_job.cc:1178:16: warning: Called C++ object pointer is null } else if (meta->smallest.size() > 0) { ^~~~~~~~~~~~~~~~~~~~~ db/compaction_job.cc:1201:33: warning: Access to field 'marked_for_compaction' results in a dereference of a null pointer (loaded from variable 'meta') meta->marked_for_compaction = sub_compact->builder->NeedCompact(); ~~~~ db/version_set.cc:2770:26: warning: Called C++ object pointer is null uint32_t cf_id = last_writer->cfd->GetID(); ^~~~~~~~~~~~~~~~~~~~~~~~~ Closes https://github.com/facebook/rocksdb/pull/4072 Differential Revision: D8685852 Pulled By: miasantreble fbshipit-source-id: b0e2fd9dfc1cbba2317723e09886384b9b1c9085 | 29 June 2018, 02:12:35 UTC |
8ad63a4 | Manuel Ung | 29 June 2018, 01:46:39 UTC | WriteUnPrepared: Add new WAL marker kTypeBeginUnprepareXID (#4069) Summary: This adds a new WAL marker of type kTypeBeginUnprepareXID. Also, DBImpl now contains a field called batch_per_txn (meaning one WriteBatch per transaction, or possibly multiple WriteBatches). This would also indicate that this DB is using WriteUnprepared policy. Recovery code would be able to make use of this extra field on DBImpl in a separate diff. For now, it is just used to determine whether the WAL is compatible or not. Closes https://github.com/facebook/rocksdb/pull/4069 Differential Revision: D8675099 Pulled By: lth fbshipit-source-id: ca27cae1738e46d65f2bb92860fc759deb874749 | 29 June 2018, 01:58:29 UTC |
25403c2 | Andrew Kryczka | 28 June 2018, 20:02:55 UTC | Prefetch cache lines for filter lookup (#4068) Summary: Since the filter data is unaligned, even though we ensure all probes are within a span of `cache_line_size` bytes, those bytes can span two cache lines. In that case I doubt hardware prefetching does a great job considering we don't necessarily access those two cache lines in order. This guess seems correct since adding explicit prefetch instructions reduced filter lookup overhead by 19.4%. Closes https://github.com/facebook/rocksdb/pull/4068 Differential Revision: D8674189 Pulled By: ajkr fbshipit-source-id: 747427d9a17900151c17820488e3f7efe06b1871 | 28 June 2018, 20:20:29 UTC |
52d4c9b | Anand Ananthabhotla | 28 June 2018, 19:23:57 UTC | Allow DB resume after background errors (#3997) Summary: Currently, if RocksDB encounters errors during a write operation (user requested or BG operations), it sets DBImpl::bg_error_ and fails subsequent writes. This PR allows the DB to be resumed for certain classes of errors. It consists of 3 parts - 1. Introduce Status::Severity in rocksdb::Status to indicate whether a given error can be recovered from or not 2. Refactor the error handling code so that setting bg_error_ and deciding on severity is in one place 3. Provide an API for the user to clear the error and resume the DB instance This whole change is broken up into multiple PRs. Initially, we only allow clearing the error for Status::NoSpace() errors during background flush/compaction. Subsequent PRs will expand this to include more errors and foreground operations such as Put(), and implement a polling mechanism for out-of-space errors. Closes https://github.com/facebook/rocksdb/pull/3997 Differential Revision: D8653831 Pulled By: anand1976 fbshipit-source-id: 6dc835c76122443a7668497c0226b4f072bc6afd | 28 June 2018, 19:34:40 UTC |
26d67e3 | Yanqin Jin | 28 June 2018, 19:16:10 UTC | Support group commits of version edits (#3944) Summary: This PR supports the group commit of multiple version edit entries corresponding to different column families. Column family drop/creation still cannot be grouped. This PR is a subset of [PR 3752](https://github.com/facebook/rocksdb/pull/3752). Closes https://github.com/facebook/rocksdb/pull/3944 Differential Revision: D8432536 Pulled By: riversand963 fbshipit-source-id: 8f11bd05193b6c0d9272d82e44b676abfac113cb | 28 June 2018, 19:34:39 UTC |
0a5b5d8 | Maysam Yabandeh | 28 June 2018, 15:31:19 UTC | Remove ReadOnly part of PinnableSliceAndMmapReads from Lite (#4070) Summary: Lite does not support readonly DBs. Closes https://github.com/facebook/rocksdb/pull/4070 Differential Revision: D8677858 Pulled By: maysamyabandeh fbshipit-source-id: 536887d2363ee2f5d8e1ea9f1a511e643a1707fa | 28 June 2018, 15:42:17 UTC |
b557499 | Taewook Oh | 28 June 2018, 04:59:09 UTC | Suppress leak warning for clang(LLVM) asan (#4066) Summary: Instead of __SANITIZE_ADDRESS__ macro, LLVM uses __has_feature(address_sanitzer) to check if ASAN is enabled for the build. I tested it with MySQL sanitizer build that uses RocksDB as a submodule. Closes https://github.com/facebook/rocksdb/pull/4066 Reviewed By: riversand963 Differential Revision: D8668941 Pulled By: taewookoh fbshipit-source-id: af4d1da180c1470d257a228f431eebc61490bc36 | 28 June 2018, 05:13:48 UTC |
7f850b8 | Yanqin Jin | 28 June 2018, 03:55:07 UTC | Remove 'ALIGNAS' from StatisticsImpl. (#4061) Summary: Remove over-alignment on `StatisticsImpl` whose benefit is vague and causes UBSAN check to fail due to `std::make_shared` not respecting the over-alignment requirement. Test plan ``` $ make clean && COMPILE_WITH_UBSAN=1 OPT=-g make -j16 ubsan_check ``` Closes https://github.com/facebook/rocksdb/pull/4061 Differential Revision: D8656506 Pulled By: riversand963 fbshipit-source-id: db355ae9c7bdd2c9e9c5e63cabba13d8d82cc5f9 | 28 June 2018, 03:59:45 UTC |
14f409c | Zhongyi Xie | 28 June 2018, 03:38:53 UTC | PrefixMayMatch: remove unnecessary check for prefix_extractor_ (#4067) Summary: with https://github.com/facebook/rocksdb/pull/3601 and https://github.com/facebook/rocksdb/pull/3899, `prefix_extractor_` is not really being used in block based filter and full filter's version of `PrefixMayMatch` because now `prefix_extractor` is passed as an argument. Also it is now possible that prefix_extractor_ may be initialized to nullptr when a non-standard prefix_extractor is used and also for ROCKSDB_LITE. Removing these checks should not break any existing tests. Closes https://github.com/facebook/rocksdb/pull/4067 Differential Revision: D8669002 Pulled By: miasantreble fbshipit-source-id: 0e701ba912b8a26734fadb72d15bb1b266b6176a | 28 June 2018, 03:42:43 UTC |
1f6efab | Zhichao Cao | 28 June 2018, 00:34:07 UTC | Add bottommost_compression_opts to for bottommost_compression (#3985) Summary: …ression For `CompressionType` we have options `compression` and `bottommost_compression`. Thus, to make the compression options consitent with the compression type when bottommost_compression is enabled, we add the bottommost_compression_opts Closes https://github.com/facebook/rocksdb/pull/3985 Reviewed By: riversand963 Differential Revision: D8385911 Pulled By: zhichao-cao fbshipit-source-id: 07bc533dd61bcf1cef5927d8d62901c13d38d5fc | 28 June 2018, 00:42:38 UTC |
235ab9d | Maysam Yabandeh | 28 June 2018, 00:09:29 UTC | Pin mmap files in ReadOnlyDB (#4053) Summary: https://github.com/facebook/rocksdb/pull/3881 fixed a bug where PinnableSlice pin mmap files which could be deleted with background compaction. This is however a non-issue for ReadOnlyDB when there is no compaction running and max_open_files is -1. This patch reenables the pinning feature for that case. Closes https://github.com/facebook/rocksdb/pull/4053 Differential Revision: D8662546 Pulled By: maysamyabandeh fbshipit-source-id: 402962602eb0f644e17822748332999c3af029fd | 28 June 2018, 00:13:34 UTC |
e8f9d7f | Maximilian Alexander | 27 June 2018, 22:40:33 UTC | Added PingCaps Rust RocksDB and ObjectiveRocks (#4065) Summary: 1. I added PingCap's more up-to-date Rust Binding of RocksDB 2. I also added ObjectiveRocks which is a very nice binding for _both_ Swift and Objective-C Closes https://github.com/facebook/rocksdb/pull/4065 Differential Revision: D8670340 Pulled By: siying fbshipit-source-id: 3db28bf3a464c3e050df52cc92b19248b7f43944 | 27 June 2018, 22:43:21 UTC |
818c84e | chouxi | 27 June 2018, 19:22:17 UTC | Store timestamp in deadlock detection (#4060) Summary: - Summary Add timestamp into the DeadlockInfo to store the timestamp when deadlock detected on the rocksdb side. - Testplan: `make check -j64` Closes https://github.com/facebook/rocksdb/pull/4060 Differential Revision: D8655380 Pulled By: chouxi fbshipit-source-id: f58e1aa5e09eb1d1eed0a181d4e2304aaf01efe8 | 27 June 2018, 19:27:58 UTC |
e5ae1bb | Daniel Black | 27 June 2018, 19:09:50 UTC | Remove bogus gcc-8.1 warning (#3870) Summary: Various rearrangements of the cch maths failed or replacing = '\0' with memset failed to convince the compiler it was nul terminated. So took the perverse option of changing strncpy to strcpy. Return null if memory couldn't be allocated. util/status.cc: In static member function ‘static const char* rocksdb::Status::CopyState(const char*)’: util/status.cc:28:15: error: ‘char* strncpy(char*, const char*, size_t)’ output truncated before terminating nul copying as many bytes from a string as its length [-Werror=stringop-truncation] std::strncpy(result, state, cch - 1); ~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~ util/status.cc:19:18: note: length computed here std::strlen(state) + 1; // +1 for the null terminator ~~~~~~~~~~~^~~~~~~ cc1plus: all warnings being treated as errors make: *** [Makefile:645: shared-objects/util/status.o] Error 1 closes #2705 Closes https://github.com/facebook/rocksdb/pull/3870 Differential Revision: D8594114 Pulled By: anand1976 fbshipit-source-id: ab20f3a456a711e4d29144ebe630e4fe3c99ec25 | 27 June 2018, 19:23:07 UTC |
a16e00b | Manuel Ung | 27 June 2018, 19:05:29 UTC | WriteUnPrepared Txn: Disable seek to snapshot optimization (#3955) Summary: This is implemented by extending ReadCallback with another function `MaxUnpreparedSequenceNumber` which returns the largest visible sequence number for the current transaction, if there is uncommitted data written to DB. Otherwise, it returns zero, indicating no uncommitted data. There are the places where reads had to be modified. - Get and Seek/Next was just updated to seek to max(snapshot_seq, MaxUnpreparedSequenceNumber()) instead, and iterate until a key was visible. - Prev did not need need updates since it did not use the Seek to sequence number optimization. Assuming that locks were held when writing unprepared keys, and ValidateSnapshot runs, there should only be committed keys and unprepared keys of the current transaction, all of which are visible. Prev will simply iterate to get the last visible key. - Reseeking to skip keys optimization was also disabled for write unprepared, since it's possible to hit the max_skip condition even while reseeking. There needs to be some way to resolve infinite looping in this case. Closes https://github.com/facebook/rocksdb/pull/3955 Differential Revision: D8286688 Pulled By: lth fbshipit-source-id: 25e42f47fdeb5f7accea0f4fd350ef35198caafe | 27 June 2018, 19:23:07 UTC |
17339dc | Nikhil Benesch | 27 June 2018, 03:18:43 UTC | Add table property tracking number of range deletions (#4016) Summary: Add a new table property, rocksdb.num.range-deletions, which tracks the number of range deletions in a block-based table. Range deletions are no longer counted in rocksdb.num.entries; as discovered in PR #3778, there are various code paths that implicitly assume that rocksdb.num.entries counts only true keys, not range deletions. /cc ajkr nvanbenschoten Closes https://github.com/facebook/rocksdb/pull/4016 Differential Revision: D8527575 Pulled By: ajkr fbshipit-source-id: 92e7edbe78fda53756a558013c9fb496e7764fd7 | 27 June 2018, 03:27:35 UTC |
408205a | Zhongyi Xie | 26 June 2018, 22:56:26 UTC | use user_key and iterate_upper_bound to determine compatibility of bloom filters (#3899) Summary: Previously in https://github.com/facebook/rocksdb/pull/3601 bloom filter will only be checked if `prefix_extractor` in the mutable_cf_options matches the one found in the SST file. This PR relaxes the requirement by checking if all keys in the range [user_key, iterate_upper_bound) all share the same prefix after transforming using the BF in the SST file. If so, the bloom filter is considered compatible and will continue to be looked at. Closes https://github.com/facebook/rocksdb/pull/3899 Differential Revision: D8157459 Pulled By: miasantreble fbshipit-source-id: 18d17cba56a1005162f8d5db7a27aba277089c41 | 26 June 2018, 22:57:26 UTC |
967aa81 | Bas van Schaik | 26 June 2018, 19:37:07 UTC | Create lgtm.yml for LGTM.com C/C++ analysis (#4058) Summary: As discussed with thatsafunnyname [here](https://discuss.lgtm.com/t/c-c-lang-missing-for-facebook-rocksdb/1079): this configuration enables C/C++ analysis for RocksDB on LGTM.com. The initial commit will contain a build command (simple `make`) that previously resulted in a build error. The build log will then be available on LGTM.com for you to investigate (if you like). I'll immediately add a second commit to this PR to correct the build command to `make static_lib`, which worked when I tested it earlier today. If you like you can also enable automatic code review in pull requests. This will alert you to any new code issues before they actually get merged into `master`. Here's an example of how that works for the AMPHTML project: https://github.com/ampproject/amphtml/pull/13060. You can enable it yourself here: https://lgtm.com/projects/g/facebook/rocksdb/ci/. I'll also add a badge to your README.md in a separate commit — feel free to remove that from this PR if you don't like it. (Full disclosure: I'm part of the LGTM.com team :slightly_smiling_face:. Ping samlanning) Closes https://github.com/facebook/rocksdb/pull/4058 Differential Revision: D8648410 Pulled By: ajkr fbshipit-source-id: 98d55fc19cff1b07268ac8425b63e764806065aa | 26 June 2018, 19:43:04 UTC |
2694b6d | Peter (Stig) Edwards | 26 June 2018, 19:35:26 UTC | Remove unused imports, from python scripts. (#4057) Summary: Also remove redefined variable. As reported on https://lgtm.com/projects/g/facebook/rocksdb/ Closes https://github.com/facebook/rocksdb/pull/4057 Differential Revision: D8648342 Pulled By: ajkr fbshipit-source-id: afd2ba84d1364d316010179edd44777e64ca9183 | 26 June 2018, 19:43:04 UTC |
a8e503e | Andrew Kryczka | 26 June 2018, 17:28:41 UTC | Fix universal compaction scheduling conflict with CompactFiles (#4055) Summary: Universal size-amp-triggered compaction was pulling the final sorted run into the compaction without checking whether any of its files are already being compacted. When all compactions are automatic, it is safe since it verifies the second-last sorted run is not already being compacted, which implies the last sorted run is also not being compacted (in automatic compaction multiple sorted runs are always compacted together). But with manual compaction, files in the last sorted run can be compacted independently, so the last sorted run also must be checked. We were seeing the below assertion failure in `db_stress`. Also the test case included in this PR repros the failure. ``` db_universal_compaction_test: db/compaction.cc:312: void rocksdb::Compaction::MarkFilesBeingCompacted(bool): Assertion `mark_as_compacted ? !inputs_[i][j]->being_compacted : inputs_[i][j]->being_compacted' failed. Aborted (core dumped) ``` Closes https://github.com/facebook/rocksdb/pull/4055 Differential Revision: D8630094 Pulled By: ajkr fbshipit-source-id: ac3b30a874678b76e113d4f6c42c1260411b08f8 | 26 June 2018, 17:44:56 UTC |
346d106 | Daniel Black | 26 June 2018, 05:42:35 UTC | Align StatisticsImpl / StatisticsData (#4036) Summary: Pinned the alignment of StatisticsData to the cacheline size rather than just extending its size (which could go over two cache lines)if unaligned in allocation. Avoid compile errors in the process as per individual commit messages. strengthen static_assert to CACHELINE rather than the highest common multiple. Closes https://github.com/facebook/rocksdb/pull/4036 Differential Revision: D8582844 Pulled By: yiwu-arbug fbshipit-source-id: 363c37029f28e6093e06c60b987bca9aa204bc71 | 26 June 2018, 05:58:19 UTC |
6d454d7 | Yi Wu | 26 June 2018, 05:32:29 UTC | BlobDB: is_fifo=true also evict non-TTL blob files (#4049) Summary: Previously with is_fifo=true we only evict TTL file. Changing it to also evict non-TTL files from oldest to newest, after exhausted TTL files. Closes https://github.com/facebook/rocksdb/pull/4049 Differential Revision: D8604597 Pulled By: yiwu-arbug fbshipit-source-id: bc4209ee27c1528ce4b72833e6f1e1bff80082c1 | 26 June 2018, 05:43:05 UTC |
189f0c2 | Sagar Vemuri | 25 June 2018, 20:07:38 UTC | Make BlockBasedTableIterator compaction-aware (#4048) Summary: Pass in `for_compaction` to `BlockBasedTableIterator` via `BlockBasedTableReader::NewIterator`. In 7103559f49b46b3287973045f741c0679e3e9e44, `for_compaction` was set in `BlockBasedTable::Rep` via `BlockBasedTable::SetupForCompaction`. In hindsight it was not the right decision; it also caused TSAN to complain. Closes https://github.com/facebook/rocksdb/pull/4048 Differential Revision: D8601056 Pulled By: sagar0 fbshipit-source-id: 30127e898c15c38c1080d57710b8c5a6d64a0ab3 | 25 June 2018, 20:19:27 UTC |
a71e467 | Yi Wu | 24 June 2018, 06:08:33 UTC | Blob DB: enable readahead for garbage collection (#3648) Summary: Enable readahead for blob DB garbage collection, which should improve GC performance a little bit. Closes https://github.com/facebook/rocksdb/pull/3648 Differential Revision: D7383791 Pulled By: yiwu-arbug fbshipit-source-id: 642b3327f7105eca85986d3fb2d8f960a3d83cf1 | 24 June 2018, 06:12:00 UTC |
2729dd7 | Yanqin Jin | 23 June 2018, 23:56:49 UTC | Reclaim memory allocated to backup_engine. Summary: Closes https://github.com/facebook/rocksdb/pull/4045 Differential Revision: D8595609 Pulled By: riversand963 fbshipit-source-id: 5ba5954d804b82b0e7264b2e18e1da4c94103b53 | 24 June 2018, 00:12:14 UTC |
80ade9a | Maysam Yabandeh | 22 June 2018, 22:14:05 UTC | Pin top-level index on partitioned index/filter blocks (#4037) Summary: Top-level index in partitioned index/filter blocks are small and could be pinned in memory. So far we use that by cache_index_and_filter_blocks to false. This however make it difficult to keep account of the total memory usage. This patch introduces pin_top_level_index_and_filter which in combination with cache_index_and_filter_blocks=true keeps the top-level index in cache and yet pinned them to avoid cache misses and also cache lookup overhead. Closes https://github.com/facebook/rocksdb/pull/4037 Differential Revision: D8596218 Pulled By: maysamyabandeh fbshipit-source-id: 3a5f7f9ca6b4b525b03ff6bd82354881ae974ad2 | 22 June 2018, 22:27:46 UTC |
c726f7f | Yi Wu | 22 June 2018, 18:39:51 UTC | Fix dangling checkpoint pointer in db_stress (#4042) Summary: Fix db_stress failed to delete checkpoint pointer. It's caught by asan_crash test. Closes https://github.com/facebook/rocksdb/pull/4042 Differential Revision: D8592604 Pulled By: yiwu-arbug fbshipit-source-id: 7b2d67d5e3dfb05f71c33fcf320482303e97d3ef | 22 June 2018, 18:43:50 UTC |
64c85d0 | Adam Retter | 22 June 2018, 17:49:11 UTC | Set DEBUG_LEVEL=0 for RocksJava Mac Release (#4040) Summary: Closes https://github.com/facebook/rocksdb/issues/2717 Closes https://github.com/facebook/rocksdb/pull/4040 Differential Revision: D8592058 Pulled By: sagar0 fbshipit-source-id: d01099a1067aa32659abb0b4bed641d919a3927e | 22 June 2018, 17:57:48 UTC |
795e663 | Zhongyi Xie | 22 June 2018, 04:16:49 UTC | option for timing measurement of non-blocking ops during compaction (#4029) Summary: For example calling CompactionFilter is always timed and gives the user no way to disable. This PR will disable the timer if `Statistics::stats_level_` (which is part of DBOptions) is `kExceptDetailedTimers` Closes https://github.com/facebook/rocksdb/pull/4029 Differential Revision: D8583670 Pulled By: miasantreble fbshipit-source-id: 913be9fe433ae0c06e88193b59d41920a532307f | 22 June 2018, 04:28:05 UTC |
0a5b16c | Andrew Kryczka | 21 June 2018, 23:22:57 UTC | Cleanup staging directory at start of checkpoint (#4035) Summary: - Attempt to clean the checkpoint staging directory before starting a checkpoint. It was already cleaned up at the end of checkpoint. But it wasn't cleaned up in the edge case where the process crashed while staging checkpoint files. - Attempt to clean the checkpoint directory before calling `Checkpoint::Create` in `db_stress`. This handles the case where checkpoint directory was created by a previous `db_stress` run but the process crashed before cleaning it up. - Use `DestroyDB` for cleaning checkpoint directory since a checkpoint is a DB. Closes https://github.com/facebook/rocksdb/pull/4035 Reviewed By: yiwu-arbug Differential Revision: D8580223 Pulled By: ajkr fbshipit-source-id: 28c667400e249fad0fdedc664b349031b7b61599 | 21 June 2018, 23:27:12 UTC |
645e57c | Sagar Vemuri | 21 June 2018, 21:45:49 UTC | Assert for Direct IO at the beginning in PositionedRead (#3891) Summary: Moved the direct-IO assertion to the top in `PosixSequentialFile::PositionedRead`, as it doesn't make sense to check for sector alignments before checking for direct IO. Closes https://github.com/facebook/rocksdb/pull/3891 Differential Revision: D8267972 Pulled By: sagar0 fbshipit-source-id: 0ecf77c0fb5c35747a4ddbc15e278918c0849af7 | 21 June 2018, 21:58:01 UTC |
58c2214 | Yi Wu | 21 June 2018, 21:30:00 UTC | Update TARGETS file (#4028) Summary: -Wshorten-64-to-32 is invalid flag in fbcode. Changing it to -Warrowing. Closes https://github.com/facebook/rocksdb/pull/4028 Differential Revision: D8553694 Pulled By: yiwu-arbug fbshipit-source-id: 1523cbcb4c76cf1d2b10a4d28b5f58c78e6cb876 | 21 June 2018, 21:42:39 UTC |
3974959 | Yanqin Jin | 21 June 2018, 18:09:09 UTC | Fix a warning (treated as error) caused by type mismatch. Summary: Closes https://github.com/facebook/rocksdb/pull/4032 Differential Revision: D8573061 Pulled By: riversand963 fbshipit-source-id: 112324dcb35956d6b3ec891073f4f21493933c8b | 21 June 2018, 18:13:09 UTC |
7103559 | Sagar Vemuri | 21 June 2018, 18:02:49 UTC | Improve direct IO range scan performance with readahead (#3884) Summary: This PR extends the improvements in #3282 to also work when using Direct IO. We see **4.5X performance improvement** in seekrandom benchmark doing long range scans, when using direct reads, on flash. **Description:** This change improves the performance of iterators doing long range scans (e.g. big/full index or table scans in MyRocks) by using readahead and prefetching additional data on each disk IO, and storing in a local buffer. This prefetching is automatically enabled on noticing more than 2 IOs for the same table file during iteration. The readahead size starts with 8KB and is exponentially increased on each additional sequential IO, up to a max of 256 KB. This helps in cutting down the number of IOs needed to complete the range scan. **Implementation Details:** - Used `FilePrefetchBuffer` as the underlying buffer to store the readahead data. `FilePrefetchBuffer` can now take file_reader, readahead_size and max_readahead_size as input to the constructor, and automatically do readahead. - `FilePrefetchBuffer::TryReadFromCache` can now call `FilePrefetchBuffer::Prefetch` if readahead is enabled. - `AlignedBuffer` (which is the underlying store for `FilePrefetchBuffer`) now takes a few additional args in `AlignedBuffer::AllocateNewBuffer` to allow copying data from the old buffer. - Made sure not to re-read partial chunks of data that were already available in the buffer, from device again. - Fixed a couple of cases where `AlignedBuffer::cursize_` was not being properly kept up-to-date. **Constraints:** - Similar to #3282, this gets currently enabled only when ReadOptions.readahead_size = 0 (which is the default value). - Since the prefetched data is stored in a temporary buffer allocated on heap, this could increase the memory usage if you have many iterators doing long range scans simultaneously. - Enabled only for user reads, and disabled for compactions. Compaction reads are controlled by the options `use_direct_io_for_flush_and_compaction` and `compaction_readahead_size`, and the current feature takes precautions not to mess with them. **Benchmarks:** I used the same benchmark as used in #3282. Data fill: ``` TEST_TMPDIR=/data/users/$USER/benchmarks/iter ./db_bench -benchmarks=fillrandom -num=1000000000 -compression_type="none" -level_compaction_dynamic_level_bytes ``` Do a long range scan: Seekrandom with large number of nexts ``` TEST_TMPDIR=/data/users/$USER/benchmarks/iter ./db_bench -benchmarks=seekrandom -use_direct_reads -duration=60 -num=1000000000 -use_existing_db -seek_nexts=10000 -statistics -histogram ``` ``` Before: seekrandom : 37939.906 micros/op 26 ops/sec; 29.2 MB/s (1636 of 1999 found) With this change: seekrandom : 8527.720 micros/op 117 ops/sec; 129.7 MB/s (6530 of 7999 found) ``` ~4.5X perf improvement. Taken on an average of 3 runs. Closes https://github.com/facebook/rocksdb/pull/3884 Differential Revision: D8082143 Pulled By: sagar0 fbshipit-source-id: 4d7a8561cbac03478663713df4d31ad2620253bb | 21 June 2018, 18:13:08 UTC |
524c6e6 | Yanqin Jin | 21 June 2018, 15:34:24 UTC | Add file name info to SequentialFileReader. (#4026) Summary: We potentially need this information for tracing, profiling and diagnosis. Closes https://github.com/facebook/rocksdb/pull/4026 Differential Revision: D8555214 Pulled By: riversand963 fbshipit-source-id: 4263e06c00b6d5410b46aa46eb4e358ff2161dd2 | 21 June 2018, 15:42:24 UTC |
14cee19 | Andrew Kryczka | 21 June 2018, 05:14:12 UTC | Support file ingestion in stress test (#4018) Summary: Once per `ingest_external_file_one_in` operations, uses SstFileWriter to create a file containing `ingest_external_file_width` consecutive keys. The file is named containing the thread ID to avoid clashes. The file is then added to the DB using `IngestExternalFile`. We can't enable it by default in crash test because `nooverwritepercent` and `test_batches_snapshot` both must be zero for the DB's whole lifetime. Perhaps we should setup a separate test with that config as range deletion also requires it. Closes https://github.com/facebook/rocksdb/pull/4018 Differential Revision: D8507698 Pulled By: ajkr fbshipit-source-id: 1437ea26fd989349a9ce8b94117241c65e40f10f | 21 June 2018, 05:27:45 UTC |
61d69d4 | Dmitri Smirnov | 20 June 2018, 00:07:49 UTC | Hide jemalloc aligned allocation functions into .cc (#4025) Summary: so they could be overriden Closes https://github.com/facebook/rocksdb/pull/4025 Differential Revision: D8526287 Pulled By: siying fbshipit-source-id: 9537b299dc907b4d1eeaf77a8784b13cb058280d | 20 June 2018, 00:12:06 UTC |
28a9d89 | Maysam Yabandeh | 19 June 2018, 21:06:17 UTC | Fix the bug with duplicate prefix in partition filters (#4024) Summary: https://github.com/facebook/rocksdb/pull/3764 introduced an optimization feature to skip duplicate prefix entires in full bloom filters. Unfortunately it also introduces a bug in partitioned full filters, where the duplicate prefix should still be inserted if it is in a new partition. The patch fixes the bug by resetting the duplicate detection logic each time a partition is cut. This bug could result into false negatives, which means that DB could skip an existing key. Closes https://github.com/facebook/rocksdb/pull/4024 Differential Revision: D8518866 Pulled By: maysamyabandeh fbshipit-source-id: 044f4d988e606a330ecafd8c79daceb68b8796bf | 19 June 2018, 21:12:46 UTC |
92ee335 | Siying Dong | 19 June 2018, 16:42:40 UTC | BlockBasedTableIterator to keep BlockIter after out of upper bound (#4004) Summary: b555ed30a4a93b80a3ac4781c6721ab988e03b5b makes the BlockBasedTableIterator to be invalidated if the current position if over the upper bound. However, this can bring performance regression to the case of multiple Seek()s hitting the same data block but all out of upper bound. For example, if an SST file has a data block containing following keys : {a, z} The user sets the upper bound to be "x", and it executed following queries: Seek("b") Seek("c") Seek("d") Before the upper bound optimization, these queries always come to this same current data block of the iterator, but now inside each Seek() the data block is read from the block cache but is returned again. To prevent this regression case, we keep the current data block iterator if it is upper bound. Closes https://github.com/facebook/rocksdb/pull/4004 Differential Revision: D8463192 Pulled By: siying fbshipit-source-id: 8710628b30acde7063a097c3184d6c4333a8ef81 | 19 June 2018, 16:57:11 UTC |
7f3a634 | Andrew Kryczka | 19 June 2018, 15:56:43 UTC | Support pipelined write in stress/crash tests Summary: Closes https://github.com/facebook/rocksdb/pull/4019 Differential Revision: D8508681 Pulled By: ajkr fbshipit-source-id: 23a3c07d642386446e322b02e69cdf70d12ef009 | 19 June 2018, 16:14:12 UTC |
8585059 | Andrew Kryczka | 19 June 2018, 02:23:42 UTC | Support backup and checkpoint in db_stress (#4005) Summary: Add the `backup_one_in` and `checkpoint_one_in` options to periodically trigger backups and checkpoints. The directory names contain thread ID to avoid clashing with parallel backups/checkpoints. Enable checkpoint in crash test so our CI runs will use it. Didn't enable backup in crash test since it copies all the files which is too slow. Closes https://github.com/facebook/rocksdb/pull/4005 Differential Revision: D8472275 Pulled By: ajkr fbshipit-source-id: ff91bdc37caac4ffd97aea8df96b3983313ac1d5 | 19 June 2018, 02:28:18 UTC |
de2c6fb | Andrew Kryczka | 19 June 2018, 00:42:48 UTC | Fix stderr processing in crash test (#4006) Summary: Fixed bug where `db_stress` output a line with a warning followed by a line with an error, and `db_crashtest.py` considered that a success. For example: ``` WARNING: prefix_size is non-zero but memtablerep != prefix_hash open error: Corruption: SST file is ahead of WALs ``` Closes https://github.com/facebook/rocksdb/pull/4006 Differential Revision: D8473463 Pulled By: ajkr fbshipit-source-id: 60461bdd7491d9d26c63f7d4ee522a0f88ba3de7 | 19 June 2018, 00:58:13 UTC |
c766887 | Tomas Kolda | 18 June 2018, 21:38:50 UTC | Fix ExternalSSTFileTest::OverlappingRanges test on Solaris Sparc (#4012) Summary: Fix of #4011 Closes https://github.com/facebook/rocksdb/pull/4012 Differential Revision: D8499173 Pulled By: sagar0 fbshipit-source-id: cbb2b90c544ed364a3640ea65835d577b2dbc5df | 18 June 2018, 21:57:37 UTC |
7b4b43f | Tomas Kolda | 18 June 2018, 20:52:46 UTC | zLinux build error with gcc and IBM Java headers (#4013) Summary: `SetByteArrayRegion` does not have const source buffer thus compilation error. I have made that same as in other JNI files (const_cast). It was missing for new transaction functionality added recently. Closes https://github.com/facebook/rocksdb/pull/4013 Differential Revision: D8493290 Pulled By: sagar0 fbshipit-source-id: 14afedf365b111121bd11e68a8d546a1cae68b26 | 18 June 2018, 20:58:28 UTC |
e5bee40 | Tomas Kolda | 18 June 2018, 16:42:29 UTC | zLinux s390x support in JNI (#4009) Summary: Adding support for zLinux on s390x architecture in JNI. Closes https://github.com/facebook/rocksdb/pull/4009 Differential Revision: D8483750 Pulled By: siying fbshipit-source-id: e681657c27e7a28f1731e08e8570382de5deff44 | 18 June 2018, 16:57:02 UTC |
e750dac | Tomas Kolda | 18 June 2018, 03:54:22 UTC | Crash on Windows, because of shared_ptr reinterpret cast (#3999) Summary: For more details see #3998 Closes https://github.com/facebook/rocksdb/pull/3999 Differential Revision: D8458905 Pulled By: sagar0 fbshipit-source-id: d6e09182933253a08eaf81ac7cfe50ed3b6576c5 | 18 June 2018, 03:56:33 UTC |
80bc359 | Zhongyi Xie | 16 June 2018, 02:24:21 UTC | Should only decode restart points for uncompressed blocks (#3996) Summary: The Block object assumes contents are uncompressed. Block's constructor tries to read the number of restarts, but does not get an accurate number when its contents are compressed, which is causing issues like https://github.com/facebook/rocksdb/issues/3843. This PR address this issue by skipping reconstruction of restart points when blocks are known to be compressed. Somehow the restart points can be read directly when Snappy is used and some tests (for example https://github.com/facebook/rocksdb/blob/master/db/db_block_cache_test.cc#L196) expects blocks to be fully constructed even when Snappy compression is used, so here we keep the restart point logic for Snappy. Closes https://github.com/facebook/rocksdb/pull/3996 Differential Revision: D8416186 Pulled By: miasantreble fbshipit-source-id: 002c0b62b9e5d89fb7736563d354ce0023c8cb28 | 16 June 2018, 02:26:58 UTC |
c48764b | Anand Ananthabhotla | 16 June 2018, 00:53:12 UTC | Don't generate a notification for a 0 size SST (#4003) Summary: Don't call the OnTableFileCreated listener callback when a 0 size SST file gets created by Flush. Doing so causes an assertion failure in db_stress. It is also not correct behavior as we call env->DeleteFile() for such files right before the notification. Closes https://github.com/facebook/rocksdb/pull/4003 Differential Revision: D8461385 Pulled By: anand1976 fbshipit-source-id: ae92d4f921c2e2cff981ad58f4929ed8b609f35d | 16 June 2018, 00:57:24 UTC |
3fbc865 | zhichao-cao | 16 June 2018, 00:23:08 UTC | Add kOptionsStatistics to GetProperty() (#3966) Summary: Add a new DB property to DB::GetProperty(), which returns the option.statistics. Test is updated to pass. Closes https://github.com/facebook/rocksdb/pull/3966 Differential Revision: D8311139 Pulled By: zhichao-cao fbshipit-source-id: ea78f4727358c807b0e5a0ea62e09defb10ad9ac | 16 June 2018, 00:28:01 UTC |
7b5f7ff | Daniel Black | 15 June 2018, 23:42:35 UTC | travis: osx install zstd lz4 snappy xz (#3893) Summary: test osx against the brew libraries zstd, lz4, snappy, xz. Closes https://github.com/facebook/rocksdb/pull/3893 Differential Revision: D8461988 Pulled By: siying fbshipit-source-id: cc2a8487bcb1e98ca05bddd3a509a6896258ccf8 | 15 June 2018, 23:57:30 UTC |
906a602 | Tomas Kolda | 15 June 2018, 19:39:16 UTC | Build and tests fixes for Solaris Sparc (#4000) Summary: Here are some fixes for build on Solaris Sparc. It is also fixing CRC test on BigEndian platforms. Closes https://github.com/facebook/rocksdb/pull/4000 Differential Revision: D8455394 Pulled By: ajkr fbshipit-source-id: c9289a7b541a5628139c6b77e84368e14dc3d174 | 15 June 2018, 19:42:53 UTC |
f23fed1 | 奏之章 | 15 June 2018, 19:28:06 UTC | Delay verify compaction output table (#3979) Summary: Verify table will load SST into `TableCache` it occupy memory & `TableCache`‘s capacity ... but no logic use them it's unnecessary ... so , we verify them after all sub compact finished Closes https://github.com/facebook/rocksdb/pull/3979 Differential Revision: D8389946 Pulled By: ajkr fbshipit-source-id: 54bd4f474f9e7b3accf39c3068b1f36a27ec4c49 | 15 June 2018, 19:42:53 UTC |
4faaab7 | Hans-Wilhelm Warlo | 15 June 2018, 19:07:38 UTC | Benchmark sine wave write rate limit (#3914) Summary: As mentioned at the [dev forum.](https://www.facebook.com/groups/rocksdb.dev/1693425187422655/) Let me know if you would like me to do any changes! Closes https://github.com/facebook/rocksdb/pull/3914 Differential Revision: D8452824 Pulled By: siying fbshipit-source-id: 56439b3228ecdcc5a199d5198eff2fab553be961 | 15 June 2018, 19:12:03 UTC |
f5281a5 | Siying Dong | 15 June 2018, 17:59:40 UTC | tools/check_format_compatible.sh to cover forward option reading too (#3994) Summary: Make sure that some recent releases can read master's option files while ignoring unknown options. Also add two more recent release branches. Closes https://github.com/facebook/rocksdb/pull/3994 Differential Revision: D8409499 Pulled By: siying fbshipit-source-id: 1b025f19ba288da0517f6b4572797573e23e23c2 | 15 June 2018, 18:12:29 UTC |
fbe3b9e | Fenggang Wu | 15 June 2018, 17:32:20 UTC | Udpate db_universal_compaction_test according to PR #3970 (#3995) Summary: The SST file sizes changed slightly after the improvement of PR #3970 which reduces the size of the properties block. Before PR #3970 a size ratio compaction included all of the first four flushed files but it only includes two files after. We increase the size_ratio universal compaction option to make that compaction include all four files again. Closes https://github.com/facebook/rocksdb/pull/3995 Differential Revision: D8426925 Pulled By: fgwu fbshipit-source-id: 1429c38672e9f4fb4d4881fd4b06db45c4861d62 | 15 June 2018, 17:42:21 UTC |
1f32dc7 | Andrew Kryczka | 14 June 2018, 00:28:31 UTC | Check with PosixEnv before opening LOCK file (#3993) Summary: Rebased and resubmitting #1831 on behalf of stevelittle. The problem is when a single process attempts to open the same DB twice, the second attempt fails due to LOCK file held. If the second attempt had opened the LOCK file, it'll now need to close it, and closing causes the file to be unlocked. Then, any subsequent attempt to open the DB will succeed, which is the wrong behavior. The solution was to track which files a process has locked in PosixEnv, and check those before opening a LOCK file. Fixes #1780. Closes https://github.com/facebook/rocksdb/pull/3993 Differential Revision: D8398984 Pulled By: ajkr fbshipit-source-id: 2755fe66950a0c9de63075f932f9e15768041918 | 14 June 2018, 00:32:04 UTC |
7497f99 | Andrew Kryczka | 13 June 2018, 23:39:03 UTC | Run manual compaction in stress/crash tests (#3936) Summary: - Add support to `db_stress` for `CompactRange` - Enable `CompactRange` and `CompactFiles` in crash tests Closes https://github.com/facebook/rocksdb/pull/3936 Differential Revision: D8230953 Pulled By: ajkr fbshipit-source-id: 208f9980b5bc8c204b1fa726e83791ad674e21e8 | 13 June 2018, 23:45:28 UTC |
dd216dd | Andrew Kryczka | 13 June 2018, 20:37:21 UTC | Choose unique keys faster in db_stress (#3990) Summary: db_stress initialization randomly chooses a set of keys to not overwrite. It was doing it separately for each column family. That caused 30+ second initialization times for the non-simple crash tests, which have 10 CFs. This PR: - reuses the same set of randomly chosen no-overwrite keys across all CFs - logs a couple more timestamps so we can more easily see initialization time Closes https://github.com/facebook/rocksdb/pull/3990 Differential Revision: D8393821 Pulled By: ajkr fbshipit-source-id: d0b263a298df607285ffdd8b0983ff6575cc6c34 | 13 June 2018, 20:43:23 UTC |
a720401 | Andrew Kryczka | 13 June 2018, 20:03:09 UTC | Avoid acquiring SyncPoint mutex when it is disabled (#3991) Summary: In `db_stress` profile the vast majority of CPU time is spent acquiring the `SyncPoint` mutex. I mistakenly assumed #3939 had fixed this mutex contention problem by disabling `SyncPoint` processing. But actually the lock was still being acquired just to check whether processing is enabled. We can avoid that overhead by using an atomic to track whether it's enabled. Closes https://github.com/facebook/rocksdb/pull/3991 Differential Revision: D8393825 Pulled By: ajkr fbshipit-source-id: 5bc4e3c722ee7304e7a9c2439998c456b05a6897 | 13 June 2018, 20:13:18 UTC |
d82f142 | Siying Dong | 12 June 2018, 23:43:44 UTC | Fix regression bug of Prev() with upper bound (#3989) Summary: A recent change pushed down the upper bound checking to child iterators. However, this causes the logic of following sequence wrong: Seek(key); if (!Valid()) SeekToLast(); Because !Valid() may be caused by upper bounds, rather than the end of the iterator. In this case SeekToLast() points to totally wrong places. This can cause wrong results, infinite loops, or segfault in some cases. This sequence is called when changing direction from forward to backward. And this by itself also implicitly happen during reseeking optimization in Prev(). Fix this bug by using SeekForPrev() rather than this sequuence, as what is already done in prefix extrator case. Closes https://github.com/facebook/rocksdb/pull/3989 Differential Revision: D8385422 Pulled By: siying fbshipit-source-id: 429e869990cfd2dc389421e0836fc496bed67bb4 | 12 June 2018, 23:57:36 UTC |
9d34733 | Andrew Kryczka | 12 June 2018, 20:38:09 UTC | Fix argument mismatch in BlockBasedTableBuilder (#3974) Summary: The sixth argument should be `key_includes_seq` bool, the seventh a `GetContext*`. We were mistakenly passing the `GetContext*` as the sixth argument and relying on the default (nullptr) for the seventh. This would make statistics inaccurate, at least. Blame: 402b7aa0 Closes https://github.com/facebook/rocksdb/pull/3974 Differential Revision: D8344907 Pulled By: ajkr fbshipit-source-id: 3ad865a0541d6d30f75dfc726352788118cfe12e | 12 June 2018, 20:57:44 UTC |
9c7da96 | shpala | 12 June 2018, 20:36:05 UTC | Fix a crash in WinEnvIO::GetSectorSize (#3975) Summary: Fix a crash in `WinEnvIO::GetSectorSize` that happens on old Windows systems (e.g Windows 7). On old Windows systems that don't support querying StorageAccessAlignmentProperty using IOCTL_STORAGE_QUERY_PROPERTY, the flow calls a different DeviceIoControl with nullptr as lpBytesReturned. When the code reaches this point, we get an access violation. Closes https://github.com/facebook/rocksdb/pull/3975 Differential Revision: D8385186 Pulled By: ajkr fbshipit-source-id: fae4c9b4b0a52c8a10182e1b35bcaa30dc393bbb | 12 June 2018, 20:45:18 UTC |
3593275 | Fenggang Wu | 12 June 2018, 19:46:48 UTC | Remove restart point from the properties_block (#3970) Summary: Property block will be read sequentially and cached in a heap located object, so there's no need for restart points. Thus we set the restart interval to infinity to save space. Closes https://github.com/facebook/rocksdb/pull/3970 Differential Revision: D8332586 Pulled By: fgwu fbshipit-source-id: 899c3267832a81d0f084ec2db6b387332f461134 | 12 June 2018, 19:57:37 UTC |
f450294 | Fenggang Wu | 08 June 2018, 19:53:23 UTC | Change db path for BlockBasedTableTest.BadOptions (#3965) Summary: BadOptions test creates a temporary db path changed to table_block_based_bad_options_test to avoid collide with that created by the PrefixAndWholeKeyTest Closes https://github.com/facebook/rocksdb/pull/3965 Differential Revision: D8316080 Pulled By: fgwu fbshipit-source-id: bb8e0fdfdb9abf0e5ce94494b4388cd1622ee032 | 08 June 2018, 19:57:14 UTC |
3470c75 | Yanqin Jin | 07 June 2018, 22:34:42 UTC | Fix build errors. Summary: Closes https://github.com/facebook/rocksdb/pull/3967 Differential Revision: D8322775 Pulled By: riversand963 fbshipit-source-id: bd73067bd5d3ed4627348f0685bc499359ad6442 | 07 June 2018, 22:43:09 UTC |
23e1d23 | Zhichao Cao | 07 June 2018, 18:34:52 UTC | Fixed the fprintf of uint64_t by using PRIu64 (#3963) Summary: Fixed the fprintf format of uint64_t by using PRIu64 in file tools/ldb_cmd.cc Closes https://github.com/facebook/rocksdb/pull/3963 Differential Revision: D8306179 Pulled By: zhichao-cao fbshipit-source-id: 597dcd55321576801bbf2cf4714736ebc4750a0c | 07 June 2018, 18:44:48 UTC |
0a0860a | Yanqin Jin | 07 June 2018, 17:34:56 UTC | Refactoring db_stress.cc (#3902) Summary: We use `db_stress.cc` intensively to test and verify the behavior of RocksDB. Sometimes we need to add new tests for recently added features. Original `StressTest` class provides many general functionality that can be leveraged by other tests. Therefore, in this refactoring PR, I try to identify the general operations as well as operations that future tests most likely want to customize. Future tests can inherit `StressTest` and overriding the virtual functions to test custom logic. Closes https://github.com/facebook/rocksdb/pull/3902 Differential Revision: D8284607 Pulled By: riversand963 fbshipit-source-id: 019302d04665a2b18334b6d05d04a477168c8ea4 | 07 June 2018, 17:43:00 UTC |
45b6bcc | Zhongyi Xie | 07 June 2018, 06:30:26 UTC | ZSTD compression: should also expect type = kZSTDNotFinalCompression (#3964) Summary: Depending on the compression type, `CompressBlock` calls the compress method for each compression type. It calls ZSTD_Compress for both kZSTD and kZSTDNotFinalCompression (https://github.com/facebook/rocksdb/blob/master/table/block_based_table_builder.cc#L169). However currently ZSTD_Compress only expects the type to be kZSTD and this is causing assert failures and crashes. The same also applies to ZSTD_Uncompress. Closes https://github.com/facebook/rocksdb/pull/3964 Differential Revision: D8308715 Pulled By: miasantreble fbshipit-source-id: e5125f53edb829c9c33733167bec74e4793d0782 | 07 June 2018, 06:42:29 UTC |
b736521 | Maysam Yabandeh | 06 June 2018, 23:44:52 UTC | Extend format 3 to partitioned index/filters (#3958) Summary: format_version 3 changes the format of index blocks by storing user keys instead of the internal keys, which saves 8-bytes per key. This patch extends the format to top-level indexes in partitioned index/filters. Closes https://github.com/facebook/rocksdb/pull/3958 Differential Revision: D8294615 Pulled By: maysamyabandeh fbshipit-source-id: 17666cc16b8076c363972e2308e31547e835f0fe | 06 June 2018, 23:58:16 UTC |
5504a05 | Pooja Malik | 06 June 2018, 21:28:51 UTC | Adding advisor Rules and parser scripts with unit tests. (#3934) Summary: This adds some rules in the tools/advisor/advisor/rules.ini (refer this for more information) file and corresponding python parser scripts for parsing the rules file and the rocksdb LOG and OPTIONS files. This is WIP for adding rules depending on ODS. The starting point of the script is the rocksdb/tools/advisor/advisor/rule_parser.py file. Closes https://github.com/facebook/rocksdb/pull/3934 Reviewed By: maysamyabandeh Differential Revision: D8304059 Pulled By: poojam23 fbshipit-source-id: 47f2a50f04d46d40e225dd1cbf58ba490f79e239 | 06 June 2018, 21:42:59 UTC |
4420df4 | Andrew Kryczka | 05 June 2018, 21:03:42 UTC | Check conflict at output level in CompactFiles (#3926) Summary: CompactFiles checked whether the existing files conflicted with the chosen compaction. But it missed checking whether future files would conflict, i.e., when another compaction was simultaneously writing new files to the same range at the same output level. Closes https://github.com/facebook/rocksdb/pull/3926 Differential Revision: D8218996 Pulled By: ajkr fbshipit-source-id: 21cb00a6fed4c8c62d3ed2ff810962e6bdc2fdfb | 05 June 2018, 21:14:05 UTC |
f1592a0 | Zhongyi Xie | 05 June 2018, 19:51:05 UTC | run make format for PR 3838 (#3954) Summary: PR https://github.com/facebook/rocksdb/pull/3838 made some changes that triggers lint warnings. Run `make format` to fix formatting as suggested by siying . Also piggyback two changes: 1) fix singleton destruction order for windows and posix env 2) fix two clang warnings Closes https://github.com/facebook/rocksdb/pull/3954 Differential Revision: D8272041 Pulled By: miasantreble fbshipit-source-id: 7c4fd12bd17aac13534520de0c733328aa3c6c9f | 05 June 2018, 19:58:02 UTC |
812c737 | Mike Kolupaev | 05 June 2018, 18:32:07 UTC | Fix performance regression in Get() for block-based tables (#3953) Summary: This fixes a regression in one of myrocks regression tests (readwhilewriting), introduced in https://github.com/facebook/rocksdb/commit/8bf555f487d1de84a4fb19cb97b9ae1a8dbebc60 This PR changes two lines of code: one of them actually fixes the observed regression, the other is a mostly unrelated small fix that I'm piggy-backing here. EDIT: Nevermind, it fixes one line. More details in inline comments. Closes https://github.com/facebook/rocksdb/pull/3953 Differential Revision: D8270664 Pulled By: al13n321 fbshipit-source-id: a7d91e196807d1e816551591257c700f70e4ccac | 05 June 2018, 18:43:16 UTC |
d0c38c0 | Maysam Yabandeh | 05 June 2018, 02:59:44 UTC | Extend some tests to format_version=3 (#3942) Summary: format_version=3 changes the format of SST index. This is however not being tested currently since tests only work with the default format_version which is currently 2. The patch extends the most related tests to also test for format_version=3. Closes https://github.com/facebook/rocksdb/pull/3942 Differential Revision: D8238413 Pulled By: maysamyabandeh fbshipit-source-id: 915725f55753dd8e9188e802bf471c23645ad035 | 05 June 2018, 03:13:00 UTC |
2210152 | Andrew Kryczka | 04 June 2018, 22:52:31 UTC | Fix singleton destruction order of PosixEnv and SyncPoint (#3951) Summary: Ensure the PosixEnv singleton is destroyed first since its destructor waits for background threads to all complete. This ensures background threads cannot hit sync points after the SyncPoint singleton is destroyed, which was previously possible. Closes https://github.com/facebook/rocksdb/pull/3951 Differential Revision: D8265295 Pulled By: ajkr fbshipit-source-id: 7738dd458c5d993a78377dd0420e82badada81ab | 04 June 2018, 22:58:46 UTC |