swh:1:snp:5115096b921df712aeb2a08114fede57fb3331fb
- HEAD
- refs/heads/2.2.fb.branch
- refs/heads/2.3.fb.branch
- refs/heads/2.4.fb.branch
- refs/heads/2.5.fb.branch
- refs/heads/2.6.fb.branch
- refs/heads/2.7
- refs/heads/2.7.fb.branch
- refs/heads/2.8.1.fb
- refs/heads/2.8.fb
- refs/heads/2.8.fb.trunk
- refs/heads/3.0.fb
- refs/heads/3.0.fb.branch
- refs/heads/3.1.fb
- refs/heads/3.10.fb
- refs/heads/3.11.fb
- refs/heads/3.12.fb
- refs/heads/3.13.fb
- refs/heads/3.2.fb
- refs/heads/3.3.fb
- refs/heads/3.4.fb
- refs/heads/3.5.fb
- refs/heads/3.6.fb
- refs/heads/3.7.fb
- refs/heads/3.8.fb
- refs/heads/3.9.fb
- refs/heads/4.0.fb
- refs/heads/4.1.fb
- refs/heads/4.10.fb
- refs/heads/4.11.fb
- refs/heads/4.12.fb
- refs/heads/4.13.fb
- refs/heads/4.2.fb
- refs/heads/4.3.fb
- refs/heads/4.4.fb
- refs/heads/4.5.fb
- refs/heads/4.6.fb
- refs/heads/4.7.fb
- refs/heads/4.8.fb
- refs/heads/4.9.fb
- refs/heads/5.0.fb
- refs/heads/5.1.fb
- refs/heads/5.10.fb
- refs/heads/5.11.fb
- refs/heads/5.12.fb
- refs/heads/5.13.fb
- refs/heads/5.13.fb.myrocks
- refs/heads/5.14.fb
- refs/heads/5.14.fb.myrocks
- refs/heads/5.15.fb
- refs/heads/5.16.fb
- refs/heads/5.17.fb
- refs/heads/5.17.fb.myrocks
- refs/heads/5.18.fb
- refs/heads/5.2.fb
- refs/heads/5.3.fb
- refs/heads/5.4.fb
- refs/heads/5.5.fb
- refs/heads/5.6.fb
- refs/heads/5.7.fb
- refs/heads/5.7.fb.myrocks
- refs/heads/5.8.3
- refs/heads/5.8.fb
- refs/heads/5.9.fb
- refs/heads/5.9.fb.myrocks
- refs/heads/6.0.fb
- refs/heads/6.0.fb.myrocks
- refs/heads/6.1.fb
- refs/heads/6.1.fb.myrocks
- refs/heads/6.1.fb.prod201905
- refs/heads/6.10.fb
- refs/heads/6.11.fb
- refs/heads/6.12.fb
- refs/heads/6.13.fb
- refs/heads/6.13.fb.laser
- refs/heads/6.14.fb
- refs/heads/6.14.fb.laser
- refs/heads/6.15.fb
- refs/heads/6.16.fb
- refs/heads/6.17.fb
- refs/heads/6.2.fb
- refs/heads/6.3.fb
- refs/heads/6.3.fb.myrocks
- refs/heads/6.3.fb.myrocks2
- refs/heads/6.3fb
- refs/heads/6.4.fb
- refs/heads/6.5.fb
- refs/heads/6.6.fb
- refs/heads/6.7.fb
- refs/heads/6.8.fb
- refs/heads/6.9.fb
- refs/heads/adaptive
- refs/heads/ajkr-patch-1
- refs/heads/blob_shadow
- refs/heads/bottom-pri-level
- refs/heads/bugfix-build-detect
- refs/heads/checksum_readahead_mmap_fix
- refs/heads/fb-config
- refs/heads/feature/debug-rocksdbjavastatic
- refs/heads/feature/travis-arm64
- refs/heads/fix-release-notes
- refs/heads/fix-write-batch-comment
- refs/heads/format_compatible_4
- refs/heads/getmergeops
- refs/heads/gh-pages-old
- refs/heads/history-update
- refs/heads/hotfix/lambda-capture
- refs/heads/improve-support
- refs/heads/jay-stress
- refs/heads/katherinez-patch-1
- refs/heads/katherinez-patch-2
- refs/heads/master
- refs/heads/pr-sanity-check-as-GHAction
- refs/heads/pr/6062
- refs/heads/ramvadiv-patch-1
- refs/heads/scaffold
- refs/heads/siying-patch-1
- refs/heads/siying-patch-10
- refs/heads/siying-patch-2
- refs/heads/siying-patch-3
- refs/heads/siying-patch-4
- refs/heads/skip_memtable_flush
- refs/heads/testing_ppc_build
- refs/heads/tests
- refs/heads/unused-var
- refs/heads/v6.6.4
- refs/heads/yiwu_stackable
- refs/heads/yuslepukhin
- refs/remotes/origin/5.13.fb
- refs/tags/2.5.fb
- refs/tags/2.6.fb
- refs/tags/3.0.fb
- refs/tags/do-not-use-me2
- refs/tags/rocksdb-3.1
- refs/tags/rocksdb-3.10.2
- refs/tags/rocksdb-3.11
- refs/tags/rocksdb-3.11.1
- refs/tags/rocksdb-3.11.2
- refs/tags/rocksdb-3.2
- refs/tags/rocksdb-3.3
- refs/tags/rocksdb-3.4
- refs/tags/rocksdb-3.5
- refs/tags/rocksdb-3.5.1
- refs/tags/rocksdb-3.6.1
- refs/tags/rocksdb-3.6.2
- refs/tags/rocksdb-3.7
- refs/tags/rocksdb-3.8
- refs/tags/rocksdb-3.9
- refs/tags/rocksdb-3.9.1
- refs/tags/rocksdb-4.1
- refs/tags/rocksdb-5.10.2
- refs/tags/rocksdb-5.10.3
- refs/tags/rocksdb-5.10.4
- refs/tags/rocksdb-5.11.2
- refs/tags/rocksdb-5.11.3
- refs/tags/rocksdb-5.14.3
- refs/tags/rocksdb-5.2.1
- refs/tags/rocksdb-5.3.3
- refs/tags/rocksdb-5.3.4
- refs/tags/rocksdb-5.3.5
- refs/tags/rocksdb-5.3.6
- refs/tags/rocksdb-5.4.10
- refs/tags/rocksdb-5.4.5
- refs/tags/rocksdb-5.4.6
- refs/tags/rocksdb-5.5.2
- refs/tags/rocksdb-5.5.3
- refs/tags/rocksdb-5.5.4
- refs/tags/rocksdb-5.5.5
- refs/tags/rocksdb-5.5.6
- refs/tags/rocksdb-5.6.1
- refs/tags/rocksdb-5.6.2
- refs/tags/rocksdb-5.7.1
- refs/tags/rocksdb-5.7.2
- refs/tags/rocksdb-5.7.3
- refs/tags/rocksdb-5.7.5
- refs/tags/rocksdb-5.8.6
- refs/tags/rocksdb-5.8.7
- refs/tags/rocksdb-5.8.8
- refs/tags/rocksdb-5.9.2
- refs/tags/v4.0
- refs/tags/v4.1
- refs/tags/v5.10.2
- refs/tags/v5.10.3
- refs/tags/v5.10.4
- refs/tags/v5.11.2
- refs/tags/v5.11.3
- refs/tags/v5.13.3
- refs/tags/v5.14.3
- refs/tags/v5.15.10
- refs/tags/v5.18.3
- refs/tags/v5.2.1
- refs/tags/v5.3.3
- refs/tags/v5.3.4
- refs/tags/v5.3.5
- refs/tags/v5.3.6
- refs/tags/v5.4.10
- refs/tags/v5.4.5
- refs/tags/v5.4.6
- refs/tags/v5.5.2
- refs/tags/v5.5.3
- refs/tags/v5.5.4
- refs/tags/v5.5.5
- refs/tags/v5.5.6
- refs/tags/v5.6.1
- refs/tags/v5.6.2
- refs/tags/v5.7.1
- refs/tags/v5.7.2
- refs/tags/v5.7.3
- refs/tags/v5.7.5
- refs/tags/v5.8.6
- refs/tags/v5.8.7
- refs/tags/v5.8.8
- refs/tags/v5.9.2
- refs/tags/v6.0.1
- refs/tags/v6.0.2
- refs/tags/v6.1.1
- refs/tags/v6.1.2
- refs/tags/v6.10.1
- refs/tags/v6.10.2
- refs/tags/v6.11.4
- refs/tags/v6.11.6
- refs/tags/v6.12.6
- refs/tags/v6.12.7
- refs/tags/v6.13.2
- refs/tags/v6.13.3
- refs/tags/v6.14.5
- refs/tags/v6.14.6
- refs/tags/v6.15.4
- refs/tags/v6.2.2
- refs/tags/v6.2.4
- refs/tags/v6.3.6
- refs/tags/v6.4.6
- refs/tags/v6.5.2
- refs/tags/v6.5.3
- refs/tags/v6.6.3
- refs/tags/v6.6.4
- refs/tags/v6.7.3
- refs/tags/v6.8.1
- v6.15.2
- v5.8
- v5.5.1
- v5.4.7
- v5.18.4
- v5.17.2
- v5.16.6
- v5.14.2
- v5.13.4
- v5.13.2
- v5.13.1
- v5.12.5
- v5.12.4
- v5.12.3
- v5.12.2
- v5.1.4
- v5.1.3
- v5.1.2
- v5.0.2
- v5.0.1
- v4.9
- v4.8
- v4.6.1
- v4.5.1
- v4.4.1
- v4.4
- v4.3.1
- v4.3
- v4.2
- v4.13.5
- v4.13
- v4.11.2
- v3.9
- v3.8
- v3.7
- v3.6.1
- v3.5
- v3.4
- v3.3
- v3.2
- v3.13.1
- v3.13
- v3.12.1
- v3.12
- v3.11
- v3.10
- v3.1
- v3.0
- v2.8
- v2.7
- v2.6
- v2.5
- v2.4
- v2.3
- v2.2
- v2.1
- v2.0
- v1.5.9.1
- v1.5.8.2
- v1.5.8.1
- v1.5.8
- v1.5.7
- rocksdb-5.8
- rocksdb-5.4.7
- rocksdb-5.1.4
- rocksdb-5.1.3
- rocksdb-5.1.2
- rocksdb-5.0.2
- rocksdb-5.0.1
- rocksdb-4.9
- rocksdb-4.8
- rocksdb-4.6.1
- rocksdb-4.5.1
- rocksdb-4.4.1
- rocksdb-4.4
- rocksdb-4.3.1
- rocksdb-4.3
- rocksdb-4.2
- rocksdb-4.13.5
- rocksdb-4.13
- rocksdb-4.11.2
- rocksdb-3.10.1
- 2.8.fb
- 2.7.fb
- 2.4.fb
- 2.3.fb
- 2.2.fb
- 2.1.fb
- 2.0.fb
- 1.5.9.fb
- 1.5.9.2.fb
- 1.5.9.1.fb
- 1.5.8.fb
- 1.5.8.2.fb
- 1.5.8.1.fb
- 1.5.7.fb
Permalinks
To reference or cite the objects present in the Software Heritage archive, permalinks based on SoftWare Hash IDentifiers (SWHIDs) must be used.
Select below a type of object currently browsed in order to display its associated SWHID and permalink.
Revision | Author | Date | Message | Commit Date |
---|---|---|---|---|
cda4242 | Adam Retter | 08 March 2018, 19:16:46 UTC | Java wrapper for Native Comparators Summary: This is an abstraction for working with custom Comparators implemented in native C++ code from Java. Native code must directly extend `rocksdb::Comparator`. When the native code comparator is compiled into the RocksDB codebase, you can then create a Java Class, and JNI stub to wrap it. Useful if the C++/JNI barrier overhead is too much for your applications comparator performance. An example is provided in `java/rocksjni/native_comparator_wrapper_test.cc` and `java/src/main/java/org/rocksdb/NativeComparatorWrapperTest.java`. Closes https://github.com/facebook/rocksdb/pull/3334 Differential Revision: D7172605 Pulled By: miasantreble fbshipit-source-id: e24b7eb267a3bcb6afa214e0379a1d5e8a2ceabe | 08 March 2018, 22:53:40 UTC |
cfbba44 | Pooya Shareghi | 06 March 2018, 18:20:52 UTC | Added bytes XOR merge operator Summary: Closes https://github.com/facebook/rocksdb/pull/575 I fixed the merge conflicts etc. Closes https://github.com/facebook/rocksdb/pull/3065 Differential Revision: D7128233 Pulled By: sagar0 fbshipit-source-id: 2c23a48c9f0432c290b0cd16a12fb691bb37820c | 06 March 2018, 19:16:28 UTC |
1983e30 | Adam Retter | 02 March 2018, 23:33:08 UTC | Brings the Java API for WriteBatch inline with the C++ API Summary: * Exposes status * Corrects some method naming * Adds missing functionality Closes https://github.com/facebook/rocksdb/pull/3550 Differential Revision: D7140790 Pulled By: sagar0 fbshipit-source-id: cbdab6c5a7ae4f3030fb46739e9060e381b26fa6 | 05 March 2018, 20:15:20 UTC |
4c2bda9 | Adam Retter | 02 March 2018, 18:22:38 UTC | Add TransactionDB and OptimisticTransactionDB to the Java API Summary: Closes https://github.com/facebook/rocksdb/issues/697 Closes https://github.com/facebook/rocksdb/issues/1151 Closes https://github.com/facebook/rocksdb/pull/1298 Differential Revision: D7131402 Pulled By: sagar0 fbshipit-source-id: bcd34ce95ed88cc641786089ff4232df7b2f089f | 05 March 2018, 20:14:09 UTC |
67417da | Andrew Kryczka | 25 February 2018, 06:15:28 UTC | update history and bump version | 25 February 2018, 06:15:28 UTC |
04cf836 | Andrew Kryczka | 01 February 2018, 17:36:01 UTC | fix ReadaheadRandomAccessFile/iterator prefetch bug Summary: `ReadaheadRandomAccessFile` is used by iterators for file reads in several cases, like in compaction when `compaction_readahead_size > 0` or `use_direct_io_for_flush_and_compaction == true`, or in user iterator when `ReadOptions::readahead_size > 0`. `ReadaheadRandomAccessFile` maintains an internal buffer for readahead data. It assumes that, if the buffer's length is less than `ReadaheadRandomAccessFile::readahead_size_`, which is fixed in the constructor, then EOF has been reached so it doesn't try reading further. Recently, d938226af405681c592f25310f41c0c933bcdb19 started calling `RandomAccessFile::Prefetch` with various lengths: 8KB, 16KB, etc. When the `RandomAccessFile` is a `ReadaheadRandomAccessFile`, it triggers the above condition and incorrectly determines EOF. If a block is partially in the readahead buffer and EOF is incorrectly decided, the result is a truncated data block. The problem is reproducible: ``` TEST_TMPDIR=/data/compaction_bench ./db_bench -benchmarks=fillrandom -write_buffer_size=1048576 -target_file_size_base=1048576 -block_size=18384 -use_direct_io_for_flush_and_compaction=true ... put error: Corruption: truncated block read from /data/compaction_bench/dbbench/000014.sst offset 20245, expected 10143 bytes, got 8427 ``` Closes https://github.com/facebook/rocksdb/pull/3454 Differential Revision: D6869405 Pulled By: ajkr fbshipit-source-id: 87001c299e7600a37c0dcccbd0368e0954c929cf | 25 February 2018, 06:11:11 UTC |
5c7fc55 | Andrew Kryczka | 22 February 2018, 21:58:31 UTC | update history and bump version | 22 February 2018, 23:23:23 UTC |
37a5d3d | Andrew Kryczka | 22 February 2018, 01:33:14 UTC | BackupEngine gluster-friendly file naming convention Summary: Use the rsync tempfile naming convention in our `BackupEngine`. The temp file follows the format, `.<filename>.<suffix>`, which is later renamed to `<filename>`. We fix `tmp` as the `<suffix>` as we don't need to use random bytes for now. The benefit is gluster treats this tempfile naming convention specially and applies hashing only to `<filename>`, so the file won't need to be linked or moved when it's renamed. Our gluster team suggested this will make things operationally easier. Closes https://github.com/facebook/rocksdb/pull/3463 Differential Revision: D6893333 Pulled By: ajkr fbshipit-source-id: fd7622978f4b2487fce33cde40dd3124f16bcaa8 | 22 February 2018, 23:20:26 UTC |
25ac8ca | Sagar Vemuri | 13 January 2018, 07:36:27 UTC | Revert Snappy version upgrade Summary: Java static builds are again broken, this time due Snappy version upgrade introduced in 90c1d81975a03b0f8b352ddc614fbc99c2496ddd (#3331). This is due to two reasons: 1. The new Snappy packages should now be downloaded from https://github.com/google/snappy/archive/<pkg.tar.gz> instead of https://github.com/google/snappy/releases/download/<pkg.tar.gz> which we are using now. 1. In addition to the the above URL change, Snappy changed its build from using autotools to CMake based : https://github.com/google/snappy/blame/e69d9f880677f2aa3488c80b953ec4309f0dfa2e/README.md#L65-L72 So more changes are needed if we are going to upgrade to 1.1.7. Hence reverting the version upgrade until we figure them out. Closes https://github.com/facebook/rocksdb/pull/3363 Differential Revision: D6716983 Pulled By: sagar0 fbshipit-source-id: f451a1bc5eb0bb090f4da07bc3e5ba72cf89aefa | 22 February 2018, 00:22:48 UTC |
45328ec | Sagar Vemuri | 21 February 2018, 03:05:21 UTC | Add rocksdb.iterator.internal-key property Summary: Added a new iterator property: `rocksdb.iterator.internal-key` to get the internal-key (converted to user key) at which the iterator stopped. Closes https://github.com/facebook/rocksdb/pull/3525 Differential Revision: D7033694 Pulled By: sagar0 fbshipit-source-id: d51e6c00f5e9d766c6276ef79774b81c6c5216f8 | 21 February 2018, 18:52:11 UTC |
ee6f4fa | Mike Kolupaev | 16 February 2018, 15:58:18 UTC | Fix deadlock in ColumnFamilyData::InstallSuperVersion() Summary: Deadlock: a memtable flush holds DB::mutex_ and calls ThreadLocalPtr::Scrape(), which locks ThreadLocalPtr mutex; meanwhile, a thread exit handler locks ThreadLocalPtr mutex and calls SuperVersionUnrefHandle, which tries to lock DB::mutex_. This deadlock is hit all the time on our workload. It blocks our release. In general, the problem is that ThreadLocalPtr takes an arbitrary callback and calls it while holding a lock on a global mutex. The same global mutex is (at least in some cases) locked by almost all ThreadLocalPtr methods, on any instance of ThreadLocalPtr. So, there'll be a deadlock if the callback tries to do anything to any instance of ThreadLocalPtr, or waits for another thread to do so. So, probably the only safe way to use ThreadLocalPtr callbacks is to do only do simple and lock-free things in them. This PR fixes the deadlock by making sure that local_sv_ never holds the last reference to a SuperVersion, and therefore SuperVersionUnrefHandle never has to do any nontrivial cleanup. I also searched for other uses of ThreadLocalPtr to see if they may have similar bugs. There's only one other use, in transaction_lock_mgr.cc, and it looks fine. Closes https://github.com/facebook/rocksdb/pull/3510 Reviewed By: sagar0 Differential Revision: D7005346 Pulled By: al13n321 fbshipit-source-id: 37575591b84f07a891d6659e87e784660fde815f | 21 February 2018, 18:52:03 UTC |
d33f155 | Siying Dong | 14 February 2018, 00:20:13 UTC | Direct I/O writable file should do fsync in Close() Summary: We don't do fsync() after truncate in direct I/O writeable file (in fact we don't do any fsync ever). This can cause metadata not persistent to disk after the file is generated. We call it instead. Closes https://github.com/facebook/rocksdb/pull/3500 Differential Revision: D6981482 Pulled By: siying fbshipit-source-id: 7e2b591b7e5dd1b96fc0775515b8b9e6092980ef | 21 February 2018, 18:51:56 UTC |
935e834 | Jun Wu | 01 February 2018, 22:15:28 UTC | crc32: suppress -Wimplicit-fallthrough warnings Summary: Workaround a bunch of "implicit-fallthrough" compiler errors, like: ``` util/crc32c.cc:533:7: error: this statement may fall through [-Werror=implicit-fallthrough=] crc = _mm_crc32_u64(crc, *(uint64_t*)(buf + offset)); ^ util/crc32c.cc:1016:9: note: in expansion of macro ‘CRCsinglet’ CRCsinglet(crc0, next, -2 * 8); ^~~~~~~~~~ util/crc32c.cc:1017:7: note: here case 1: ``` Closes https://github.com/facebook/rocksdb/pull/3339 Reviewed By: sagar0 Differential Revision: D6874736 Pulled By: quark-zju fbshipit-source-id: eec9f3bc135e12fca336928d01711006d5c3cb16 | 21 February 2018, 18:50:39 UTC |
20269d0 | Yi Wu | 30 January 2018, 02:44:01 UTC | Fix DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge dead lock Summary: In the test, there can be a dead lock between background flush thread and foreground main thread as following: * background flush thread: - holding db mutex, while - waiting on "DBImpl::FlushMemTableToOutputFile:BeforeInstallSV" sync point. * foreground thread: - waiting for db mutex to write "key2" Fixing by let background flush thread wait without holding db mutex. Closes https://github.com/facebook/rocksdb/pull/3436 Differential Revision: D6841334 Pulled By: yiwu-arbug fbshipit-source-id: b020768ac94e166e40953c5d09e505515a5f244d | 06 February 2018, 01:13:23 UTC |
8cffe39 | Mark Isaacson | 29 January 2018, 20:43:56 UTC | Suppress lint in old files Summary: Grandfather in super old lint issues to make a clean slate for moving forward that allows us to have stronger enforcement on new issues. Reviewed By: yiwu-arbug Differential Revision: D6821806 fbshipit-source-id: 22797d31ec58e9eb0255d3b66fedfcfcb0dc127c | 06 February 2018, 00:54:59 UTC |
df23e80 | Sagar Vemuri | 26 January 2018, 05:34:35 UTC | Improve performance of long range scans with readahead Summary: This change improves the performance of iterators doing long range scans (e.g. big/full table scans in MyRocks) by using readahead and prefetching additional data on each disk IO. This prefetching is automatically enabled on noticing more than 2 IOs for the same table file during iteration. The readahead size starts with 8KB and is exponentially increased on each additional sequential IO, up to a max of 256 KB. This helps in cutting down the number of IOs needed to complete the range scan. Constraints: - The prefetched data is stored by the OS in page cache. So this currently works only for non direct-reads use-cases i.e applications which use page cache. (Direct-I/O support will be enabled in a later PR). - This gets currently enabled only when ReadOptions.readahead_size = 0 (which is the default value). Thanks to siying for the original idea and implementation. **Benchmarks:** Data fill: ``` TEST_TMPDIR=/data/users/$USER/benchmarks/iter ./db_bench -benchmarks=fillrandom -num=1000000000 -compression_type="none" -level_compaction_dynamic_level_bytes ``` Do a long range scan: Seekrandom with large number of nexts ``` TEST_TMPDIR=/data/users/$USER/benchmarks/iter ./db_bench -benchmarks=seekrandom -duration=60 -num=1000000000 -use_existing_db -seek_nexts=10000 -statistics -histogram ``` Page cache was cleared before each experiment with the command: ``` sudo sh -c "echo 3 > /proc/sys/vm/drop_caches" ``` ``` Before: seekrandom : 34020.945 micros/op 29 ops/sec; 32.5 MB/s (1636 of 1999 found) With this change: seekrandom : 8726.912 micros/op 114 ops/sec; 126.8 MB/s (5702 of 6999 found) ``` ~3.9X performance improvement. Also verified with strace and gdb that the readahead size is increasing as expected. ``` strace -e readahead -f -T -t -p <db_bench process pid> ``` Closes https://github.com/facebook/rocksdb/pull/3282 Differential Revision: D6586477 Pulled By: sagar0 fbshipit-source-id: 8a118a0ed4594fbb7f5b1cafb242d7a4033cb58c | 05 February 2018, 23:00:23 UTC |
1fe7db5 | Yi Wu | 26 January 2018, 23:12:16 UTC | StackableDB optionally take shared ownership of the underlying DB Summary: Allow StackableDB optionally takes a shared_ptr on construction and thus hold shared ownership of the underlying DB. Closes https://github.com/facebook/rocksdb/pull/3423 Differential Revision: D6824163 Pulled By: yiwu-arbug fbshipit-source-id: dbdc30c42e007533a987ef413785e192340f03eb | 31 January 2018, 19:07:20 UTC |
7bcf290 | Sagar Vemuri | 12 January 2018, 18:53:43 UTC | Fix PowerPC dynamic java build Summary: Java build on PPC64le has been broken since a few months, due to #2716. Fixing it with the least amount of changes. (We should cleanup a little around this code when time permits). This should fix the build failures seen in http://140.211.168.68:8080/job/Rocksdb/ . Closes https://github.com/facebook/rocksdb/pull/3359 Differential Revision: D6712938 Pulled By: sagar0 fbshipit-source-id: 3046e8f072180693de2af4762934ec1ace309ca4 | 26 January 2018, 19:23:18 UTC |
22a24c0 | Adam Retter | 11 January 2018, 21:21:35 UTC | FreeBSD build support for RocksDB and RocksJava Summary: Tested on a clean FreeBSD 11.01 x64. Closes https://github.com/facebook/rocksdb/pull/1423 Closes https://github.com/facebook/rocksdb/pull/3357 Differential Revision: D6705868 Pulled By: sagar0 fbshipit-source-id: cbccbbdafd4f42922512ca03619a5d5583a425fd | 26 January 2018, 19:22:51 UTC |
c33a483 | Siying Dong | 10 January 2018, 02:12:32 UTC | Remove GCC parameter "-march=native" for ARM Summary: Most popular versions of GCC can't identify platform on ARM if "-march=native" is specified. Remove it to unblock most people. Closes https://github.com/facebook/rocksdb/pull/3346 Differential Revision: D6690544 Pulled By: siying fbshipit-source-id: bbaba9fe2645b6b37144b36ea75beeff88992b49 | 26 January 2018, 19:20:38 UTC |
880cf72 | Yi Wu | 19 January 2018, 01:32:50 UTC | Fix Flush() keep waiting after flush finish Summary: Flush() call could be waiting indefinitely if min_write_buffer_number_to_merge is used. Consider the sequence: 1. User call Flush() with flush_options.wait = true 2. The manual flush started in the background 3. New memtable become immutable because of writes. The new memtable will not trigger flush if min_write_buffer_number_to_merge is not reached. 4. The manual flush finish. Because of the new memtable created at step 3 not being flush, previous logic of WaitForFlushMemTable() keep waiting, despite the memtables it intent to flush has been flushed. Here instead of checking if there are any more memtables to flush, WaitForFlushMemTable() also check the id of the earliest memtable. If the id is larger than that of latest memtable at the time flush was initiated, it means all the memtable at the time of flush start has all been flush. Closes https://github.com/facebook/rocksdb/pull/3378 Differential Revision: D6746789 Pulled By: yiwu-arbug fbshipit-source-id: 35e698f71c7f90b06337a93e6825f4ea3b619bfa | 19 January 2018, 01:50:07 UTC |
326389b | Zhongyi Xie | 08 January 2018, 19:53:22 UTC | Update HISTORY.md for release 5.11 | 10 January 2018, 00:48:58 UTC |
677f249 | Sagar Vemuri | 10 January 2018, 00:27:01 UTC | Fix Travis build failures in CMake RocksJava Summary: Fixed RocksJava travis build failure due to a missing file in java/CMakeLists.txt. (from #3332). Closes https://github.com/facebook/rocksdb/pull/3344 Differential Revision: D6686472 Pulled By: sagar0 fbshipit-source-id: dd3281dff1342c3a7235c402890420aa56db0fe3 | 10 January 2018, 00:42:15 UTC |
6aa95f4 | Siying Dong | 09 January 2018, 22:46:30 UTC | Fix a wrong log formatting Summary: I experienced weird segfault because of this mismatch of type in log formatting. Fix it. Closes https://github.com/facebook/rocksdb/pull/3345 Differential Revision: D6687224 Pulled By: siying fbshipit-source-id: c51fb1c008b7ebc3efdc353a4adad3e8f5b3e9de | 09 January 2018, 22:58:33 UTC |
0f0d2ab | Andrew Kryczka | 09 January 2018, 20:51:10 UTC | fix DBImpl instance variable naming Summary: got confused while reading `FindObsoleteFiles` due to thinking it's a local variable, so renamed it properly Closes https://github.com/facebook/rocksdb/pull/3342 Differential Revision: D6684797 Pulled By: ajkr fbshipit-source-id: a4df0aae1cccce99d4dd4d164aadc85b17707132 | 09 January 2018, 20:56:58 UTC |
46ec524 | Yi Wu | 09 January 2018, 18:53:35 UTC | Fix db_bench write being disabled in lite build Summary: The macro was added by mistake in #2372 Closes https://github.com/facebook/rocksdb/pull/3343 Differential Revision: D6681356 Pulled By: yiwu-arbug fbshipit-source-id: 4180172fb0eaef4189c07f219241e0c261c03461 | 09 January 2018, 18:57:29 UTC |
00b33c2 | Maysam Yabandeh | 09 January 2018, 16:47:46 UTC | WritePrepared Txn: address some pending TODOs Summary: This patch addresses a couple of minor TODOs for WritePrepared Txn such as double checking some assert statements at runtime as well, skip extra AddPrepared in non-2pc transactions, and safety check for infinite loops. Closes https://github.com/facebook/rocksdb/pull/3302 Differential Revision: D6617002 Pulled By: maysamyabandeh fbshipit-source-id: ef6673c139cb49f64c0879508d2f573b78609aca | 09 January 2018, 16:57:20 UTC |
24e2c16 | Chris Lu | 09 January 2018, 01:16:22 UTC | add support for allow_ingest_behind in C API Summary: https://github.com/facebook/rocksdb/wiki/Creating-and-Ingesting-SST-files Need to expose these functions in the C API to be used by Go bindings. Closes https://github.com/facebook/rocksdb/pull/3011 Differential Revision: D6679563 Pulled By: sagar0 fbshipit-source-id: 536f844ddaeb0172c6d7e416d2a75e8f9e57c8ef | 09 January 2018, 01:26:31 UTC |
398d72f | Adam Retter | 08 January 2018, 20:20:48 UTC | Add autotune and #getBytesPerSecond() to RocksJava RateLimiter Summary: Closes https://github.com/facebook/rocksdb/pull/3332 Differential Revision: D6667680 Pulled By: ajkr fbshipit-source-id: b2bb6889257850a4eb6f6cbd7106f62df7b82730 | 08 January 2018, 20:30:52 UTC |
30a017f | Yi Wu | 06 January 2018, 00:35:49 UTC | Blob DB: avoid having a separate read of checksum Summary: Previously on a blob db read, we are making a read of the blob value, and then make another read to get CRC checksum. I'm combining the two read into one. readrandom db_bench with 1G database with base db size of 13M, value size 1k: `./db_bench --db=/home/yiwu/tmp/db_bench --use_blob_db --value_size=1024 --num=1000000 --benchmarks=readrandom --use_existing_db --cache_size=32000000` master: throughput 234MB/s, get micros p50 5.984 p95 9.998 p99 20.817 p100 787 this PR: throughput 261MB/s, get micros p50 5.157 p95 9.928 p99 20.724 p100 190 Closes https://github.com/facebook/rocksdb/pull/3301 Differential Revision: D6615950 Pulled By: yiwu-arbug fbshipit-source-id: 052410c6d8539ec0cc305d53793bbc8f3616baa3 | 06 January 2018, 00:41:58 UTC |
3e955fa | Sagar Vemuri | 05 January 2018, 23:30:12 UTC | Fix zstd/zdict include path for java static build Summary: With the ZSTD dictionary generator support added in #3057 `PORTABLE=1 ROCKSDB_NO_FBCODE=1 make rocksdbjavastatic` fails as it can't find zdict.h. Specifically due to: https://github.com/facebook/rocksdb/blob/e3a06f12d27fd50af7b6c5941973f529601f9a3e/util/compression.h#L39 In java static builds zstd code gets directly downloaded from https://github.com/facebook/zstd , and in there zdict.h is under dictBuilder directory. So, I modified libzstd.a target to use `make install` to collect all the header files into a single location and used that as the zstd's include path. Closes https://github.com/facebook/rocksdb/pull/3260 Differential Revision: D6669850 Pulled By: sagar0 fbshipit-source-id: f8a7562a670e5aed4c4fb6034a921697590d7285 | 05 January 2018, 23:41:46 UTC |
84ddbd1 | Dmitri Smirnov | 05 January 2018, 22:52:57 UTC | Make Windows dep switches compatible with other builds Summary: Make dependacies switches compatible with other OS builds TODO: Make find_package work for Windows. Closes https://github.com/facebook/rocksdb/pull/3322 Differential Revision: D6667637 Pulled By: sagar0 fbshipit-source-id: 5afcd7bbfe69465310a4fbc8e589f01e506b95f5 | 05 January 2018, 22:56:54 UTC |
90c1d81 | Adam Retter | 05 January 2018, 19:59:01 UTC | Update javastatic dependencies Summary: 1. Snappy 1.1.7 2. LZ4 1.8.0 3. ZSTD 1.3.3 Closes https://github.com/facebook/rocksdb/pull/3331 Differential Revision: D6667933 Pulled By: ajkr fbshipit-source-id: 21c526609df7580481195a389d31f733e2695e65 | 05 January 2018, 20:11:44 UTC |
1c9ada5 | Maysam Yabandeh | 04 January 2018, 19:10:03 UTC | Remove assert(s.ok()) from ::DeleteFile Summary: DestroyDB that is used in tests loops over the files returned by ::GetChildren and delete them one by one. Such files might be already deleted in the file system (during DeleteObsoleteFileImpl for example) but will get actually deleted with a delay sometimes before ::DeleteFile is called on the file name. We have some test failures where FaultInjectionTestEnv::DeleteFile fails on assert(s.ok()) during DestroyDB. This patch removes the assert statement to fix that. Closes https://github.com/facebook/rocksdb/pull/3324 Differential Revision: D6659545 Pulled By: maysamyabandeh fbshipit-source-id: 4c9552fbcd494dcf3e61d475c11fc965c4388b2c | 04 January 2018, 19:11:45 UTC |
ea8ccd2 | Andrew Kryczka | 03 January 2018, 20:37:07 UTC | fix powerpc java static build Summary: added support for C and asm files as required for e612e317409e8a9d74cf05db0bd733403305f459. Closes https://github.com/facebook/rocksdb/pull/3299 Differential Revision: D6612479 Pulled By: ajkr fbshipit-source-id: 6263ed7c1602f249460421825c76b5721f396163 | 03 January 2018, 20:41:37 UTC |
00e5e1e | Adam Retter | 03 January 2018, 20:22:40 UTC | Fixes the build on Windows Summary: As discovered during v5.9.2 release, and forward-ported. Closes https://github.com/facebook/rocksdb/pull/3323 Differential Revision: D6657209 Pulled By: siying fbshipit-source-id: b560d9f8ddb89e0ffaff7c895ec80f68ddf7dab4 | 03 January 2018, 20:27:12 UTC |
ccc095a | Siying Dong | 02 January 2018, 18:34:51 UTC | Speed up BlockTest.BlockReadAmpBitmap Summary: BlockTest.BlockReadAmpBitmap is too slow and times out in some environments. Speed it up by: (1) improve the way the verification is done. With this it is 5 times faster (2) run fewer tests for large blocks. This cut it down by another 10 times. Now it can finish in similar time as other tests. Closes https://github.com/facebook/rocksdb/pull/3313 Differential Revision: D6643711 Pulled By: siying fbshipit-source-id: c2397d666eab5421a78ca87e1e45491e0f832a6d | 02 January 2018, 18:41:28 UTC |
b5c99cc | burtonli | 22 December 2017, 02:37:27 UTC | Disable onboard cache for compaction output Summary: FILE_FLAG_WRITE_THROUGH is for disabling device on-board cache in windows API, which should be disabled if user doesn't need system cache. There was a perf issue related with this, we found during memtable flush, the high percentile latency jumps significantly. During profiling, we found those high latency (P99.9) read requests got queue-jumped by write requests from memtable flush and takes 80ms or even more time to wait, even when SSD overall IO throughput is relatively low. After enabling FILE_FLAG_WRITE_THROUGH, we rerun the test found high percentile latency drops a lot without observable impact on writes. Scenario 1: 40MB/s + 40MB/s R/W compaction throughput  Original | FILE_FLAG_WRITE_THROUGH | Percentage reduction --------------------------------------------------------------- P99.9 | 56.897 ms | 35.593 ms | -37.4% P99 | 3.905 ms | 3.896 ms | -2.8% Scenario 2: 14MB/s + 14MB/s R/W compaction throughput, cohosted with 100+ other rocksdb instances have manually triggered memtable flush operations (memtable is tiny), creating a lot of randomized the small file writes operations during test. Original | FILE_FLAG_WRITE_THROUGH | Percentage reduction --------------------------------------------------------------- P99.9 | 86.227 ms | 50.436 ms | -41.5% P99 | 8.415 ms | 3.356 ms | -60.1% Closes https://github.com/facebook/rocksdb/pull/3225 Differential Revision: D6624174 Pulled By: miasantreble fbshipit-source-id: 321b86aee9d74470840c70e5d0d4fa9880660a91 | 22 December 2017, 02:41:34 UTC |
f00e176 | Andrew Kryczka | 21 December 2017, 00:10:03 UTC | fix ForwardIterator reference to temporary object Summary: Fixes the following ASAN error: ``` ==2108042==ERROR: AddressSanitizer: stack-use-after-scope on address 0x7fc50ae9b868 at pc 0x7fc5112aff55 bp 0x7fff9eb9dc10 sp 0x7fff9eb9dc08 === How to use this, how to get the raw stack trace, and more: fburl.com/ASAN === READ of size 8 at 0x7fc50ae9b868 thread T0 SCARINESS: 23 (8-byte-read-stack-use-after-scope) #0 rocksdb/dbformat.h:164 rocksdb::InternalKeyComparator::user_comparator() const #1 librocksdb_src_rocksdb_lib.so+0x1429a7d rocksdb::RangeDelAggregator::InitRep(std::vector<...> const&) #2 librocksdb_src_rocksdb_lib.so+0x142ceae rocksdb::RangeDelAggregator::AddTombstones(std::unique_ptr<...>) #3 librocksdb_src_rocksdb_lib.so+0x1382d88 rocksdb::ForwardIterator::RebuildIterators(bool) #4 librocksdb_src_rocksdb_lib.so+0x1382362 rocksdb::ForwardIterator::ForwardIterator(rocksdb::DBImpl*, rocksdb::ReadOptions const&, rocksdb::ColumnFamilyData*, rocksdb::SuperVersion*) #5 librocksdb_src_rocksdb_lib.so+0x11f433f rocksdb::DBImpl::NewIterator(rocksdb::ReadOptions const&, rocksdb::ColumnFamilyHandle*) #6 rocksdb/src/include/rocksdb/db.h:382 rocksdb::DB::NewIterator(rocksdb::ReadOptions const&) #7 rocksdb/db_range_del_test.cc:807 rocksdb::DBRangeDelTest_TailingIteratorRangeTombstoneUnsupported_Test::TestBody() #18 rocksdb/db_range_del_test.cc:1006 main Address 0x7fc50ae9b868 is located in stack of thread T0 at offset 104 in frame #0 librocksdb_src_rocksdb_lib.so+0x13825af rocksdb::ForwardIterator::RebuildIterators(bool) ``` Closes https://github.com/facebook/rocksdb/pull/3300 Differential Revision: D6612989 Pulled By: ajkr fbshipit-source-id: e7ea2ed914c1b80a8a29d71d92440a6bd9cbcc80 | 21 December 2017, 00:12:04 UTC |
02a2c11 | Maysam Yabandeh | 20 December 2017, 19:31:39 UTC | Blog post for WritePrepared Txn Summary: Blog post to introduce the next generation of transaction engine at RocksDB. Closes https://github.com/facebook/rocksdb/pull/3296 Differential Revision: D6612932 Pulled By: maysamyabandeh fbshipit-source-id: 5bfa91ce84e937f5e4346bbda5a4725d0a7fd131 | 20 December 2017, 19:42:15 UTC |
0ef3fdd | Maysam Yabandeh | 20 December 2017, 16:10:19 UTC | Disable need_log_sync on bg err Summary: When there is a background error PreprocessWrite returns without marking the logs synced. If we keep need_log_sync to true, it would try to sync them at the end, which would break the logic. The patch would unset need_log_sync if the logs end up not being marked for sync in PreprocessWrite. Closes https://github.com/facebook/rocksdb/pull/3293 Differential Revision: D6602347 Pulled By: maysamyabandeh fbshipit-source-id: 37ee04209e8dcfd78de891654ce50d0954abeb38 | 20 December 2017, 16:12:24 UTC |
58b841b | Wouter Beek | 20 December 2017, 16:02:53 UTC | FIXED: string buffers potentially too small to fit formatted write Summary: This fixes the following warnings when compiled with GCC7: util/transaction_test_util.cc: In static member function ‘static rocksdb::Status rocksdb::RandomTransactionInserter::DBGet(rocksdb::DB*, rocksdb::Transaction*, rocksdb::ReadOptions&, uint16_t, uint64_t, bool, uint64_t*, std::__cxx11::string*, bool*)’: util/transaction_test_util.cc:75:8: error: ‘snprintf’ output may be truncated before the last format character [-Werror=format-truncation=] Status RandomTransactionInserter::DBGet( ^~~~~~~~~~~~~~~~~~~~~~~~~ util/transaction_test_util.cc:84:11: note: ‘snprintf’ output between 5 and 6 bytes into a destination of size 5 snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1); ~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ util/transaction_test_util.cc: In static member function ‘static rocksdb::Status rocksdb::RandomTransactionInserter::Verify(rocksdb::DB*, uint16_t, uint64_t, bool, rocksdb::Random64*)’: util/transaction_test_util.cc:245:8: error: ‘snprintf’ output may be truncated before the last format character [-Werror=format-truncation=] Status RandomTransactionInserter::Verify(DB* db, uint16_t num_sets, ^~~~~~~~~~~~~~~~~~~~~~~~~ util/transaction_test_util.cc:268:13: note: ‘snprintf’ output between 5 and 6 bytes into a destination of size 5 snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1); ~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Closes https://github.com/facebook/rocksdb/pull/3295 Differential Revision: D6609411 Pulled By: maysamyabandeh fbshipit-source-id: 33f0add471056eb59db2f8bd4366e6dfbb1a187d | 20 December 2017, 16:12:22 UTC |
f54d7f5 | yingsu00 | 20 December 2017, 02:20:50 UTC | Port 3 way SSE4.2 crc32c implementation from Folly Summary: **# Summary** RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain. This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used. **# Performance Test Results** We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm. Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s. 1) ReadRandom in db_bench overall metrics PER RUN Algorithm | run | micros/op | ops/sec |Throughput (MB/s) 3-way | 1 | 4.143 | 241387 | 26.7 3-way | 2 | 3.775 | 264872 | 29.3 3-way | 3 | 4.116 | 242929 | 26.9 FastCrc32c|1 | 4.037 | 247727 | 27.4 FastCrc32c|2 | 4.648 | 215166 | 23.8 FastCrc32c|3 | 4.352 | 229799 | 25.4 AVG Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s) 3-way | 4.01 | 249,729 | 27.63 FastCrc32c | 4.35 | 230,897 | 25.53 2) Crc32c computation CPU cost (inclusive samples percentage) PER RUN Implementation | run | TotalSamples | Crc32c percentage 3-way  | 1  | 4,572,250,000 | 4.37% 3-way  | 2  | 3,779,250,000 | 4.62% 3-way  | 3  | 4,129,500,000 | 4.48% FastCrc32c   | 1  | 4,663,500,000 | 11.24% FastCrc32c   | 2  | 4,047,500,000 | 12.34% FastCrc32c   | 3  | 4,366,750,000 | 11.68% **# Test Plan** make -j64 corruption_test && ./corruption_test By default it uses 3-way SSE algorithm NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test make clean && DEBUG_LEVEL=0 make -j64 db_bench make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench Closes https://github.com/facebook/rocksdb/pull/3173 Differential Revision: D6330882 Pulled By: yingsu00 fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9 | 20 December 2017, 02:26:49 UTC |
e763e1b | Yi Wu | 20 December 2017, 00:50:59 UTC | BlobDB: dump blob db options on open Summary: We dump blob db options on blob db open, but it was removed by mistake in #3246. Adding it back. Closes https://github.com/facebook/rocksdb/pull/3298 Differential Revision: D6607177 Pulled By: yiwu-arbug fbshipit-source-id: 2a4aacbfa52fd8f1878dc9e1fbb95fe48faf80c0 | 20 December 2017, 00:57:12 UTC |
48cf8da | Yi Wu | 20 December 2017, 00:34:25 UTC | BlobDB: update blob_db_options.bytes_per_sync behavior Summary: Previously, if blob_db_options.bytes_per_sync, there is a background job to call fsync() for every bytes_per_sync bytes written to a blob file. With the change we simply pass bytes_per_sync as env_options_ to blob files so that sync_file_range() will be used instead. Closes https://github.com/facebook/rocksdb/pull/3297 Differential Revision: D6606994 Pulled By: yiwu-arbug fbshipit-source-id: 452424be52e32ba92f5ea603b564e9b88929af47 | 20 December 2017, 00:41:41 UTC |
0614942 | Yi Wu | 19 December 2017, 06:13:08 UTC | WritePrepared Txn: Return NotSupported on iterator refresh Summary: A proper implementation of Iterator::Refresh() for WritePreparedTxnDB would require release and acquire another snapshot. Since MyRocks don't make use of Iterator::Refresh(), we just simply mark it as not supported. Closes https://github.com/facebook/rocksdb/pull/3290 Differential Revision: D6599931 Pulled By: yiwu-arbug fbshipit-source-id: 4e1632d967316431424f6e458254ecf9a97567cf | 19 December 2017, 06:29:30 UTC |
1563801 | Andrew Kryczka | 19 December 2017, 01:48:49 UTC | blog post for auto-tuned rate limiter Summary: Wrote the blog post. Closes https://github.com/facebook/rocksdb/pull/3289 Differential Revision: D6599031 Pulled By: ajkr fbshipit-source-id: 77ee553196f225f20c56112d2c015b6fa14f1b83 | 19 December 2017, 01:56:50 UTC |
2190e96 | Yi Wu | 19 December 2017, 01:48:41 UTC | Remove incorrect comment Summary: We actually create individual compaction filter from compaction filter factory per sub-compaction in `CompactionJob::ProcessKeyValueCompaction`: https://github.com/facebook/rocksdb/blob/master/db/compaction_job.cc#L742 The comment seems incorrect. Closes https://github.com/facebook/rocksdb/pull/3288 Differential Revision: D6598455 Pulled By: yiwu-arbug fbshipit-source-id: a6bc059a9103b87a73ae6ec4bb01ca33f5d48cf5 | 19 December 2017, 01:56:47 UTC |
0faa026 | Maysam Yabandeh | 18 December 2017, 22:39:49 UTC | WritePrepared Txn: make buck tests parallel Summary: The TSAN version of tests could take quite long. Make the buck tests parallel to avoid timeouts. Closes https://github.com/facebook/rocksdb/pull/3280 Differential Revision: D6581594 Pulled By: maysamyabandeh fbshipit-source-id: 3f8476d8c69f0183e394fa8a2089dd8d4e90c90c | 18 December 2017, 22:42:09 UTC |
78c2eed | Maysam Yabandeh | 18 December 2017, 22:13:45 UTC | fix release order in validateNumberOfEntries Summary: ScopedArenaIterator should be defined after range_del_agg so that it destructs the assigned iterator, which depends on range_del_agg, before it range_del_agg is already destructed. Closes https://github.com/facebook/rocksdb/pull/3281 Differential Revision: D6592332 Pulled By: maysamyabandeh fbshipit-source-id: 89a15d8ed13d0fc856b0c47dce3d91778738dbac | 18 December 2017, 22:27:28 UTC |
aa6509d | Guo Xiao | 18 December 2017, 20:23:40 UTC | Fix build for linux Summary: * Include `unistd.h` for `sleep(3)` * Include `sys/time.h` for `gettimeofday(3)` * Include `utils/random.h` for `Random64` Error messages: utilities/persistent_cache/hash_table_bench.cc: In constructor ‘rocksdb::HashTableBenchmark::HashTableBenchmark(rocksdb::HashTableImpl<long unsigned int, std::__cxx11::basic_string<char> >*, size_t, size_t, size_t, size_t)’: utilities/persistent_cache/hash_table_bench.cc:76:28: error: ‘sleep’ was not declared in this scope /* sleep override */ sleep(1); ^~~~~ utilities/persistent_cache/hash_table_bench.cc:76:28: note: suggested alternative: ‘strsep’ /* sleep override */ sleep(1); ^~~~~ strsep utilities/persistent_cache/hash_table_bench.cc: In member function ‘void rocksdb::HashTableBenchmark::RunRead()’: utilities/persistent_cache/hash_table_bench.cc:107:5: error: ‘Random64’ was not declared in this scope Random64 rgen(time(nullptr)); ^~~~~~~~ utilities/persistent_cache/hash_table_bench.cc:107:5: note: suggested alternative: ‘random_r’ Random64 rgen(time(nullptr)); ^~~~~~~~ random_r utilities/persistent_cache/hash_table_bench.cc:110:18: error: ‘rgen’ was not declared in this scope size_t k = rgen.Next() % max_prepop_key; ^~~~ utilities/persistent_cache/hash_table_bench.cc: In static member function ‘static uint64_t rocksdb::HashTableBenchmark::NowInMillSec()’: utilities/persistent_cache/hash_table_bench.cc:153:5: error: ‘gettimeofday’ was not declared in this scope gettimeofday(&tv, /*tz=*/nullptr); ^~~~~~~~~~~~ make[2]: *** [CMakeFiles/hash_table_bench.dir/build.make:63: CMakeFiles/hash_table_bench.dir/utilities/persistent_cache/hash_table_bench.cc.o] Error 1 make[1]: *** [CMakeFiles/Makefile2:3346: CMakeFiles/hash_table_bench.dir/all] Error 2 make[1]: *** Waiting for unfinished jobs.... Closes https://github.com/facebook/rocksdb/pull/3283 Differential Revision: D6594850 Pulled By: ajkr fbshipit-source-id: fd83957338c210cdfd253763347aafd39476824f | 18 December 2017, 20:28:03 UTC |
a6d3c76 | Maysam Yabandeh | 18 December 2017, 16:03:18 UTC | WritePrepared Txn: non-2pc write in one round Summary: Currently non-2pc writes do the 2nd dummy write to actually commit the transaction. This was necessary to ensure that publishing the commit sequence number will be done only from one queue (the queue that does not write to memtable). This is however not necessary when we have only one write queue, which is actually the setup that would be used by non-2pc writes. This patch eliminates the 2nd write when two_write_queues are disabled by updating the commit map in the 1st write. Closes https://github.com/facebook/rocksdb/pull/3277 Differential Revision: D6575392 Pulled By: maysamyabandeh fbshipit-source-id: 8ab458f7ca506905962f9166026b2ec81e749c46 | 18 December 2017, 16:19:43 UTC |
fccc12f | Anand Ananthabhotla | 16 December 2017, 02:45:38 UTC | Add a histogram stat for memtable flush Summary: Add a new histogram stat called rocksdb.db.flush.micros for memtable flush Closes https://github.com/facebook/rocksdb/pull/3269 Differential Revision: D6559496 Pulled By: anand1976 fbshipit-source-id: f5c771ba2568630458751795e8c37a493ff9b14d | 16 December 2017, 02:57:00 UTC |
95583e1 | Maysam Yabandeh | 16 December 2017, 00:15:58 UTC | db_stress: skip snapshot check if cf is dropped Summary: We added a new verification that ensures a value that snapshot reads when is released is the same as when it was created. This test however fails when the cf is dropped in between. The patch skips the tests if that was the case. Closes https://github.com/facebook/rocksdb/pull/3279 Differential Revision: D6581584 Pulled By: maysamyabandeh fbshipit-source-id: afe37d371c0f91818d2e279b3949b810e112e8eb | 16 December 2017, 00:28:04 UTC |
237b292 | Yi Wu | 15 December 2017, 21:18:32 UTC | BlobDB: Remove the need to get sequence number per write Summary: Previously we store sequence number range of each blob files, and use the sequence number range to check if the file can be possibly visible by a snapshot. But it adds complexity to the code, since the sequence number is only available after a write. (The current implementation get sequence number by calling GetLatestSequenceNumber(), which is wrong.) With the patch, we are not storing sequence number range, and check if snapshot_sequence < obsolete_sequence to decide if the file is visible by a snapshot (previously we check if first_sequence <= snapshot_sequence < obsolete_sequence). Closes https://github.com/facebook/rocksdb/pull/3274 Differential Revision: D6571497 Pulled By: yiwu-arbug fbshipit-source-id: ca06479dc1fcd8782f6525b62b7762cd47d61909 | 15 December 2017, 21:27:30 UTC |
a79c7c0 | Andrew Kryczka | 15 December 2017, 20:22:13 UTC | fix backup meta-file buffer overrun Summary: - check most times after calling snprintf that the buffer didn't fill up. Previously we'd proceed and use `buf_size - len` as the length in subsequent calls, which underflowed as those are unsigned size_t. - replace some memcpys with snprintf for consistency Closes https://github.com/facebook/rocksdb/pull/3255 Differential Revision: D6541464 Pulled By: ajkr fbshipit-source-id: 8610ea6a24f38e0a37c6d17bc65b7c712da6d932 | 15 December 2017, 20:29:16 UTC |
5a7e084 | Andrew Kryczka | 14 December 2017, 22:41:59 UTC | fix ThreadStatus for bottom-pri compaction threads Summary: added `ThreadType::BOTTOM_PRIORITY` which is used in the `ThreadStatus` object to indicate the thread is used for bottom-pri compactions. Previously there was a bug where we mislabeled such threads as `ThreadType::LOW_PRIORITY`. Closes https://github.com/facebook/rocksdb/pull/3270 Differential Revision: D6559428 Pulled By: ajkr fbshipit-source-id: 96b1a50a9c19492b1a5fd1b77cf7061a6f9f1d1c | 14 December 2017, 22:57:49 UTC |
b4d88d7 | Orvid King | 14 December 2017, 19:57:11 UTC | Fix the build with MSVC 2017 Summary: There were a few places where MSVC's implicit truncation warnings were getting triggered, which was causing the MSVC build to fail due to warnings being treated as errors. This resolves the issues by making the truncations in some places explicit, and by making it so there are no truncations of literals. Fixes #3239 Supersedes #3259 Closes https://github.com/facebook/rocksdb/pull/3273 Reviewed By: yiwu-arbug Differential Revision: D6569204 Pulled By: Orvid fbshipit-source-id: c188cf1cf98d9acb6d94b71875041cc81f8ff088 | 14 December 2017, 20:02:22 UTC |
def6a00 | Siying Dong | 14 December 2017, 18:18:01 UTC | Print out compression type of new SST files in logging Summary: Closes https://github.com/facebook/rocksdb/pull/3264 Differential Revision: D6552768 Pulled By: siying fbshipit-source-id: 6303110aff22f341d5cff41f8d2d4f138a53652d | 14 December 2017, 18:27:43 UTC |
6b77c07 | Siying Dong | 14 December 2017, 18:17:22 UTC | NUMBER_BLOCK_COMPRESSED, etc, shouldn't be treated as timer counter Summary: NUMBER_BLOCK_DECOMPRESSED and NUMBER_BLOCK_COMPRESSED are not reported unless the stats level contain detailed timers, which is wrong. They are normal counters. Fix it. Closes https://github.com/facebook/rocksdb/pull/3263 Differential Revision: D6552519 Pulled By: siying fbshipit-source-id: 40899ccea7b2856bb39752616657c0bfd432f6f9 | 14 December 2017, 18:27:43 UTC |
cd2e5ca | Maysam Yabandeh | 13 December 2017, 19:51:12 UTC | WritePrepared Txn: make db_stress transactional Summary: Add "--use_txn" option to use transactional API in db_stress, default being WRITE_PREPARED policy, which is the main intention of modifying db_stress. It also extend the existing snapshots to verify that before releasing a snapshot a read from it returns the same value as before. Closes https://github.com/facebook/rocksdb/pull/3243 Differential Revision: D6556912 Pulled By: maysamyabandeh fbshipit-source-id: 1ae31465be362d44bd06e635e2e9e49a1da11268 | 13 December 2017, 19:57:29 UTC |
546a632 | Maysam Yabandeh | 13 December 2017, 15:55:02 UTC | disableWAL with WriteImplWALOnly Summary: Currently WriteImplWALOnly simply returns when disableWAL is set. This is an incorrect behavior since it does not allocated the sequence number, which is a side-effect of writing to the WAL. This patch fixes the issue. Closes https://github.com/facebook/rocksdb/pull/3262 Differential Revision: D6550974 Pulled By: maysamyabandeh fbshipit-source-id: 745a83ae8f04e7ca6c8ffb247d6ef16c287c52e7 | 13 December 2017, 15:57:44 UTC |
35dfbd5 | Maysam Yabandeh | 13 December 2017, 15:52:02 UTC | WritePrepared Txn: GC old_commit_map_ Summary: Garbage collect entries from old_commit_map_ when the corresponding snapshots are released. Closes https://github.com/facebook/rocksdb/pull/3247 Differential Revision: D6528478 Pulled By: maysamyabandeh fbshipit-source-id: 15d1566d85d4ac07036bc0dc47418f6c3228d4bf | 13 December 2017, 15:57:43 UTC |
51c2ea0 | Zhongyi Xie | 13 December 2017, 05:06:26 UTC | Reduce heavy hitter for Get operation Summary: This PR addresses the following heavy hitters in `Get` operation by moving calls to `StatisticsImpl::recordTick` from `BlockBasedTable` to `Version::Get` - rocksdb.block.cache.bytes.write - rocksdb.block.cache.add - rocksdb.block.cache.data.miss - rocksdb.block.cache.data.bytes.insert - rocksdb.block.cache.data.add - rocksdb.block.cache.hit - rocksdb.block.cache.data.hit - rocksdb.block.cache.bytes.read The db_bench statistics before and after the change are: |1GB block read|Children |Self |Command |Shared Object |Symbol| |---|---|---|---|---|---| |master: |4.22% |1.31% |db_bench |db_bench |[.] rocksdb::StatisticsImpl::recordTick| |updated: |0.51% |0.21% |db_bench |db_bench |[.] rocksdb::StatisticsImpl::recordTick| | |0.14% |0.14% |db_bench |db_bench |[.] rocksdb::GetContext::record_counters| |1MB block read|Children |Self |Command |Shared Object |Symbol| |---|---|---|---|---|---| |master: |3.48% |1.08% |db_bench |db_bench |[.] rocksdb::StatisticsImpl::recordTick| |updated: |0.80% |0.31% |db_bench |db_bench |[.] rocksdb::StatisticsImpl::recordTick| | |0.35% |0.35% |db_bench |db_bench |[.] rocksdb::GetContext::record_counters| Closes https://github.com/facebook/rocksdb/pull/3172 Differential Revision: D6330532 Pulled By: miasantreble fbshipit-source-id: 2b492959e00a3db29e9437ecdcc5e48ca4ec5741 | 13 December 2017, 05:11:33 UTC |
9089373 | Islam AbdelRahman | 13 December 2017, 02:15:15 UTC | Fix DeleteScheduler::MarkAsTrash() handling existing trash Summary: DeleteScheduler::MarkAsTrash() don't handle existing .trash files correctly This cause rocksdb to not being able to delete existing .trash files on restart Closes https://github.com/facebook/rocksdb/pull/3261 Differential Revision: D6548003 Pulled By: IslamAbdelRahman fbshipit-source-id: c3800639412e587a690062c63076a5a08881e0e6 | 13 December 2017, 02:17:13 UTC |
7393ef7 | Yi Wu | 12 December 2017, 20:08:52 UTC | Fix BlockFetcher ASAN error Summary: Some call sites of BlockFetcher create temporary ReadOptions and pass to BlockFetcher. The temporary object will be gone after BlockFetcher construction but BlockFetcher keep its reference, causing stack-use-after-scope. Fixing it. Closes https://github.com/facebook/rocksdb/pull/3258 Differential Revision: D6547152 Pulled By: yiwu-arbug fbshipit-source-id: 6b49e9dd46bb72307f5d8f88ea15faacff35b9bc | 12 December 2017, 20:12:38 UTC |
4bcb7fb | Souvik Banerjee | 12 December 2017, 20:07:35 UTC | Update transaction_test_util.cc Summary: Fixes a compile error on gcc 7.2.1 (-Werror=format-truncation=). Closes https://github.com/facebook/rocksdb/pull/3248 Differential Revision: D6546515 Pulled By: yiwu-arbug fbshipit-source-id: bd78cca63f2af376faceccb1838d2d4cc9208fef | 12 December 2017, 20:12:38 UTC |
e3a06f1 | Yi Wu | 12 December 2017, 19:02:07 UTC | WritePrepared Txn: fix compaction filter snapshot checks Summary: Add snapshot_checker check whenever we need to check sequence against snapshots and decide what to do with an input key. The changes are related to one of: * compaction filter * single delete * delete at bottom level * merge Closes https://github.com/facebook/rocksdb/pull/3251 Differential Revision: D6537850 Pulled By: yiwu-arbug fbshipit-source-id: 3faba40ed5e37779f4a0cb7ae78af9546659c7f2 | 12 December 2017, 19:12:24 UTC |
a9c8d4e | Siying Dong | 12 December 2017, 02:14:12 UTC | Fix memory issue introduced by 2f1a3a4d748ea92c282a1302b1523adc6d67ce81 Summary: Closes https://github.com/facebook/rocksdb/pull/3256 Differential Revision: D6541714 Pulled By: siying fbshipit-source-id: 40efd89b68587a9d58cfe6f4eebd771c2d9f1542 | 12 December 2017, 02:27:28 UTC |
bb5ed4b | Zhongyi Xie | 12 December 2017, 00:14:05 UTC | exclude DynamicUniversalCompactionOptions from ROCKSDB_LITE Summary: since [SetOptions](https://github.com/facebook/rocksdb/blob/master/db/db_impl.cc#L494) is not supported in ROCKSDB_LITE Right now unit test under lite is broken Closes https://github.com/facebook/rocksdb/pull/3253 Differential Revision: D6539428 Pulled By: miasantreble fbshipit-source-id: 13172b8ecbd75682330726498ea198969bc3e637 | 12 December 2017, 00:28:20 UTC |
0d5692e | Siying Dong | 11 December 2017, 23:34:51 UTC | Switch version to 5.10 Summary: Closes https://github.com/facebook/rocksdb/pull/3252 Differential Revision: D6539373 Pulled By: siying fbshipit-source-id: ce7c3d3fe625852179055295da9cf7bc80755025 | 11 December 2017, 23:42:01 UTC |
2f1a3a4 | Siying Dong | 11 December 2017, 23:16:37 UTC | Refactor ReadBlockContents() Summary: Divide ReadBlockContents() to multiple sub-functions. Maintaining the input and intermediate data in a new class BlockFetcher. I hope in general it makes the code easier to maintain. Another motivation to do it is to clearly divide the logic before file reading and after file reading. The refactor will help us evaluate how can we make I/O async in the future. Closes https://github.com/facebook/rocksdb/pull/3244 Differential Revision: D6520983 Pulled By: siying fbshipit-source-id: 338d90bc0338472d46be7a7682028dc9114b12e9 | 11 December 2017, 23:27:32 UTC |
9a27ac5 | Yi Wu | 11 December 2017, 21:46:57 UTC | Fix drop column family data race Summary: A data race is caught by tsan_crash test between compaction and DropColumnFamily: https://gist.github.com/yiwu-arbug/5a2b4baae05eeb99ae1719b650f30a44 Compaction checks if the column family has been dropped on each key input, while user can issue DropColumnFamily which updates cfd->dropped_, causing the data race. Fixing it by making cfd->dropped_ an atomic. Closes https://github.com/facebook/rocksdb/pull/3250 Differential Revision: D6535991 Pulled By: yiwu-arbug fbshipit-source-id: 5571df020beae7fa7db6fff5ad0d598f49962895 | 11 December 2017, 21:57:48 UTC |
fcc8a65 | Zhongyi Xie | 11 December 2017, 21:12:12 UTC | Make Universal compaction options dynamic Summary: Let me know if more test coverage is needed Closes https://github.com/facebook/rocksdb/pull/3213 Differential Revision: D6457165 Pulled By: miasantreble fbshipit-source-id: 3f944abff28aa7775237f1c4f61c64ccbad4eea9 | 11 December 2017, 21:27:06 UTC |
250a51a | Yi Wu | 11 December 2017, 20:01:22 UTC | BlobDB: refactor DB open logic Summary: Refactor BlobDB open logic. List of changes: Major: * On reopen, mark blob files found as immutable, do not use them for writing new keys. * Not to scan the whole file to find file footer. Instead just seek to the end of the file and try to read footer. Minor: * Move most of the real logic from blob_db.cc to blob_db_impl.cc. * Not to hold shared_ptr of event listeners in global maps in blob_db.cc * Some changes to BlobFile interface. * Improve logging and error handling. Closes https://github.com/facebook/rocksdb/pull/3246 Differential Revision: D6526147 Pulled By: yiwu-arbug fbshipit-source-id: 9dc4cdd63359a2f9b696af817086949da8d06952 | 11 December 2017, 20:12:38 UTC |
6a183d1 | Prashant D | 11 December 2017, 19:48:47 UTC | Fix coverity issues compaction_job, compaction_picker Summary: db/compaction_job.cc: ReportStartedCompaction(compaction); CID 1419863 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR) 2. uninit_member: Non-static class member bottommost_level_ is not initialized in this constructor nor in any functions that it calls. db/compaction_picker_universal.cc: 7struct InputFileInfo { 2. uninit_member: Non-static class member level is not initialized in this constructor nor in any functions that it calls. CID 1405355 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR) 4. uninit_member: Non-static class member index is not initialized in this constructor nor in any functions that it calls. 38 InputFileInfo() : f(nullptr) {} db/dbformat.h: ParsedInternalKey() 84 : sequence(kMaxSequenceNumber) // Make code analyzer happy CID 1168095 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR) 2. uninit_member: Non-static class member type is not initialized in this constructor nor in any functions that it calls. 85 {} // Intentionally left uninitialized (for speed) Closes https://github.com/facebook/rocksdb/pull/3091 Differential Revision: D6534558 Pulled By: yiwu-arbug fbshipit-source-id: 5ada975956196d267b3f149386842af71eda7553 | 11 December 2017, 19:57:15 UTC |
e3814a8 | Andrew Kryczka | 08 December 2017, 00:09:25 UTC | revert fbcode build behavior Summary: Closes https://github.com/facebook/rocksdb/pull/3242 Differential Revision: D6514255 Pulled By: ajkr fbshipit-source-id: c39fa8e745866b052649d02bf339e794d77e96a3 | 08 December 2017, 00:12:52 UTC |
fe608e3 | Dmitri Smirnov | 07 December 2017, 21:41:35 UTC | Fix a race condition in WindowsThread (port::Thread) Summary: Fix a race condition when we create a thread and immediately destroy This case should be supported. What happens is that the thread function needs the Data instance to actually run but has no shared ownership and must rely on the WindowsThread instance to continue existing. To address this we change unique_ptr to shared_ptr and then acquire an additional refcount for the threadproc which destroys it just before the thread exit. We choose to allocate shared_ptr instance on the heap as this allows the original thread to continue w/o waiting for the new thread to start running. Closes https://github.com/facebook/rocksdb/pull/3240 Differential Revision: D6511324 Pulled By: yiwu-arbug fbshipit-source-id: 4633ff7996daf4d287a9fe34f60c1dd28cf4ff36 | 07 December 2017, 21:42:53 UTC |
34aa245 | Prashant D | 07 December 2017, 19:52:12 UTC | Fix coverity issues version, write_batch Summary: db/version_builder.cc: 117 base_vstorage_->InternalComparator(); CID 1351713 (#1 of 1): Uninitialized pointer field (UNINIT_CTOR) 2. uninit_member: Non-static class member field level_zero_cmp_.internal_comparator is not initialized in this constructor nor in any functions that it calls. db/version_edit.h: 145 FdWithKeyRange() 146 : fd(), 147 smallest_key(), 148 largest_key() { CID 1418254 (#1 of 1): Uninitialized pointer field (UNINIT_CTOR) 2. uninit_member: Non-static class member file_metadata is not initialized in this constructor nor in any functions that it calls. 149 } db/version_set.cc: 120 } CID 1322789 (#1 of 1): Uninitialized pointer field (UNINIT_CTOR) 4. uninit_member: Non-static class member curr_file_level_ is not initialized in this constructor nor in any functions that it calls. 121 } db/write_batch.cc: 939 assert(cf_mems_); CID 1419862 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR) 3. uninit_member: Non-static class member rebuilding_trx_seq_ is not initialized in this constructor nor in any functions that it calls. 940 } Closes https://github.com/facebook/rocksdb/pull/3092 Differential Revision: D6505666 Pulled By: yiwu-arbug fbshipit-source-id: fd2c68948a0280772691a419d72ac7e190951d86 | 07 December 2017, 19:57:36 UTC |
baff91c | Prashant D | 07 December 2017, 19:50:49 UTC | table: Fix coverity issues Summary: table/block.cc: 420 } CID 1396127 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR) 7. uninit_member: Non-static class member restart_offset_ is not initialized in this constructor nor in any functions that it calls. 421} table/block_based_table_builder.cc: CID 1418259 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR) 7. uninit_member: Non-static class member compressed_cache_key_prefix_size is not initialized in this constructor nor in any functions that it calls. table/block_based_table_reader.h: 3. uninit_member: Non-static class member index_type is not initialized in this constructor nor in any functions that it calls. CID 1396147 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR) 5. uninit_member: Non-static class member hash_index_allow_collision is not initialized in this constructor nor in any functions that it calls. 413 global_seqno(kDisableGlobalSequenceNumber) {} 414 table/cuckoo_table_reader.cc: 55 if (hash_funs == user_props.end()) { 56 status_ = Status::Corruption("Number of hash functions not found"); 5. uninit_member: Non-static class member is_last_level_ is not initialized in this constructor nor in any functions that it calls. 7. uninit_member: Non-static class member identity_as_first_hash_ is not initialized in this constructor nor in any functions that it calls. 9. uninit_member: Non-static class member use_module_hash_ is not initialized in this constructor nor in any functions that it calls. 11. uninit_member: Non-static class member num_hash_func_ is not initialized in this constructor nor in any functions that it calls. 13. uninit_member: Non-static class member key_length_ is not initialized in this constructor nor in any functions that it calls. 15. uninit_member: Non-static class member user_key_length_ is not initialized in this constructor nor in any functions that it calls. 17. uninit_member: Non-static class member value_length_ is not initialized in this constructor nor in any functions that it calls. 19. uninit_member: Non-static class member bucket_length_ is not initialized in this constructor nor in any functions that it calls. 21. uninit_member: Non-static class member cuckoo_block_size_ is not initialized in this constructor nor in any functions that it calls. 23. uninit_member: Non-static class member cuckoo_block_bytes_minus_one_ is not initialized in this constructor nor in any functions that it calls. CID 1322785 (#2 of 2): Uninitialized scalar field (UNINIT_CTOR) 25. uninit_member: Non-static class member table_size_ is not initialized in this constructor nor in any functions that it calls. 57 return; table/plain_table_index.h: 2. uninit_member: Non-static class member index_size_ is not initialized in this constructor nor in any functions that it calls. CID 1322801 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR) 4. uninit_member: Non-static class member sub_index_size_ is not initialized in this constructor nor in any functions that it calls. 128 huge_page_tlb_size_(huge_page_tlb_size) {} 129 Closes https://github.com/facebook/rocksdb/pull/3113 Differential Revision: D6505719 Pulled By: yiwu-arbug fbshipit-source-id: 38f44d8f9dfefb4c2e25d83b8df25a5201c75618 | 07 December 2017, 19:57:36 UTC |
2e3a009 | Andrew Kryczka | 07 December 2017, 19:07:44 UTC | fix ASAN for DeleteFilesInRange test case Summary: error message was ``` ==3095==ERROR: AddressSanitizer: stack-use-after-scope on address 0x7ffd18216c40 at pc 0x0000005edda1 bp 0x7ffd18215550 sp 0x7ffd18214d00 ... Address 0x7ffd18216c40 is located in stack of thread T0 at offset 1952 in frame #0 internal_repo_rocksdb/db_compaction_test.cc:1520 rocksdb::DBCompactionTest_DeleteFileRangeFileEndpointsOverlapBug_Test::TestBody() ``` It was unsafe to have slices referring to the temporary string objects' buffers, as those strings were destroyed before the slices were used. Fixed it by assigning the strings returned by `Key()` to local variables. Closes https://github.com/facebook/rocksdb/pull/3238 Differential Revision: D6507864 Pulled By: ajkr fbshipit-source-id: dd07de1a0070c6748c1ab4f3d7bd31f9a81889d0 | 07 December 2017, 19:12:43 UTC |
e1c569c | Yi Wu | 07 December 2017, 06:54:11 UTC | Fix clang-analyzer false-positive on ldb_cmd.cc Summary: clang-analyzer complaint about db_ being nullptr, but it couldn't be because it checks exec_stats before proceed. Add an assert to get around the false-positive. Test Plan `make analyze` Closes https://github.com/facebook/rocksdb/pull/3236 Differential Revision: D6505417 Pulled By: yiwu-arbug fbshipit-source-id: e5b65764ea994dd9e4bab3e697b97dc70dc22cab | 07 December 2017, 06:58:46 UTC |
bbef8c3 | Sagar Vemuri | 07 December 2017, 04:43:52 UTC | Log GetCurrentTime failures during Flush and Compaction Summary: `GetCurrentTime()` is used to populate `creation_time` table property during flushes and compactions. It is safe to ignore `GetCurrentTime()` failures here but they should be logged. (Note that `creation_time` property was introduced as part of TTL-based FIFO compaction in #2480.) Tes Plan: `make check` Closes https://github.com/facebook/rocksdb/pull/3231 Differential Revision: D6501935 Pulled By: sagar0 fbshipit-source-id: 376adcf4ab801d3a43ec4453894b9a10909c8eb6 | 07 December 2017, 04:56:53 UTC |
d51fcb2 | Sagar Vemuri | 07 December 2017, 04:40:46 UTC | Blob DB: Add db_bench options Summary: Adding more BlobDB db_bench options which are needed for benchmarking. Closes https://github.com/facebook/rocksdb/pull/3230 Differential Revision: D6500711 Pulled By: sagar0 fbshipit-source-id: 91d63122905854ef7c9148a0235568719146e6c5 | 07 December 2017, 04:44:12 UTC |
53a516a | Andrew Kryczka | 07 December 2017, 03:46:44 UTC | update history for recent commits Summary: I browsed through the history since 5.9 was released and found some changes worth mentioning in HISTORY.md. Closes https://github.com/facebook/rocksdb/pull/3237 Differential Revision: D6506472 Pulled By: ajkr fbshipit-source-id: 627ce9f94ca33df9f0f231a9c5ced3624b05506c | 07 December 2017, 03:56:17 UTC |
78d1a5e | Andrew Kryczka | 07 December 2017, 02:49:26 UTC | Preserve overlapping file endpoint invariant Summary: Fix for #2833. - In `DeleteFilesInRange`, use `GetCleanInputsWithinInterval` instead of `GetOverlappingInputs` to make sure we get a clean cut set of files to delete. - In `GetCleanInputsWithinInterval`, support nullptr as `begin_key` or `end_key`. - In `GetOverlappingInputsRangeBinarySearch`, move the assertion for non-empty range away from `ExtendFileRangeWithinInterval`, which should be allowed to return an empty range (via `end_index < begin_index`). Closes https://github.com/facebook/rocksdb/pull/2843 Differential Revision: D5772387 Pulled By: ajkr fbshipit-source-id: e554e8461823c6be82b21a9262a2da02b3957881 | 07 December 2017, 02:56:54 UTC |
a7d3277 | Yi Wu | 07 December 2017, 01:04:32 UTC | Fix write_callback_test compile error Summary: Rename shadow variable name db_impl. Fixing #3227 Closes https://github.com/facebook/rocksdb/pull/3235 Differential Revision: D6504051 Pulled By: yiwu-arbug fbshipit-source-id: 186c9378dabb11f8d6db56f45c95cc3b029fcb88 | 07 December 2017, 01:12:27 UTC |
7f04af3 | Yi Wu | 07 December 2017, 00:38:49 UTC | ldb to allow db with --try_load_options and without an options file Summary: This is to fix tools/check_format_compatible.sh. The tool try to open old versions of rocksdb with the provided options file. When options file is missing (e.g. rocksdb 2.2), it should still proceed with default options. Closes https://github.com/facebook/rocksdb/pull/3232 Differential Revision: D6503955 Pulled By: yiwu-arbug fbshipit-source-id: e44cfcce7ddc7d12cf83466ed3f3fe7624aa78b8 | 07 December 2017, 00:42:26 UTC |
b5798bd | Yi Wu | 06 December 2017, 23:56:37 UTC | Add missing recent versions to format compatible test Summary: Add recent versions for format compatible test. We should probably update the script to auto include available versions (by looking at include/rocksdb/versions.h and deduce branch names), but we can do it later. Closes https://github.com/facebook/rocksdb/pull/3233 Differential Revision: D6503631 Pulled By: yiwu-arbug fbshipit-source-id: e2b01d1ef6e784ff6ffa1bd75d741755e3c69a8c | 07 December 2017, 00:13:50 UTC |
20995c5 | Yi Wu | 06 December 2017, 19:54:45 UTC | Make iterator invalid on Merge error Summary: Since #1665, on merge error, iterator will be set to corrupted status, but it doesn't invalidate the iterator. Fixing it. Closes https://github.com/facebook/rocksdb/pull/3226 Differential Revision: D6499094 Pulled By: yiwu-arbug fbshipit-source-id: 80222930f949e31f90a6feaa37ddc3529b510d2c | 06 December 2017, 19:56:39 UTC |
36911f5 | Maysam Yabandeh | 06 December 2017, 17:32:09 UTC | WritePrepared Txn: stress test Summary: Augment the existing MySQLStyleTransactionTest to check for more core case scenarios. The changes showed effective in revealing the bugs reported in https://github.com/facebook/rocksdb/pull/3205 and https://github.com/facebook/rocksdb/pull/3101 Closes https://github.com/facebook/rocksdb/pull/3222 Differential Revision: D6476862 Pulled By: maysamyabandeh fbshipit-source-id: 5068497702d67ffc206a58ed96f8578fbb510137 | 06 December 2017, 17:42:28 UTC |
1b0c58d | Flarnie Marchan | 06 December 2017, 02:41:08 UTC | Add Code of Conduct Summary: In the past Facebook didn't promote including a Code of Conduct when creating new projects, and many projects skipped this important document. Let's fix it. :) **why make this change?:** Facebook Open Source provides a Code of Conduct statement for all projects to follow, to promote a welcoming and safe open source community. Exposing the COC via a separate markdown file is a standard being promoted by Github via the Community Profile in order to meet their Open Source Guide's recommended community standards. As you can see, adding this file will improve [the rocksdb community profile](https://github.com/facebook/rocksdb/community) checklist and increase the visibility of our COC. **test plan:** Viewing it on my branch - <img width="1008" alt="screen shot 2017-12-03 at 5 05 45 pm" src="https://user-images.githubusercontent.com/1114467/33532198-66012a56-d84c-11e7-8fab-29ed410bd600.png"> <img width="1015" alt="screen shot 2017-12-03 at 5 05 59 pm" src="https://user-images.githubusercontent.com/1114467/33532199-661813d8-d84c-11e7-941e-94754dd481e5.png"> **issue:** internal task t23481323 Closes https://github.com/facebook/rocksdb/pull/3219 Reviewed By: yiwu-arbug Differential Revision: D6494234 Pulled By: flarnie fbshipit-source-id: 55b59db335cc5546f3a1c968322b9281a3dc3aaf | 06 December 2017, 02:42:35 UTC |
a37d734 | Adam Novak | 05 December 2017, 23:05:41 UTC | Add ROCKSDB_DISABLE_* environment variables Summary: Should fix #3036. Closes https://github.com/facebook/rocksdb/pull/3042 Differential Revision: D6452921 Pulled By: sagar0 fbshipit-source-id: eaf11e43fee1f8747006530cfc0c7a358f1c2f0f | 05 December 2017, 23:12:46 UTC |
4634c73 | Alex Robinson | 04 December 2017, 09:53:49 UTC | Update DBOptions::IncreaseParallelism to use newer background settings Summary: The Options header file recommends using max_background_jobs rather than directly setting max_background_compactions or max_background_flushes. I've personally seen a performance problem where stalls were happening because the one background flushing thread was blocked that was fixed by this change - https://github.com/cockroachdb/cockroach/issues/19699#issuecomment-347672485 Closes https://github.com/facebook/rocksdb/pull/3208 Differential Revision: D6473178 Pulled By: ajkr fbshipit-source-id: 67c892ceb7b1909d251492640cb15a0f2262b7ed | 04 December 2017, 09:56:15 UTC |
57056bb | Andrew Kryczka | 02 December 2017, 02:26:15 UTC | gflags in cmake on linux Summary: We should use it if available otherwise the tools builds never work. Thanks to #3212, we can set -DGFLAGS=1 and it'll be independent of the namespace with which gflags was compiled. Closes https://github.com/facebook/rocksdb/pull/3214 Differential Revision: D6462214 Pulled By: ajkr fbshipit-source-id: db4e5f1b905322e3119554a9d01b57532c499384 | 02 December 2017, 02:28:24 UTC |
63f1c0a | Andrew Kryczka | 01 December 2017, 18:40:45 UTC | fix gflags namespace Summary: I started adding gflags support for cmake on linux and got frustrated that I'd need to duplicate the build_detect_platform logic, which determines namespace based on attempting compilation. We can do it differently -- use the GFLAGS_NAMESPACE macro if available, and if not, that indicates it's an old gflags version without configurable namespace so we can simply hardcode "google". Closes https://github.com/facebook/rocksdb/pull/3212 Differential Revision: D6456973 Pulled By: ajkr fbshipit-source-id: 3e6d5bde3ca00d4496a120a7caf4687399f5d656 | 01 December 2017, 18:42:05 UTC |
18dcf7f | Maysam Yabandeh | 01 December 2017, 07:39:56 UTC | WritePrepared Txn: PreReleaseCallback Summary: Add PreReleaseCallback to be called at the end of WriteImpl but before publishing the sequence number. The callback is used in WritePrepareTxn to i) update the commit map, ii) update the last published sequence number in the 2nd write queue. It also ensures that all the commits will go to the 2nd queue. These changes will ensure that the commit map is updated before the sequence number is published and used by reading snapshots. If we use two write queues, the snapshots will use the seq number published by the 2nd queue. If we use one write queue (the default, the snapshots will use the last seq number in the memtable, which also indicates the last published seq number. Closes https://github.com/facebook/rocksdb/pull/3205 Differential Revision: D6438959 Pulled By: maysamyabandeh fbshipit-source-id: f8b6c434e94bc5f5ab9cb696879d4c23e2577ab9 | 01 December 2017, 07:50:45 UTC |
3e40a5e | Nikhil Benesch | 01 December 2017, 06:37:19 UTC | add missing config checks to CMakeLists.txt Summary: Bring CMakeLists.txt back up to parity with build_detect_platform. Closes https://github.com/facebook/rocksdb/pull/3211 Differential Revision: D6452908 Pulled By: ajkr fbshipit-source-id: 93f5f336ad7eff6ecf65dec47bfaf114dd24cfb2 | 01 December 2017, 06:57:00 UTC |
fad1405 | Andres Suarez | 30 November 2017, 23:22:41 UTC | Remove `import` use from TARGETS Summary: We're moving away from `import`. The equivalent internal construct that gets the directory from `fbcode/` is `package_name()`. This is a Skylark friendly wrapper around [`get_base_path`]. The additional whitespace change is from running `python ./buckifier/buckify_rocksdb.py`. [`get_base_path`]: https://buckbuild.com/function/get_base_path.html Closes https://github.com/facebook/rocksdb/pull/3210 Reviewed By: yiwu-arbug Differential Revision: D6451242 Pulled By: zertosh fbshipit-source-id: 445757261de0ec89d5d332c1ba9af097086326dc | 30 November 2017, 23:27:34 UTC |
54095d3 | Yi Wu | 30 November 2017, 18:40:39 UTC | TARGETS file not include tests in opt mode Summary: Do not build the tests in opt mode, since SyncPoint and other test code will not be included. Closes https://github.com/facebook/rocksdb/pull/3204 Differential Revision: D6431154 Pulled By: yiwu-arbug fbshipit-source-id: c404ef042c1a6f679e5c1dc57600b3d8cb52fc28 | 30 November 2017, 18:56:58 UTC |