swh:1:snp:5115096b921df712aeb2a08114fede57fb3331fb
- HEAD
- refs/heads/2.2.fb.branch
- refs/heads/2.3.fb.branch
- refs/heads/2.4.fb.branch
- refs/heads/2.5.fb.branch
- refs/heads/2.6.fb.branch
- refs/heads/2.7
- refs/heads/2.7.fb.branch
- refs/heads/2.8.1.fb
- refs/heads/2.8.fb
- refs/heads/2.8.fb.trunk
- refs/heads/3.0.fb
- refs/heads/3.0.fb.branch
- refs/heads/3.1.fb
- refs/heads/3.10.fb
- refs/heads/3.11.fb
- refs/heads/3.12.fb
- refs/heads/3.13.fb
- refs/heads/3.2.fb
- refs/heads/3.3.fb
- refs/heads/3.4.fb
- refs/heads/3.5.fb
- refs/heads/3.6.fb
- refs/heads/3.7.fb
- refs/heads/3.8.fb
- refs/heads/3.9.fb
- refs/heads/4.0.fb
- refs/heads/4.1.fb
- refs/heads/4.10.fb
- refs/heads/4.11.fb
- refs/heads/4.12.fb
- refs/heads/4.13.fb
- refs/heads/4.2.fb
- refs/heads/4.3.fb
- refs/heads/4.4.fb
- refs/heads/4.5.fb
- refs/heads/4.6.fb
- refs/heads/4.7.fb
- refs/heads/4.8.fb
- refs/heads/4.9.fb
- refs/heads/5.0.fb
- refs/heads/5.1.fb
- refs/heads/5.10.fb
- refs/heads/5.11.fb
- refs/heads/5.12.fb
- refs/heads/5.13.fb
- refs/heads/5.13.fb.myrocks
- refs/heads/5.14.fb
- refs/heads/5.14.fb.myrocks
- refs/heads/5.15.fb
- refs/heads/5.16.fb
- refs/heads/5.17.fb
- refs/heads/5.17.fb.myrocks
- refs/heads/5.18.fb
- refs/heads/5.2.fb
- refs/heads/5.3.fb
- refs/heads/5.4.fb
- refs/heads/5.5.fb
- refs/heads/5.6.fb
- refs/heads/5.7.fb
- refs/heads/5.7.fb.myrocks
- refs/heads/5.8.3
- refs/heads/5.8.fb
- refs/heads/5.9.fb
- refs/heads/5.9.fb.myrocks
- refs/heads/6.0.fb
- refs/heads/6.0.fb.myrocks
- refs/heads/6.1.fb
- refs/heads/6.1.fb.myrocks
- refs/heads/6.1.fb.prod201905
- refs/heads/6.10.fb
- refs/heads/6.11.fb
- refs/heads/6.12.fb
- refs/heads/6.13.fb
- refs/heads/6.13.fb.laser
- refs/heads/6.14.fb
- refs/heads/6.14.fb.laser
- refs/heads/6.15.fb
- refs/heads/6.16.fb
- refs/heads/6.17.fb
- refs/heads/6.2.fb
- refs/heads/6.3.fb
- refs/heads/6.3.fb.myrocks
- refs/heads/6.3.fb.myrocks2
- refs/heads/6.3fb
- refs/heads/6.4.fb
- refs/heads/6.5.fb
- refs/heads/6.6.fb
- refs/heads/6.7.fb
- refs/heads/6.8.fb
- refs/heads/6.9.fb
- refs/heads/adaptive
- refs/heads/ajkr-patch-1
- refs/heads/blob_shadow
- refs/heads/bottom-pri-level
- refs/heads/bugfix-build-detect
- refs/heads/checksum_readahead_mmap_fix
- refs/heads/fb-config
- refs/heads/feature/debug-rocksdbjavastatic
- refs/heads/feature/travis-arm64
- refs/heads/fix-release-notes
- refs/heads/fix-write-batch-comment
- refs/heads/format_compatible_4
- refs/heads/getmergeops
- refs/heads/gh-pages-old
- refs/heads/history-update
- refs/heads/hotfix/lambda-capture
- refs/heads/improve-support
- refs/heads/jay-stress
- refs/heads/katherinez-patch-1
- refs/heads/katherinez-patch-2
- refs/heads/master
- refs/heads/pr-sanity-check-as-GHAction
- refs/heads/pr/6062
- refs/heads/ramvadiv-patch-1
- refs/heads/scaffold
- refs/heads/siying-patch-1
- refs/heads/siying-patch-10
- refs/heads/siying-patch-2
- refs/heads/siying-patch-3
- refs/heads/siying-patch-4
- refs/heads/skip_memtable_flush
- refs/heads/testing_ppc_build
- refs/heads/tests
- refs/heads/unused-var
- refs/heads/v6.6.4
- refs/heads/yiwu_stackable
- refs/heads/yuslepukhin
- refs/remotes/origin/5.13.fb
- refs/tags/2.5.fb
- refs/tags/2.6.fb
- refs/tags/3.0.fb
- refs/tags/do-not-use-me2
- refs/tags/rocksdb-3.1
- refs/tags/rocksdb-3.10.2
- refs/tags/rocksdb-3.11
- refs/tags/rocksdb-3.11.1
- refs/tags/rocksdb-3.11.2
- refs/tags/rocksdb-3.2
- refs/tags/rocksdb-3.3
- refs/tags/rocksdb-3.4
- refs/tags/rocksdb-3.5
- refs/tags/rocksdb-3.5.1
- refs/tags/rocksdb-3.6.1
- refs/tags/rocksdb-3.6.2
- refs/tags/rocksdb-3.7
- refs/tags/rocksdb-3.8
- refs/tags/rocksdb-3.9
- refs/tags/rocksdb-3.9.1
- refs/tags/rocksdb-4.1
- refs/tags/rocksdb-5.10.2
- refs/tags/rocksdb-5.10.3
- refs/tags/rocksdb-5.10.4
- refs/tags/rocksdb-5.11.2
- refs/tags/rocksdb-5.11.3
- refs/tags/rocksdb-5.14.3
- refs/tags/rocksdb-5.2.1
- refs/tags/rocksdb-5.3.3
- refs/tags/rocksdb-5.3.4
- refs/tags/rocksdb-5.3.5
- refs/tags/rocksdb-5.3.6
- refs/tags/rocksdb-5.4.10
- refs/tags/rocksdb-5.4.5
- refs/tags/rocksdb-5.4.6
- refs/tags/rocksdb-5.5.2
- refs/tags/rocksdb-5.5.3
- refs/tags/rocksdb-5.5.4
- refs/tags/rocksdb-5.5.5
- refs/tags/rocksdb-5.5.6
- refs/tags/rocksdb-5.6.1
- refs/tags/rocksdb-5.6.2
- refs/tags/rocksdb-5.7.1
- refs/tags/rocksdb-5.7.2
- refs/tags/rocksdb-5.7.3
- refs/tags/rocksdb-5.7.5
- refs/tags/rocksdb-5.8.6
- refs/tags/rocksdb-5.8.7
- refs/tags/rocksdb-5.8.8
- refs/tags/rocksdb-5.9.2
- refs/tags/v4.0
- refs/tags/v4.1
- refs/tags/v5.10.2
- refs/tags/v5.10.3
- refs/tags/v5.10.4
- refs/tags/v5.11.2
- refs/tags/v5.11.3
- refs/tags/v5.13.3
- refs/tags/v5.14.3
- refs/tags/v5.15.10
- refs/tags/v5.18.3
- refs/tags/v5.2.1
- refs/tags/v5.3.3
- refs/tags/v5.3.4
- refs/tags/v5.3.5
- refs/tags/v5.3.6
- refs/tags/v5.4.10
- refs/tags/v5.4.5
- refs/tags/v5.4.6
- refs/tags/v5.5.2
- refs/tags/v5.5.3
- refs/tags/v5.5.4
- refs/tags/v5.5.5
- refs/tags/v5.5.6
- refs/tags/v5.6.1
- refs/tags/v5.6.2
- refs/tags/v5.7.1
- refs/tags/v5.7.2
- refs/tags/v5.7.3
- refs/tags/v5.7.5
- refs/tags/v5.8.6
- refs/tags/v5.8.7
- refs/tags/v5.8.8
- refs/tags/v5.9.2
- refs/tags/v6.0.1
- refs/tags/v6.0.2
- refs/tags/v6.1.1
- refs/tags/v6.1.2
- refs/tags/v6.10.1
- refs/tags/v6.10.2
- refs/tags/v6.11.4
- refs/tags/v6.11.6
- refs/tags/v6.12.6
- refs/tags/v6.12.7
- refs/tags/v6.13.2
- refs/tags/v6.13.3
- refs/tags/v6.14.5
- refs/tags/v6.14.6
- refs/tags/v6.15.4
- refs/tags/v6.2.2
- refs/tags/v6.2.4
- refs/tags/v6.3.6
- refs/tags/v6.4.6
- refs/tags/v6.5.2
- refs/tags/v6.5.3
- refs/tags/v6.6.3
- refs/tags/v6.6.4
- refs/tags/v6.7.3
- refs/tags/v6.8.1
- v6.15.2
- v5.8
- v5.5.1
- v5.4.7
- v5.18.4
- v5.17.2
- v5.16.6
- v5.14.2
- v5.13.4
- v5.13.2
- v5.13.1
- v5.12.5
- v5.12.4
- v5.12.3
- v5.12.2
- v5.1.4
- v5.1.3
- v5.1.2
- v5.0.2
- v5.0.1
- v4.9
- v4.8
- v4.6.1
- v4.5.1
- v4.4.1
- v4.4
- v4.3.1
- v4.3
- v4.2
- v4.13.5
- v4.13
- v4.11.2
- v3.9
- v3.8
- v3.7
- v3.6.1
- v3.5
- v3.4
- v3.3
- v3.2
- v3.13.1
- v3.13
- v3.12.1
- v3.12
- v3.11
- v3.10
- v3.1
- v3.0
- v2.8
- v2.7
- v2.6
- v2.5
- v2.4
- v2.3
- v2.2
- v2.1
- v2.0
- v1.5.9.1
- v1.5.8.2
- v1.5.8.1
- v1.5.8
- v1.5.7
- rocksdb-5.8
- rocksdb-5.4.7
- rocksdb-5.1.4
- rocksdb-5.1.3
- rocksdb-5.1.2
- rocksdb-5.0.2
- rocksdb-5.0.1
- rocksdb-4.9
- rocksdb-4.8
- rocksdb-4.6.1
- rocksdb-4.5.1
- rocksdb-4.4.1
- rocksdb-4.4
- rocksdb-4.3.1
- rocksdb-4.3
- rocksdb-4.2
- rocksdb-4.13.5
- rocksdb-4.13
- rocksdb-4.11.2
- rocksdb-3.10.1
- 2.8.fb
- 2.7.fb
- 2.4.fb
- 2.3.fb
- 2.2.fb
- 2.1.fb
- 2.0.fb
- 1.5.9.fb
- 1.5.9.2.fb
- 1.5.9.1.fb
- 1.5.8.fb
- 1.5.8.2.fb
- 1.5.8.1.fb
- 1.5.7.fb
Permalinks
To reference or cite the objects present in the Software Heritage archive, permalinks based on SoftWare Hash IDentifiers (SWHIDs) must be used.
Select below a type of object currently browsed in order to display its associated SWHID and permalink.
Revision | Author | Date | Message | Commit Date |
---|---|---|---|---|
b858da7 | Mayank Agarwal | 20 June 2013, 18:50:33 UTC | Simplify bucketing logic in ldb-ttl Summary: [start_time, end_time) is waht I'm following for the buckets and the whole time-range. Also cleaned up some code in db_ttl.* Not correcting the spacing/indenting convention for util/ldb_cmd.cc in this diff. Test Plan: python ldb_test.py, make ttl_test, Run mcrocksdb-backup tool, Run the ldb tool on 2 mcrocksdb production backups form sigmafio033.prn1 Reviewers: vamsi, haobo Reviewed By: vamsi Differential Revision: https://reviews.facebook.net/D11433 | 21 June 2013, 16:49:24 UTC |
61f1baa | Mayank Agarwal | 19 June 2013, 02:57:54 UTC | Introducing timeranged scan, timeranged dump in ldb. Also the ability to count in time-batches during Dump Summary: Scan and Dump commands in ldb use iterator. We need to also print timestamp for ttl databases for debugging. For this I create a TtlIterator class pointer in these functions and assign it the value of Iterator pointer which actually points to t TtlIterator object, and access the new function ValueWithTS which can return TS also. Buckets feature for dump command: gives a count of different key-values in the specified time-range distributed across the time-range partitioned according to bucket-size. start_time and end_time are specified in unixtimestamp and bucket in seconds on the user-commandline Have commented out 3 ines from ldb_test.py so that the test does not break right now. It breaks because timestamp is also printed now and I have to look at wildcards in python to compare properly. Test Plan: python tools/ldb_test.py Reviewers: vamsi, dhruba, haobo, sheki Reviewed By: vamsi CC: leveldb Differential Revision: https://reviews.facebook.net/D11403 | 20 June 2013, 01:45:13 UTC |
0f78fad | Haobo Xu | 19 June 2013, 20:32:03 UTC | [RocksDB] add back --mmap_read options to crashtest Summary: As title, now that db_stress supports --map_read properly Test Plan: make crash_test Reviewers: vamsi, emayanke, dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11391 | 19 June 2013, 23:15:59 UTC |
4deaa0d | Haobo Xu | 19 June 2013, 19:39:24 UTC | [RocksDB] Minor change to statistics.h Summary: as title, use initialize list so that lines fit in 80 chars. Test Plan: make check; Reviewers: sheki, dhruba Differential Revision: https://reviews.facebook.net/D11385 | 19 June 2013, 19:44:42 UTC |
96be2c4 | Haobo Xu | 17 June 2013, 23:13:32 UTC | [RocksDB] Add mmap_read option for db_stress Summary: as title, also removed an incorrect assertion Test Plan: make check; db_stress --mmap_read=1; db_stress --mmap_read=0 Reviewers: dhruba, emayanke CC: leveldb Differential Revision: https://reviews.facebook.net/D11367 | 19 June 2013, 17:28:32 UTC |
5ef6bb8 | Abhishek Kona | 19 June 2013, 03:28:41 UTC | [rocksdb][refactor] statistic printing code to one place Summary: $title Test Plan: db_bench --statistics=1 Reviewers: haobo Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D11373 | 19 June 2013, 03:28:41 UTC |
09de7a3 | Jim Paton | 18 June 2013, 23:57:42 UTC | Fix Zlib_Compress and Zlib_Uncompress Summary: Zlib_{Compress,Uncompress} did not handle very small input buffers properly. In addition, they did not call inflate/deflate until Z_STREAM_END was returned; it was possible for them to exit when only Z_OK had returned. This diff also fixes a bunch of lint errors. Test Plan: Run make check Reviewers: dhruba, sheki, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11301 | 18 June 2013, 23:57:42 UTC |
3cc1af2 | Haobo Xu | 14 June 2013, 05:49:46 UTC | [RocksDB] Option for incremental sync Summary: This diff added an option to control the incremenal sync frequency. db_bench has a new flag bytes_per_sync for easy tuning exercise. Test Plan: make check; db_bench Reviewers: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11295 | 18 June 2013, 22:00:32 UTC |
79f4fd2 | Abhishek Kona | 18 June 2013, 21:58:00 UTC | [Rocksdb] Simplify Printing code in db_bench Summary: simplify the printing code in db_bench use TickersMap and HistogramsNameMap introduced in previous diffs. Test Plan: ./db_bench --statistics=1 and see if all the statistics are printed Reviewers: haobo, dhruba Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D11355 | 18 June 2013, 21:58:00 UTC |
6acbe0f | Dhruba Borthakur | 11 June 2013, 21:23:58 UTC | Compact multiple memtables before flushing to storage. Summary: Merge multiple multiple memtables in memory before writing it out to a file in L0. There is a new config parameter min_write_buffer_number_to_merge that specifies the number of write buffers that should be merged together to a single file in storage. The system will not flush wrte buffers to storage unless at least these many buffers have accumulated in memory. The default value of this new parameter is 1, which means that a write buffer will be immediately flushed to disk as soon it is ready. Test Plan: make check Differential Revision: https://reviews.facebook.net/D11241 | 18 June 2013, 21:28:04 UTC |
f561b3a | Abhishek Kona | 17 June 2013, 21:32:48 UTC | [Rocksdb] Rename one stat key from leveldb to rocksdb | 17 June 2013, 21:33:05 UTC |
836534d | Dhruba Borthakur | 17 June 2013, 20:51:12 UTC | Enhance dbstress to allow specifying compaction trigger for L0. Summary: Rocksdb allos specifying the number of files in L0 that triggers compactions. Expose this api as a command line parameter for running db_stress. Test Plan: Run test Reviewers: sheki, emayanke Reviewed By: emayanke CC: leveldb Differential Revision: https://reviews.facebook.net/D11343 | 17 June 2013, 21:15:09 UTC |
0012468 | Abhishek Kona | 17 June 2013, 20:58:17 UTC | [rocksdb] do not trim range for level0 in manual compaction Summary: https://code.google.com/p/leveldb/issues/detail?can=1&q=178&colspec=ID%20Type%20Status%20Priority%20Milestone%20Owner%20Summary&id=178 Ported the solution as is to RocksDB. Test Plan: moved the unit test as manual_compaction_test Reviewers: dhruba Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11331 | 17 June 2013, 20:58:17 UTC |
39ee47f | Abhishek Kona | 17 June 2013, 17:11:10 UTC | [Rocksdb] Record WriteBlock Times into a histogram Summary: Add a histogram to track WriteBlock times Test Plan: db_bench and print Reviewers: haobo, dhruba Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D11319 | 17 June 2013, 17:11:10 UTC |
8926b72 | Deon Nicholas | 14 June 2013, 23:44:39 UTC | Minor tweaks to StringAppend MergeOperator. Summary: I'm concerned about a random seg-fault that sometimes occurs when running stringappend_test. I will investigate further. First, I am removing stringappend_test from the regular release tests, and making some clean-ups to the code. Test Plan: 1. make stringappend_test 2. ./stringappend_test Reviewers: haobo, dhruba Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D11313 | 14 June 2013, 23:44:39 UTC |
bff718d | Abhishek Kona | 14 June 2013, 23:17:56 UTC | [Rocksdb] Implement filluniquerandom Summary: Use a bit set to keep track of which random number is generated. Currently only supports single-threaded. All our perf tests are run with threads=1 Copied over bitset implementation from common/datastructures Test Plan: printed the generated keys, and verified all keys were present. Reviewers: MarkCallaghan, haobo, dhruba Reviewed By: MarkCallaghan CC: leveldb Differential Revision: https://reviews.facebook.net/D11247 | 14 June 2013, 23:17:56 UTC |
2a52e1d | Deon Nicholas | 14 June 2013, 23:00:47 UTC | Fix db_bench for release build. Test Plan: make release Reviewers: haobo, dhruba, jpaton Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D11307 | 14 June 2013, 23:00:47 UTC |
1afdf28 | Haobo Xu | 06 June 2013, 17:31:37 UTC | [RocksDB] Compaction Filter Cleanup Summary: This hopefully gives the right semantics to compaction filter. Will write a small wiki to explain the ideas. Test Plan: make check; db_stress Reviewers: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11121 | 14 June 2013, 21:23:08 UTC |
7a5f71d | Abhishek Kona | 14 June 2013, 00:25:09 UTC | [Rocksdb] measure table open io in a histogram Summary: Table is setup for compaction using Table::SetupForCompaction. So read block calls can be differentiated b/w Gets/Compaction. Use this and measure times. Test Plan: db_bench --statistics=1 Reviewers: dhruba, haobo Reviewed By: haobo CC: leveldb, MarkCallaghan Differential Revision: https://reviews.facebook.net/D11217 | 14 June 2013, 00:25:09 UTC |
0c2a2dd | Haobo Xu | 13 June 2013, 20:45:51 UTC | [RocksDB] Fix build. Removed deprecated option --mmap_read from db_crashtest Summary: As title Test Plan: db_crashtest Reviewers: vamsi, emayanke CC: leveldb Differential Revision: https://reviews.facebook.net/D11271 | 13 June 2013, 20:48:35 UTC |
778e179 | Haobo Xu | 04 June 2013, 22:51:50 UTC | [RocksDB] Sync file to disk incrementally Summary: During compaction, we sync the output files after they are fully written out. This causes unnecessary blocking of the compaction thread and burstiness of the write traffic. This diff simply asks the OS to sync data incrementally as they are written, on the background. The hope is that, at the final sync, most of the data are already on disk and we would block less on the sync call. Thus, each compaction runs faster and we could use fewer number of compaction threads to saturate IO. In addition, the write traffic will be smoothed out, hopefully reducing the IO P99 latency too. Some quick tests show 10~20% improvement in per thread compaction throughput. Combined with posix advice on compaction read, just 5 threads are enough to almost saturate the udb flash bandwidth for 800 bytes write only benchmark. What's more promising is that, with saturated IO, iostat shows average wait time is actually smoother and much smaller. For the write only test 800bytes test: Before the change: await occillate between 10ms and 3ms After the change: await ranges 1-3ms Will test against read-modify-write workload too, see if high read latency P99 could be resolved. Will introduce a parameter to control the sync interval in a follow up diff after cleaning up EnvOptions. Test Plan: make check; db_bench; db_stress Reviewers: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11115 | 12 June 2013, 19:53:59 UTC |
4985a9f | Deon Nicholas | 12 June 2013, 19:42:21 UTC | [Rocksdb] [Multiget] Introduced multiget into db_bench Summary: Preliminary! Introduced the --use_multiget=1 and --keys_per_multiget=n flags for db_bench. Also updated and tested the ReadRandom() method to include an option to use multiget. By default, keys_per_multiget=100. Preliminary tests imply that multiget is at least 1.25x faster per key than regular get. Will continue adding Multiget for ReadMissing, ReadHot, RandomWithVerify, ReadRandomWriteRandom; soon. Will also think about ways to better verify benchmarks. Test Plan: 1. make db_bench 2. ./db_bench --benchmarks=fillrandom 3. ./db_bench --benchmarks=readrandom --use_existing_db=1 --use_multiget=1 --threads=4 --keys_per_multiget=100 4. ./db_bench --benchmarks=readrandom --use_existing_db=1 --threads=4 5. Verify ops/sec (and 1000000 of 1000000 keys found) Reviewers: haobo, MarkCallaghan, dhruba Reviewed By: MarkCallaghan CC: leveldb Differential Revision: https://reviews.facebook.net/D11127 | 12 June 2013, 19:42:21 UTC |
bdf1085 | Haobo Xu | 07 June 2013, 22:35:17 UTC | [RocksDB] cleanup EnvOptions Summary: This diff simplifies EnvOptions by treating it as POD, similar to Options. - virtual functions are removed and member fields are accessed directly. - StorageOptions is removed. - Options.allow_readahead and Options.allow_readahead_compactions are deprecated. - Unused global variables are removed: useOsBuffer, useFsReadAhead, useMmapRead, useMmapWrite Test Plan: make check; db_stress Reviewers: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11175 | 12 June 2013, 18:17:19 UTC |
5679107 | Deon Nicholas | 11 June 2013, 18:19:49 UTC | Completed the implementation and test cases for Redis API. Summary: Completed the implementation for the Redis API for Lists. The Redis API uses rocksdb as a backend to persistently store maps from key->list. It supports basic operations for appending, inserting, pushing, popping, and accessing a list, given its key. Test Plan: - Compile with: make redis_test - Test with: ./redis_test - Run all unit tests (for all rocksdb) with: make all check - To use an interactive REDIS client use: ./redis_test -m - To clean the database before use: ./redis_test -m -d Reviewers: haobo, dhruba, zshao Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10833 | 11 June 2013, 18:19:49 UTC |
e673d5d | Dhruba Borthakur | 03 June 2013, 19:57:23 UTC | Do not submit multiple simultaneous seek-compaction requests. Summary: The code was such that if multi-threaded-compactions as well as seek compaction are enabled then it submits multiple compaction request for the same range of keys. This causes extraneous sst-files to accumulate at various levels. Test Plan: I am not able to write a very good unit test for this one but can easily reproduce this bug with 'dbstress' with the following options. batch=1;maxk=100000000;ops=100000000;ro=0;fm=2;bpl=10485760;of=500000; wbn=3; mbc=20; mb=2097152; wbs=4194304; dds=1; sync=0; t=32; bs=16384; cs=1048576; of=500000; ./db_stress --disable_seek_compaction=0 --mmap_read=0 --threads=$t --block_size=$bs --cache_size=$cs --open_files=$of --verify_checksum=1 --db=/data/mysql/leveldb/dbstress.dir --sync=$sync --disable_wal=1 --disable_data_sync=$dds --write_buffer_size=$wbs --target_file_size_base=$mb --target_file_size_multiplier=$fm --max_write_buffer_number=$wbn --max_background_compactions=$mbc --max_bytes_for_level_base=$bpl --reopen=$ro --ops_per_thread=$ops --max_key=$maxk --test_batches_snapshots=$batch Reviewers: leveldb, emayanke Reviewed By: emayanke Differential Revision: https://reviews.facebook.net/D11055 | 10 June 2013, 22:49:19 UTC |
3c35eda | Mayank Agarwal | 15 May 2013, 22:00:57 UTC | Make Write API work for TTL databases Summary: Added logic to make another WriteBatch with Timestamps during the Write function execution in TTL class. Also expanded the ttl_test to test for it. Have done nothing for Merge for now. Test Plan: make ttl_test;./ttl_test Reviewers: haobo, vamsi, dhruba Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10827 | 10 June 2013, 22:23:44 UTC |
1b69f1e | Dhruba Borthakur | 10 June 2013, 20:28:58 UTC | Fix refering freed memory in earlier commit. Summary: Fix refering freed memory in earlier commit by https://reviews.facebook.net/D11181 Test Plan: make check Reviewers: haobo, sheki Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D11193 | 10 June 2013, 22:08:13 UTC |
4a8554d | Abhishek Kona | 10 June 2013, 20:14:14 UTC | [Rocksdb] fix wrong assert Summary: the assert was wrong in D11145. Broke build Test Plan: make db_bench run it Reviewers: dhruba, haobo, emayanke Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11187 | 10 June 2013, 20:14:14 UTC |
c5de1b9 | Dhruba Borthakur | 10 June 2013, 15:13:23 UTC | Print name of user comparator in LOG. Summary: The current code prints the name of the InternalKeyComparator in the log file. We would also like to print the name of the user-specified comparator for easier debugging. Test Plan: make check Reviewers: sheki Reviewed By: sheki CC: leveldb Differential Revision: https://reviews.facebook.net/D11181 | 10 June 2013, 19:11:55 UTC |
a4913c5 | Abhishek Kona | 10 June 2013, 18:57:55 UTC | [rocksdb] names for all metrics provided in statistics.h Summary: Provide a map of histograms and ticker vs strings. Fb303 libraries can use this to provide the mapping. We will not have to duplicate the code during release. Test Plan: db_bench with statistics=1 Reviewers: dhruba, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11145 | 10 June 2013, 18:57:55 UTC |
184343a | Mayank Agarwal | 09 June 2013, 03:15:10 UTC | Max_mem_compaction_level can have maximum value of num_levels-1 Summary: Without this files could be written out to a level greater than the maximum level possible and is the source of the segfaults that wormhole awas getting. The sequence of steps that was followed: 1. WriteLevel0Table was called when memtable was to be flushed for a file. 2. PickLevelForMemTableOutput was called to determine the level to which this file should be pushed. 3. PickLevelForMemTableOutput returned a wrong result because max_mem_compaction_level was equal to 2 even when num_levels was equal to 0. The fix to re-initialize max_mem_compaction_level based on num_levels passed seems correct. Test Plan: make all check; Also made a dummy file to mimic the wormhole-file behaviour which was causing the segfaults and found that the same segfault occurs without this change and not with this. Reviewers: dhruba, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11157 | 09 June 2013, 17:38:55 UTC |
7a6bd8e | Mayank Agarwal | 08 June 2013, 19:29:43 UTC | Modifying options to db_stress when it is run with db_crashtest Summary: These extra options caught some bugs. Will be run via Jenkins now with the crash_test Test Plan: ./make crashtest Reviewers: dhruba, vamsi Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11151 | 09 June 2013, 16:58:46 UTC |
3bb9449 | Vamsi Ponnekanti | 07 June 2013, 18:06:20 UTC | [Fix whilebox crash test failure] Summary: I think the check for "error" that I added had caused false alarm. Fixed that. Test Plan: Revert Plan: OK Task ID: # Reviewers: emayanke, dhruba Reviewed By: emayanke Differential Revision: https://reviews.facebook.net/D11139 | 07 June 2013, 18:34:46 UTC |
e982b5a | Abhishek Kona | 07 June 2013, 17:02:28 UTC | [Rocksdb] measure table open io in a histogram Summary: as title Test Plan: db_bench --statistics=1 check for statistic. Reviewers: dhruba, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11109 | 07 June 2013, 17:02:28 UTC |
8ef328e | Jim Paton | 05 June 2013, 17:37:38 UTC | ctags and cscope support to Makefile Summary: Added a target to Makefile called 'tags' that runs ctags and cscope on all *.cc and *.h file Test Plan: Run 'make tags'. Then start vim and do :set tags=./tags :cs add cscope.out These commands should give you no error messages. You should then be able to access cscope db and ctags as normal in vim. Reviewers: dhruba Differential Revision: https://reviews.facebook.net/D11103 | 07 June 2013, 16:13:40 UTC |
5cf7a00 | Vamsi Ponnekanti | 24 May 2013, 02:10:13 UTC | [Make most of the changes suggested by Aaron] Summary: $title Test Plan: Revert Plan: OK Task ID: # Reviewers: emayanke, akushner Reviewed By: akushner Differential Revision: https://reviews.facebook.net/D10923 | 07 June 2013, 00:31:45 UTC |
db1f0cd | Deon Nicholas | 06 June 2013, 00:25:16 UTC | Fixed valgrind errors | 06 June 2013, 00:25:16 UTC |
d8c7c45 | Deon Nicholas | 05 June 2013, 18:22:38 UTC | Very basic Multiget and simple test cases. Summary: Implemented the MultiGet operator which takes in a list of keys and returns their associated values. Currently uses std::vector as its container data structure. Otherwise, it works identically to "Get". Test Plan: 1. make db_test ; compile it 2. ./db_test ; test it 3. make all check ; regress / run all tests 4. make release ; (optional) compile with release settings Reviewers: haobo, MarkCallaghan, dhruba Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10875 | 05 June 2013, 18:22:38 UTC |
d91b42e | Abhishek Kona | 05 June 2013, 18:06:21 UTC | [Rocksdb] Measure all FSYNC/SYNC times Summary: Add stop watches around all sync calls. Test Plan: db_bench check if respective histograms are printed Reviewers: haobo, dhruba Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D11073 | 05 June 2013, 18:06:21 UTC |
ee522d0 | Abhishek Kona | 05 June 2013, 17:48:24 UTC | [Rocksdb] Log on disable/enable file deletions Summary: as title Test Plan: compile Reviewers: dhruba, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D11085 | 05 June 2013, 17:48:24 UTC |
043573b | Haobo Xu | 04 June 2013, 06:23:03 UTC | [RocksDB] Include 64bit random number generator Summary: As title. Test Plan: make check; Reviewers: chip, MarkCallaghan CC: leveldb Differential Revision: https://reviews.facebook.net/D11061 | 04 June 2013, 20:52:27 UTC |
d9f538e | Mark Callaghan | 03 June 2013, 15:16:16 UTC | Improve output for GetProperty('leveldb.stats') Summary: Display separate values for read, write & total compaction IO. Display compaction amplification and write amplification. Add similar values for the period since the last call to GetProperty. Results since the server started are reported as "cumulative" stats. Results since the last call to GetProperty are reported as "interval" stats. Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0 1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0 2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0 >>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted Uptime(secs): 122.2 total, 0.9 interval Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write Amplification cumulative: 6.0 write, 11.0 compaction Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write Amplification interval: 1.0 write, 1.0 compaction >>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown Task ID: # Blame Rev: Test Plan: make check, run db_bench Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: haobo Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D11049 | 04 June 2013, 03:24:49 UTC |
2b1fb5b | Haobo Xu | 31 May 2013, 05:57:22 UTC | [RocksDB] Add score column to leveldb.stats Summary: Added the 'score' column to the compaction stats output, which shows the level total size devided by level target size. Could be useful when monitoring compaction decisions... Test Plan: make check; db_bench Reviewers: dhruba CC: leveldb, MarkCallaghan Differential Revision: https://reviews.facebook.net/D11025 | 04 June 2013, 00:38:27 UTC |
d897d33 | Haobo Xu | 31 May 2013, 23:30:17 UTC | [RocksDB] Introduce Fast Mutex option Summary: This diff adds an option to specify whether PTHREAD_MUTEX_ADAPTIVE_NP will be enabled for the rocksdb single big kernel lock. db_bench also have this option now. Quickly tested 8 thread cpu bound 100 byte random read. No fast mutex: ~750k/s ops With fast mutex: ~880k/s ops Test Plan: make check; db_bench; db_stress Reviewers: dhruba CC: MarkCallaghan, leveldb Differential Revision: https://reviews.facebook.net/D11031 | 02 June 2013, 06:11:34 UTC |
ab8d2f6 | Haobo Xu | 17 May 2013, 22:53:01 UTC | [RocksDB] [Performance] Allow different posix advice to be applied to the same table file Summary: Current posix advice implementation ties up the access pattern hint with the creation of a file. It is not possible to apply different advice for different access (random get vs compaction read), without keeping two open files for the same table. This patch extended the RandomeAccessFile interface to accept new access hint at anytime. Particularly, we are able to set different access hint on the same table file based on when/how the file is used. Two options are added to set the access hint, after the file is first opened and after the file is being compacted. Test Plan: make check; db_stress; db_bench Reviewers: dhruba Reviewed By: dhruba CC: MarkCallaghan, leveldb Differential Revision: https://reviews.facebook.net/D10905 | 31 May 2013, 02:08:44 UTC |
2df65c1 | Haobo Xu | 28 May 2013, 19:35:43 UTC | [RocksDB] Dump counters and histogram data periodically with compaction stats Summary: As title Test Plan: make check Reviewers: dhruba Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10995 | 29 May 2013, 19:00:18 UTC |
a8d807e | Dhruba Borthakur | 28 May 2013, 21:00:10 UTC | Record the number of open db iterators. Summary: Enhance the statitics to report the number of open db iterators. Test Plan: make check Reviewers: haobo, emayanke Reviewed By: emayanke CC: leveldb Differential Revision: https://reviews.facebook.net/D10983 | 29 May 2013, 15:47:08 UTC |
fb684da | Haobo Xu | 28 May 2013, 18:38:47 UTC | [RocksDB] Fix CorruptionTest Summary: Overriding block_size_deviation to zero, so that CorruptionTest can pass. Test Plan: make check Reviewers: dhruba Reviewed By: dhruba Differential Revision: https://reviews.facebook.net/D10977 | 28 May 2013, 19:36:42 UTC |
4c47d8f | heyongqiang | 15 May 2013, 17:34:02 UTC | add block deviation option to terminate a block before it exceeds block_size Summary: a new option block_size_deviation is added. Test Plan: run db_test and db_bench Reviewers: dhruba, haobo Reviewed By: haobo Differential Revision: https://reviews.facebook.net/D10821 | 24 May 2013, 23:21:52 UTC |
4b29651 | heyongqiang | 15 May 2013, 17:34:02 UTC | add block deviation option to terminate a block before it exceeds block_size Summary: a new option block_size_deviation is added. Test Plan: run db_test and db_bench Reviewers: dhruba, haobo Reviewed By: haobo Differential Revision: https://reviews.facebook.net/D10821 | 24 May 2013, 22:52:49 UTC |
ef15b9d | Haobo Xu | 24 May 2013, 19:52:45 UTC | [RocksDB] Fix MaybeDumpStats Summary: MaybeDumpStats was causing lock problem Test Plan: make check; db_stress Reviewers: dhruba Reviewed By: dhruba Differential Revision: https://reviews.facebook.net/D10935 | 24 May 2013, 22:43:16 UTC |
0e879c9 | Haobo Xu | 10 May 2013, 22:21:04 UTC | [RocksDB] dump leveldb.stats periodically in LOG file. Summary: Added an option stats_dump_period_sec to dump leveldb.stats to LOG periodically for diagnosis. By defauly, it's set to a very big number 3600 (1 hour). Test Plan: make check; Reviewers: dhruba Reviewed By: dhruba CC: leveldb, zshao Differential Revision: https://reviews.facebook.net/D10761 | 23 May 2013, 23:56:59 UTC |
2654186 | Dhruba Borthakur | 23 May 2013, 18:56:19 UTC | The max size of the write buffer size can be 64 GB. Summary: There was an artifical limit on the size of the write buffer size. Test Plan: make check Reviewers: haobo Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10911 | 23 May 2013, 22:00:27 UTC |
898f793 | Dhruba Borthakur | 23 May 2013, 17:56:36 UTC | Fix valgrind errors introduced by https://reviews.facebook.net/D10863 Summary: The valgrind errors were in the unit tests where we change the number of levels of a database using internal methods. Test Plan: valgrind ./reduce_levels_test valgrind ./db_test Reviewers: emayanke Reviewed By: emayanke CC: leveldb Differential Revision: https://reviews.facebook.net/D10893 | 23 May 2013, 19:14:41 UTC |
c2e2460 | Haobo Xu | 23 May 2013, 18:34:58 UTC | [RocksDB] Expose DBStatistics Summary: Make Statistics usable by client Test Plan: make check; db_bench Reviewers: dhruba Reviewed By: dhruba Differential Revision: https://reviews.facebook.net/D10899 | 23 May 2013, 18:49:38 UTC |
760dd47 | Vamsi Ponnekanti | 05 April 2013, 06:49:43 UTC | [Kill randomly at various points in source code for testing] Summary: This is initial version. A few ways in which this could be extended in the future are: (a) Killing from more places in source code (b) Hashing stack and using that hash in determining whether to crash. This is to avoid crashing more often at source lines that are executed more often. (c) Raising exceptions or returning errors instead of killing Test Plan: This whole thing is for testing. Here is part of output: python2.7 tools/db_crashtest2.py -d 600 Running db_stress db_stress retncode -15 output LevelDB version : 1.5 Number of threads : 32 Ops per thread : 10000000 Read percentage : 50 Write-buffer-size : 4194304 Delete percentage : 30 Max key : 1000 Ratio #ops/#keys : 320000 Num times DB reopens: 0 Batches/snapshots : 1 Purge redundant % : 50 Num keys per lock : 4 Compression : snappy ------------------------------------------------ No lock creation because test_batches_snapshots set 2013/04/26-17:55:17 Starting database operations Created bg thread 0x7fc1f07ff700 ... finished 60000 ops Running db_stress db_stress retncode -15 output LevelDB version : 1.5 Number of threads : 32 Ops per thread : 10000000 Read percentage : 50 Write-buffer-size : 4194304 Delete percentage : 30 Max key : 1000 Ratio #ops/#keys : 320000 Num times DB reopens: 0 Batches/snapshots : 1 Purge redundant % : 50 Num keys per lock : 4 Compression : snappy ------------------------------------------------ Created bg thread 0x7ff0137ff700 No lock creation because test_batches_snapshots set 2013/04/26-17:56:15 Starting database operations ... finished 90000 ops Revert Plan: OK Task ID: #2252691 Reviewers: dhruba, emayanke Reviewed By: emayanke CC: leveldb, haobo Differential Revision: https://reviews.facebook.net/D10581 | 22 May 2013, 01:21:49 UTC |
87d0af1 | Haobo Xu | 21 May 2013, 18:53:33 UTC | [RocksDB] Introduce an option to skip log error on recovery Summary: Currently, with paranoid_check on, DB::Open will fail on any log read error on recovery. If client is ok with losing most recent updates, we could simply skip those errors. However, it's important to introduce an additional flag, so that paranoid_check can still guard against more serious problems. Test Plan: make check; db_stress Reviewers: dhruba, emayanke Reviewed By: emayanke CC: leveldb, emayanke Differential Revision: https://reviews.facebook.net/D10869 | 21 May 2013, 21:30:36 UTC |
d1aaaf7 | Dhruba Borthakur | 21 May 2013, 18:37:06 UTC | Ability to set different size fanout multipliers for every level. Summary: There is an existing field Options.max_bytes_for_level_multiplier that sets the multiplier for the size of each level in the database. This patch introduces the ability to set different multipliers for every level in the database. The size of a level is determined by using both max_bytes_for_level_multiplier as well as the per-level fanout. size of level[i] = size of level[i-1] * max_bytes_for_level_multiplier * fanout[i-1] The default value of fanout is 1, so that it is backward compatible. Test Plan: make check Reviewers: haobo, emayanke Reviewed By: emayanke CC: leveldb Differential Revision: https://reviews.facebook.net/D10863 | 21 May 2013, 20:50:20 UTC |
c3c13db | Haobo Xu | 21 May 2013, 04:28:09 UTC | [RocksDB] [Performance Bug] MemTable::Get Slow Summary: The merge operator diff introduced a performance problem in MemTable::Get. An exit condition is missed when the current key does not match the user key. This could lead to full memtable scan if the user key is not found. Test Plan: make check; db_bench Reviewers: dhruba Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10851 | 21 May 2013, 20:40:38 UTC |
3827403 | Mayank Agarwal | 21 May 2013, 18:27:23 UTC | Check to db_stress to not allow disable_wal and reopens set together Summary: db can't reopen safely with disable_wal set! Test Plan: make db_stress; run db_stress with disable_wal and reopens set and see error Reviewers: dhruba, vamsi Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10857 | 21 May 2013, 18:49:29 UTC |
839f6db | Haobo Xu | 25 March 2013, 17:56:48 UTC | [RocksDB] Fix PosixLogger and AutoRollLogger thread safety Summary: PosixLogger and AutoRollLogger do not seem to be thread safe. For PosixLogger, log_size_ is not atomically updated. For AutoRollLogger, the underlying logger_ might be deleted by one thread while still being accessed by another. Test Plan: make check Reviewers: kailiu, dhruba, heyongqiang Reviewed By: kailiu CC: leveldb, zshao, sheki Differential Revision: https://reviews.facebook.net/D9699 | 21 May 2013, 18:39:44 UTC |
15ccd10 | Mayank Agarwal | 20 May 2013, 21:45:22 UTC | A nit to db_stress to terminate generated value at proper length Summary: Will help while debugging if the generated value is truncated at proper length. Test Plan: make db_stress;/db_stress --max_key=10000 --db=/tmp/mcr --threads=1 --ops_per_thread=10000 Reviewers: dhruba, vamsi Reviewed By: vamsi Differential Revision: https://reviews.facebook.net/D10845 | 21 May 2013, 01:13:32 UTC |
8a59ed9 | Haobo Xu | 17 May 2013, 23:14:07 UTC | [RockdDB] fix build Summary: assert => ASSERT_TRUE Test Plan: make release; make check Reviewers: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10839 | 17 May 2013, 23:15:44 UTC |
e117430 | Abhishek Kona | 17 May 2013, 17:55:34 UTC | [RocksDB] Simplify StopWatch implementation Summary: Make stop watch a simple implementation, instead of subclass of a virtual class Allocate stop watches off the stack instead of heap. Code is more terse now. Test Plan: make all check, db_bench with --statistics=1 Reviewers: haobo, dhruba Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10809 | 17 May 2013, 17:55:34 UTC |
446151c | Abhishek Kona | 16 May 2013, 17:40:30 UTC | [Rocksdb] Remove unused double apis to record into histograms Summary: Statistics.h and histogram.h had double based api's to record values. Remove them as they are not used anywhere Test Plan: make all check Reviewers: haobo, dhruba Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10815 | 16 May 2013, 17:40:30 UTC |
8a48410 | Mayank Agarwal | 14 May 2013, 02:11:56 UTC | Enhance the ldb tool to support ttl databases Summary: ldb works with raw data from the database and needs to be aware of ttl-database to work with it meaningfully. '-ttl' option now tells it that. Also added onto the ldb_test.py test. This option may be specified alongwith put, get, scan or dump. There is no support to provide a ttl-value and it uses default forever because there is no use-case for this currently. Test Plan: make ldb_test; python tools/ldb_test.py Reviewers: dhruba, sheki, haobo, vamsi Reviewed By: sheki CC: leveldb Differential Revision: https://reviews.facebook.net/D10797 | 15 May 2013, 19:10:00 UTC |
8c9411c | Mayank Agarwal | 14 May 2013, 23:08:03 UTC | Modify build_detect_platform to run fbcode.*.* irrespective of $PATH Summary: This caused crash when the jenkins machine changed because the PATH variable is different. Test Plan: make Reviewers: dhruba, sheki Reviewed By: sheki CC: leveldb Differential Revision: https://reviews.facebook.net/D10803 | 15 May 2013, 05:09:01 UTC |
accd3de | Deon Nicholas | 10 May 2013, 17:40:10 UTC | Implemented StringAppendOperator and unit tests. Summary: Implemented the StringAppendOperator class (subclass of MergeOperator). Found in utilities/merge_operators/string_append/stringappend.{h,cc} It is a rocksdb Merge Operator that supports string/list concatenation with a configurable delimiter. The tests are found in .../stringappend_test.cc. It implements a map : key -> (list of strings), with core operations Append(list_key,val) and Get(list_key). Test Plan: 1. Navigate to your rocksdb repository 2. Execute: make stringappend_test (to compile) 3. Execute: ./stringappend_test (to run the tests) 4. Execute: make all check (to test the ENTIRE rocksdb codebase / regression) Reviewers: haobo, dhruba, zshao Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10737 | 13 May 2013, 22:09:42 UTC |
4ca3c67 | Haobo Xu | 12 May 2013, 09:36:59 UTC | [RocksDB] Cleanup compaction filter to use a class interface, instead of function pointer and additional context pointer. Summary: This diff replaces compaction_filter_args and CompactionFilter with a single compaction_filter parameter. It gives CompactionFilter better encapsulation and a similar look to Comparator and MergeOpertor, which improves consistency of the overall interface. The change is not backward compatible. Nevertheless, the two references in fbcode are not in production yet. Test Plan: make check Reviewers: dhruba Reviewed By: dhruba CC: leveldb, zshao Differential Revision: https://reviews.facebook.net/D10773 | 13 May 2013, 21:06:10 UTC |
73c0a33 | Haobo Xu | 03 May 2013, 22:26:12 UTC | [RocksDB] fix compaction filter trigger condition Summary: Currently, compaction filter is run on internal key older than the oldest snapshot, which is incorrect. Compaction filter should really be run on the most recent internal key when there is no external snapshot. Test Plan: make check; db_stress Reviewers: dhruba Reviewed By: dhruba Differential Revision: https://reviews.facebook.net/D10641 | 13 May 2013, 19:33:02 UTC |
3102628 | Mayank Agarwal | 10 May 2013, 21:19:47 UTC | Bring read_only into ttl Summary: added an argument to openttldb for read only and open the db in normal readonly mode if the arguments is set to true Test Plan: make ttl_test; ./ttl_test Reviewers: dhruba, haobo, vamsi, sheki Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10749 | 10 May 2013, 23:13:26 UTC |
8d58ecd | Abhishek Kona | 10 May 2013, 21:41:45 UTC | [RocksDB] Expose compaction stalls via db_statistics Test Plan: make check Reviewers: dhruba, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10575 | 10 May 2013, 21:41:45 UTC |
d98e563 | Abhishek Kona | 10 May 2013, 20:19:39 UTC | [RocksDB] Make Ticker Atomic Summary: Our ticker was not atomic. This was based on the assumption that we will increment counts at places only protected by Mutex. This is hard to program for and easy to make mistakes (mutex may be held a few layers above etc). Also this increases the instructions executed when the mutex is held. Test Plan: make check Reviewers: haobo, dhruba Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10569 | 10 May 2013, 20:19:39 UTC |
85cccc5 | Mayank Agarwal | 10 May 2013, 01:11:37 UTC | Replacing rocksdb by leveldb in Makefile Summary: Since we are keeping 'leveldb' instead of 'rocksdb' in third-party, this is only logical. Test Plan: make clean;make Reviewers: sheki, dhruba, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10719 | 10 May 2013, 01:39:25 UTC |
c605e12 | Mayank Agarwal | 10 May 2013, 00:33:27 UTC | Adding Sanity checks in Get and Iterator->value for ttl Summary: The 2 checks added will increase reliabilty and help in debugging Test Plan: make ttl_test;./ttl_test Reviewers: vamsi, dhruba, sheki, haobo Reviewed By: vamsi Differential Revision: https://reviews.facebook.net/D10713 | 10 May 2013, 01:13:27 UTC |
ff1a080 | Mayank Agarwal | 09 May 2013, 22:52:02 UTC | Correct path of db.h in utility_db.h Summary: Will not be caught properly from file in fbcode with old path. New path fixes it. Test Plan: make Reviewers: sheki, dhruba, haobo, vamsi Reviewed By: sheki Differential Revision: https://reviews.facebook.net/D10707 | 10 May 2013, 00:32:39 UTC |
a8d3aa2 | Dhruba Borthakur | 08 May 2013, 17:19:24 UTC | Assertion failure for L0-L1 compactions. Summary: For level-0 compactions, we try to find if can include more L0 files in the same compaction run. This causes the 'smallest' and 'largest' key to get extended to a larger range. But the suceeding call to ParentRangeInCompaction() was still using the earlier values of 'smallest' and 'largest', Because of this bug, a file in L1 can be part of two concurrent compactions: one L0-L1 compaction and the other L1-L2 compaction. This should not cause any data loss, but will cause an assertion failure with debug builds. Test Plan: make check Differential Revision: https://reviews.facebook.net/D10677 | 09 May 2013, 00:10:11 UTC |
988c20b | Abhishek Kona | 06 May 2013, 18:41:01 UTC | [RocksDB] Clear Archive WAL files Summary: WAL files are moved to archive directory and clear only at DB::Open. Can lead to a lot of space consumption in a Database. Added logic to periodically clear Archive Directory too. Test Plan: make all check + add unit test Reviewers: dhruba, heyongqiang Reviewed By: heyongqiang CC: leveldb Differential Revision: https://reviews.facebook.net/D10617 | 06 May 2013, 18:41:01 UTC |
3c4efc4 | Haobo Xu | 06 May 2013, 17:35:41 UTC | [RocksDB] fix build Summary: makefile change: LIBRARY => LIBOBJECTS thanks Abhishek for reproducing this locally. Test Plan: make release Reviewers: sheki CC: leveldb Task ID: # Blame Rev: | 06 May 2013, 17:35:41 UTC |
05e8854 | Haobo Xu | 21 March 2013, 22:59:47 UTC | [Rocksdb] Support Merge operation in rocksdb Summary: This diff introduces a new Merge operation into rocksdb. The purpose of this review is mostly getting feedback from the team (everyone please) on the design. Please focus on the four files under include/leveldb/, as they spell the client visible interface change. include/leveldb/db.h include/leveldb/merge_operator.h include/leveldb/options.h include/leveldb/write_batch.h Please go over local/my_test.cc carefully, as it is a concerete use case. Please also review the impelmentation files to see if the straw man implementation makes sense. Note that, the diff does pass all make check and truly supports forward iterator over db and a version of Get that's based on iterator. Future work: - Integration with compaction - A raw Get implementation I am working on a wiki that explains the design and implementation choices, but coding comes just naturally and I think it might be a good idea to share the code earlier. The code is heavily commented. Test Plan: run all local tests Reviewers: dhruba, heyongqiang Reviewed By: dhruba CC: leveldb, zshao, sheki, emayanke, MarkCallaghan Differential Revision: https://reviews.facebook.net/D9651 | 03 May 2013, 23:59:02 UTC |
37e97b1 | Mayank Agarwal | 03 May 2013, 21:25:12 UTC | Fix invalid-read to freed memory in ttl-iterator Summary: value function in ttl-iterator was returning string which would have been freed before its usage as a slice. Thanks valgrind! Test Plan: valgrind ./ttl_test Reviewers: dhruba, haobo, sheki, vamsi Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10635 | 03 May 2013, 22:09:16 UTC |
d786b25 | Mayank Agarwal | 15 April 2013, 20:33:13 UTC | Timestamp and TTL Wrapper for rocksdb Summary: When opened with DBTimestamp::Open call, timestamps are prepended to and stripped from the value during subsequent Put and Get calls respectively. The Timestamp is used to discard values in Get and custom compaction filter which have exceeded their TTL which is specified during Open. Have made a temporary change to Makefile to let us test with the temporary file TestTime.cc. Have also changed the private members of db_impl.h to protected to let them be inherited by the new class DBTimestamp Test Plan: make db_timestamp; TestTime.cc(will not check it in) shows how to use the apis currently, but I will write unit-tests shortly Reviewers: dhruba, vamsi, haobo, sheki, heyongqiang, vkrest Reviewed By: vamsi CC: zshao, xjin, vkrest, MarkCallaghan Differential Revision: https://reviews.facebook.net/D10311 | 02 May 2013, 23:34:42 UTC |
41cb922 | Abhishek Kona | 29 April 2013, 20:19:24 UTC | Allocate the LogReporter from heap. Summary: Summary: The current code has a bug that take address of stack allocated LogReporter. It is causing SIGSEGV because the stack address is no longer valid when referenced. Test Plan: Tested on prod. Reviewers: haobo, dhruba, heyongqiang Reviewed By: heyongqiang Differential Revision: https://reviews.facebook.net/D10557 | 29 April 2013, 20:19:24 UTC |
49fbd55 | Haobo Xu | 23 April 2013, 06:47:56 UTC | [RocksDB] Refactor table.cc to reduce code duplication and improve readability. Summary: In table.cc, the code section that reads in BlockContent and then put it into a Block, appears at least 4 times. This is too much duplication. BlockReader is much shorter after the change and reads way better. D10077 attempted that for index block read. This is a complete cleanup. Test Plan: make check; ./db_stress Reviewers: dhruba, sheki Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10527 | 29 April 2013, 16:43:36 UTC |
fb96ec1 | Abhishek Kona | 25 April 2013, 20:36:47 UTC | [RocksDB] Print all internally collected histograms in db_bench. Also print p95 Summary: $title Test Plan: make db_bench . run db_bench and check for expected output Reviewers: haobo, dhruba Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10521 | 25 April 2013, 20:36:47 UTC |
ae558b5 | Mayank Agarwal | 23 April 2013, 21:16:56 UTC | make clean in valgrind_test.sh first Summary: some changes require make clean to be done before make. auto_roll_Logger_test was failing on valgrind possibly due to this. Test Plan: valgrind ./auto_roll_logger_test Reviewers: sheki Reviewed By: sheki Differential Revision: https://reviews.facebook.net/D10509 | 23 April 2013, 21:25:19 UTC |
06d3487 | Haobo Xu | 22 April 2013, 17:14:54 UTC | [RocksDB] Print stack trace to stderr instead of stdio. Summary: Some scripts (like regression_build_test.sh) redirect stdio to a tmp file and delete it on exit. This would miss the stack trace output on segfault. Output to stderr would hopefully show us the stack trace in the continuous build output. Test Plan: ./signal_test, make check Reviewers: dhruba Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10485 | 23 April 2013, 03:38:02 UTC |
958b9c8 | Kai Liu | 23 April 2013, 01:10:28 UTC | Avoid global static initialization in Env::Default() Summary: Mark's task description from #2316777 Env::Default() comes from util/env_posix.cc This is a static global. static PosixEnv default_env; Env* Env::Default() { return &default_env; } ----- These globals assume default_env was initialized first. I don't think that is safe or correct to do (http://stackoverflow.com/questions/1005685/c-static-initialization-order) const string AutoRollLoggerTest::kTestDir( test::TmpDir() + "/db_log_test"); const string AutoRollLoggerTest::kLogFile( test::TmpDir() + "/db_log_test/LOG"); Env* AutoRollLoggerTest::env = Env::Default(); Test Plan: run make clean && make && make check But how can I know if it works in Ubuntu? Reviewers: MarkCallaghan, chip Reviewed By: chip CC: leveldb, dhruba, haobo Differential Revision: https://reviews.facebook.net/D10491 | 23 April 2013, 01:10:28 UTC |
eb6d139 | Haobo Xu | 22 April 2013, 22:20:20 UTC | [RocksDB] Move table.h to table/ Summary: - don't see a point exposing table.h to the public. - fixed make clean to remove also *.d files. Test Plan: make check; db_stress Reviewers: dhruba, heyongqiang Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10479 | 22 April 2013, 23:07:56 UTC |
344e832 | Abhishek Kona | 22 April 2013, 17:41:59 UTC | [RocksDB] Fix ReadMissing in db_bench Summary: D8943 Broke read_missing. Fix it by adding a "." at the end of the generated key Test Plan: generate, print and check the key has a "." Reviewers: dhruba, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10455 | 22 April 2013, 22:44:19 UTC |
3cb7bf8 | Dhruba Borthakur | 22 April 2013, 17:41:41 UTC | Initialize parameters in the constructor. Summary: RocksDB doesn't build on Ubuntu VM .. shoudl be fixed with this patch. g++ --version g++ (Ubuntu/Linaro 4.6.3-1ubuntu5) 4.6.3 util/env_posix.cc:68:24: sorry, unimplemented: non-static data member initializers util/env_posix.cc:68:24: error: ISO C++ forbids in-class initialization of non-const static member ‘use_os_buffer’ util/env_posix.cc:113:24: sorry, unimplemented: non-static data member initializers util/env_posix.cc:113:24: error: ISO C++ forbids in-class initialization of non-const static member ‘use_os_buffer Test Plan: make check Reviewers: sheki, leveldb Reviewed By: sheki Differential Revision: https://reviews.facebook.net/D10461 | 22 April 2013, 21:41:45 UTC |
b4243e5 | Haobo Xu | 18 April 2013, 04:30:21 UTC | [RocksDB] CompactionFilter cleanup Summary: - removed the compaction_filter_value from the callback interface. Restrict compaction filter to purging values. - modify some comments to reflect curent status. Test Plan: make check Reviewers: dhruba Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10335 | 20 April 2013, 17:26:51 UTC |
b1ff9ac | Mark Callaghan | 17 April 2013, 17:56:39 UTC | Add --writes_per_second rate limit, print p99.99 in histogram Summary: Adds the --writes_per_second rate limit for the readwhilewriting test. The purpose is to optionally avoid saturating storage with writes & compaction and test read response time when some writes are being done. Changes the histogram code to also print the p99.99 value Task ID: # Blame Rev: Test Plan: make check, ran db_bench with it Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: haobo Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D10305 | 20 April 2013, 17:26:51 UTC |
e0b6092 | Haobo Xu | 16 April 2013, 21:20:05 UTC | [RocksDB] fix build Summary: forgot to include signal_test.cc Test Plan: make check Reviewers: sheki Reviewed By: sheki CC: leveldb Differential Revision: https://reviews.facebook.net/D10281 | 20 April 2013, 17:26:51 UTC |
1255dcd | Haobo Xu | 11 April 2013, 17:54:35 UTC | [RocksDB] Add stacktrace signal handler Summary: This diff provides the ability to print out a stacktrace when the process receives certain signals. Currently, we enable this for the following signals (program error related): SIGILL SIGSEGV SIGBUS SIGABRT Application simply #include "util/stack_trace.h" and call leveldb::InstallStackTraceHandler() during initialization, if signal handler is needed. It's not done automatically when openning db, because it's the application(process)'s responsibility to install signal handler and some applications might already have their own (like fbcode). Sample output: Received signal 11 (Segmentation fault) #0 0x408ff0 ./signal_test() [0x408ff0] /home/haobo/rocksdb/util/signal_test.cc:4 #1 0x40827d ./signal_test() [0x40827d] /home/haobo/rocksdb/util/signal_test.cc:24 #2 0x7f8bb183172e /usr/local/fbcode/gcc-4.7.1-glibc-2.14.1/lib/libc.so.6(__libc_start_main+0x10e) [0x7f8bb183172e] ??:0 #3 0x408ebc ./signal_test() [0x408ebc] /home/engshare/third-party/src/glibc/glibc-2.14.1/glibc-2.14.1/csu/../sysdeps/x86_64/elf/start.S:113 Segmentation fault (core dumped) For each frame, we print the raw pointer, the symbol provided by backtrace_symbols (still not good enough), and the source file/line. Note that address translation is done by directly shell out to addr2line. ??:0 means addr2line fails to do the translation. Hacky, but I think it's good for now. Test Plan: signal_test.cc Reviewers: dhruba, MarkCallaghan Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10173 | 20 April 2013, 17:26:50 UTC |
a29fc17 | Haobo Xu | 16 April 2013, 00:30:43 UTC | [RocksDB] posix_logger does not compile on non-linux platform Summary: As title. Found out this when testing stack_trace.cc portability. Test Plan: make check; manual test 'non-linux' build by forcing OS_LINUX2 Reviewers: dhruba, heyongqiang Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10263 | 16 April 2013, 02:18:51 UTC |
7c6c3c0 | Abhishek Kona | 15 April 2013, 22:27:15 UTC | [Rockdsdb] Better Error messages. Closing db instead of deleting db Summary: A better error message. A local change. Did not look at other places where this could be done. Test Plan: compile Reviewers: dhruba, MarkCallaghan Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10251 | 15 April 2013, 22:27:15 UTC |
9b81d3c | Dhruba Borthakur | 15 April 2013, 20:25:53 UTC | Simplified level_ptrs by using a std:vector Summary: Simplified level_ptrs by using a std:vector Test Plan: make check Reviewers: sheki, emayanke Reviewed By: emayanke CC: leveldb Differential Revision: https://reviews.facebook.net/D10245 | 15 April 2013, 20:52:51 UTC |
013e9eb | Haobo Xu | 11 April 2013, 23:49:53 UTC | [RocksDB] [Performance] Speed up FindObsoleteFiles Summary: FindObsoleteFiles was slow, holding the single big lock, resulted in bad p99 behavior. Didn't profile anything, but several things could be improved: 1. VersionSet::AddLiveFiles works with std::set, which is by itself slow (a tree). You also don't know how many dynamic allocations occur just for building up this tree. switched to std::vector, also added logic to pre-calculate total size and do just one allocation 2. Don't see why env_->GetChildren() needs to be mutex proteced, moved to PurgeObsoleteFiles where mutex could be unlocked. 3. switched std::set to std:unordered_set, the conversion from vector is also inside PurgeObsoleteFiles I have a feeling this should pretty much fix it. Test Plan: make check; db_stress Reviewers: dhruba, heyongqiang, MarkCallaghan Reviewed By: dhruba CC: leveldb, zshao Differential Revision: https://reviews.facebook.net/D10197 | 12 April 2013, 18:29:27 UTC |
dae7379 | Abhishek Kona | 12 April 2013, 03:21:49 UTC | [RocksDB] Expose LDB functioanality as a library call - clients can build their own LDB binary with additional options Summary: Primarily a refactor. Introduced LDBTool interface to which customers can plug in their options and this will create their own version of ldb tool. Test Plan: made ldb tool and tried it. Reviewers: dhruba, heyongqiang Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D10191 | 12 April 2013, 03:21:49 UTC |