swh:1:snp:5115096b921df712aeb2a08114fede57fb3331fb
- HEAD
- refs/heads/2.2.fb.branch
- refs/heads/2.3.fb.branch
- refs/heads/2.4.fb.branch
- refs/heads/2.5.fb.branch
- refs/heads/2.6.fb.branch
- refs/heads/2.7
- refs/heads/2.7.fb.branch
- refs/heads/2.8.1.fb
- refs/heads/2.8.fb
- refs/heads/2.8.fb.trunk
- refs/heads/3.0.fb
- refs/heads/3.0.fb.branch
- refs/heads/3.1.fb
- refs/heads/3.10.fb
- refs/heads/3.11.fb
- refs/heads/3.12.fb
- refs/heads/3.13.fb
- refs/heads/3.2.fb
- refs/heads/3.3.fb
- refs/heads/3.4.fb
- refs/heads/3.5.fb
- refs/heads/3.6.fb
- refs/heads/3.7.fb
- refs/heads/3.8.fb
- refs/heads/3.9.fb
- refs/heads/4.0.fb
- refs/heads/4.1.fb
- refs/heads/4.10.fb
- refs/heads/4.11.fb
- refs/heads/4.12.fb
- refs/heads/4.13.fb
- refs/heads/4.2.fb
- refs/heads/4.3.fb
- refs/heads/4.4.fb
- refs/heads/4.5.fb
- refs/heads/4.6.fb
- refs/heads/4.7.fb
- refs/heads/4.8.fb
- refs/heads/4.9.fb
- refs/heads/5.0.fb
- refs/heads/5.1.fb
- refs/heads/5.10.fb
- refs/heads/5.11.fb
- refs/heads/5.12.fb
- refs/heads/5.13.fb
- refs/heads/5.13.fb.myrocks
- refs/heads/5.14.fb
- refs/heads/5.14.fb.myrocks
- refs/heads/5.15.fb
- refs/heads/5.16.fb
- refs/heads/5.17.fb
- refs/heads/5.17.fb.myrocks
- refs/heads/5.18.fb
- refs/heads/5.2.fb
- refs/heads/5.3.fb
- refs/heads/5.4.fb
- refs/heads/5.5.fb
- refs/heads/5.6.fb
- refs/heads/5.7.fb
- refs/heads/5.7.fb.myrocks
- refs/heads/5.8.3
- refs/heads/5.8.fb
- refs/heads/5.9.fb
- refs/heads/5.9.fb.myrocks
- refs/heads/6.0.fb
- refs/heads/6.0.fb.myrocks
- refs/heads/6.1.fb
- refs/heads/6.1.fb.myrocks
- refs/heads/6.1.fb.prod201905
- refs/heads/6.10.fb
- refs/heads/6.11.fb
- refs/heads/6.12.fb
- refs/heads/6.13.fb
- refs/heads/6.13.fb.laser
- refs/heads/6.14.fb
- refs/heads/6.14.fb.laser
- refs/heads/6.15.fb
- refs/heads/6.16.fb
- refs/heads/6.17.fb
- refs/heads/6.2.fb
- refs/heads/6.3.fb
- refs/heads/6.3.fb.myrocks
- refs/heads/6.3.fb.myrocks2
- refs/heads/6.3fb
- refs/heads/6.4.fb
- refs/heads/6.5.fb
- refs/heads/6.6.fb
- refs/heads/6.7.fb
- refs/heads/6.8.fb
- refs/heads/6.9.fb
- refs/heads/adaptive
- refs/heads/ajkr-patch-1
- refs/heads/blob_shadow
- refs/heads/bottom-pri-level
- refs/heads/bugfix-build-detect
- refs/heads/checksum_readahead_mmap_fix
- refs/heads/fb-config
- refs/heads/feature/debug-rocksdbjavastatic
- refs/heads/feature/travis-arm64
- refs/heads/fix-release-notes
- refs/heads/fix-write-batch-comment
- refs/heads/format_compatible_4
- refs/heads/getmergeops
- refs/heads/gh-pages-old
- refs/heads/history-update
- refs/heads/hotfix/lambda-capture
- refs/heads/improve-support
- refs/heads/jay-stress
- refs/heads/katherinez-patch-1
- refs/heads/katherinez-patch-2
- refs/heads/master
- refs/heads/pr-sanity-check-as-GHAction
- refs/heads/pr/6062
- refs/heads/ramvadiv-patch-1
- refs/heads/scaffold
- refs/heads/siying-patch-1
- refs/heads/siying-patch-10
- refs/heads/siying-patch-2
- refs/heads/siying-patch-3
- refs/heads/siying-patch-4
- refs/heads/skip_memtable_flush
- refs/heads/testing_ppc_build
- refs/heads/tests
- refs/heads/unused-var
- refs/heads/v6.6.4
- refs/heads/yiwu_stackable
- refs/heads/yuslepukhin
- refs/remotes/origin/5.13.fb
- refs/tags/2.5.fb
- refs/tags/2.6.fb
- refs/tags/3.0.fb
- refs/tags/do-not-use-me2
- refs/tags/rocksdb-3.1
- refs/tags/rocksdb-3.10.2
- refs/tags/rocksdb-3.11
- refs/tags/rocksdb-3.11.1
- refs/tags/rocksdb-3.11.2
- refs/tags/rocksdb-3.2
- refs/tags/rocksdb-3.3
- refs/tags/rocksdb-3.4
- refs/tags/rocksdb-3.5
- refs/tags/rocksdb-3.5.1
- refs/tags/rocksdb-3.6.1
- refs/tags/rocksdb-3.6.2
- refs/tags/rocksdb-3.7
- refs/tags/rocksdb-3.8
- refs/tags/rocksdb-3.9
- refs/tags/rocksdb-3.9.1
- refs/tags/rocksdb-4.1
- refs/tags/rocksdb-5.10.2
- refs/tags/rocksdb-5.10.3
- refs/tags/rocksdb-5.10.4
- refs/tags/rocksdb-5.11.2
- refs/tags/rocksdb-5.11.3
- refs/tags/rocksdb-5.14.3
- refs/tags/rocksdb-5.2.1
- refs/tags/rocksdb-5.3.3
- refs/tags/rocksdb-5.3.4
- refs/tags/rocksdb-5.3.5
- refs/tags/rocksdb-5.3.6
- refs/tags/rocksdb-5.4.10
- refs/tags/rocksdb-5.4.5
- refs/tags/rocksdb-5.4.6
- refs/tags/rocksdb-5.5.2
- refs/tags/rocksdb-5.5.3
- refs/tags/rocksdb-5.5.4
- refs/tags/rocksdb-5.5.5
- refs/tags/rocksdb-5.5.6
- refs/tags/rocksdb-5.6.1
- refs/tags/rocksdb-5.6.2
- refs/tags/rocksdb-5.7.1
- refs/tags/rocksdb-5.7.2
- refs/tags/rocksdb-5.7.3
- refs/tags/rocksdb-5.7.5
- refs/tags/rocksdb-5.8.6
- refs/tags/rocksdb-5.8.7
- refs/tags/rocksdb-5.8.8
- refs/tags/rocksdb-5.9.2
- refs/tags/v4.0
- refs/tags/v4.1
- refs/tags/v5.10.2
- refs/tags/v5.10.3
- refs/tags/v5.10.4
- refs/tags/v5.11.2
- refs/tags/v5.11.3
- refs/tags/v5.13.3
- refs/tags/v5.14.3
- refs/tags/v5.15.10
- refs/tags/v5.18.3
- refs/tags/v5.2.1
- refs/tags/v5.3.3
- refs/tags/v5.3.4
- refs/tags/v5.3.5
- refs/tags/v5.3.6
- refs/tags/v5.4.10
- refs/tags/v5.4.5
- refs/tags/v5.4.6
- refs/tags/v5.5.2
- refs/tags/v5.5.3
- refs/tags/v5.5.4
- refs/tags/v5.5.5
- refs/tags/v5.5.6
- refs/tags/v5.6.1
- refs/tags/v5.6.2
- refs/tags/v5.7.1
- refs/tags/v5.7.2
- refs/tags/v5.7.3
- refs/tags/v5.7.5
- refs/tags/v5.8.6
- refs/tags/v5.8.7
- refs/tags/v5.8.8
- refs/tags/v5.9.2
- refs/tags/v6.0.1
- refs/tags/v6.0.2
- refs/tags/v6.1.1
- refs/tags/v6.1.2
- refs/tags/v6.10.1
- refs/tags/v6.10.2
- refs/tags/v6.11.4
- refs/tags/v6.11.6
- refs/tags/v6.12.6
- refs/tags/v6.12.7
- refs/tags/v6.13.2
- refs/tags/v6.13.3
- refs/tags/v6.14.5
- refs/tags/v6.14.6
- refs/tags/v6.15.4
- refs/tags/v6.2.2
- refs/tags/v6.2.4
- refs/tags/v6.3.6
- refs/tags/v6.4.6
- refs/tags/v6.5.2
- refs/tags/v6.5.3
- refs/tags/v6.6.3
- refs/tags/v6.6.4
- refs/tags/v6.7.3
- refs/tags/v6.8.1
- v6.15.2
- v5.8
- v5.5.1
- v5.4.7
- v5.18.4
- v5.17.2
- v5.16.6
- v5.14.2
- v5.13.4
- v5.13.2
- v5.13.1
- v5.12.5
- v5.12.4
- v5.12.3
- v5.12.2
- v5.1.4
- v5.1.3
- v5.1.2
- v5.0.2
- v5.0.1
- v4.9
- v4.8
- v4.6.1
- v4.5.1
- v4.4.1
- v4.4
- v4.3.1
- v4.3
- v4.2
- v4.13.5
- v4.13
- v4.11.2
- v3.9
- v3.8
- v3.7
- v3.6.1
- v3.5
- v3.4
- v3.3
- v3.2
- v3.13.1
- v3.13
- v3.12.1
- v3.12
- v3.11
- v3.10
- v3.1
- v3.0
- v2.8
- v2.7
- v2.6
- v2.5
- v2.4
- v2.3
- v2.2
- v2.1
- v2.0
- v1.5.9.1
- v1.5.8.2
- v1.5.8.1
- v1.5.8
- v1.5.7
- rocksdb-5.8
- rocksdb-5.4.7
- rocksdb-5.1.4
- rocksdb-5.1.3
- rocksdb-5.1.2
- rocksdb-5.0.2
- rocksdb-5.0.1
- rocksdb-4.9
- rocksdb-4.8
- rocksdb-4.6.1
- rocksdb-4.5.1
- rocksdb-4.4.1
- rocksdb-4.4
- rocksdb-4.3.1
- rocksdb-4.3
- rocksdb-4.2
- rocksdb-4.13.5
- rocksdb-4.13
- rocksdb-4.11.2
- rocksdb-3.10.1
- 2.8.fb
- 2.7.fb
- 2.4.fb
- 2.3.fb
- 2.2.fb
- 2.1.fb
- 2.0.fb
- 1.5.9.fb
- 1.5.9.2.fb
- 1.5.9.1.fb
- 1.5.8.fb
- 1.5.8.2.fb
- 1.5.8.1.fb
- 1.5.7.fb
Permalinks
To reference or cite the objects present in the Software Heritage archive, permalinks based on SoftWare Hash IDentifiers (SWHIDs) must be used.
Select below a type of object currently browsed in order to display its associated SWHID and permalink.
Revision | Author | Date | Message | Commit Date |
---|---|---|---|---|
812c461 | Yueh-Hsuan Chiang | 19 May 2015, 23:38:31 UTC | Dump db stats in WARN level Summary: Dump db stats in WARN level Test Plan: run db_bench and verify the LOG Reviewers: igor, MarkCallaghan Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38691 | 20 May 2015, 01:42:17 UTC |
b588505 | Yueh-Hsuan Chiang | 20 May 2015, 01:41:28 UTC | Update HISTORY.md for GetThreadList() update. Summary: Update HISTORY.md for GetThreadList() update. Test Plan: no code change Reviewers: sdong, rven, anthony, krishnanm86, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38685 | 20 May 2015, 01:41:57 UTC |
944043d | Mark Callaghan | 19 May 2015, 23:19:30 UTC | Add --wal_bytes_per_sync for db_bench and more IO stats Summary: See https://gist.github.com/mdcallag/89ebb2b8cbd331854865 for the IO stats. I added "Cumulative compaction:" and "Interval compaction:" lines. The IO rates can be confusing. Rates fro per-level stats lines, Wr(MB/s) & Rd(MB/s), are computed using the duration of the compaction job. If the job reads 10MB, writes 9MB and the job (IO & merging) takes 1 second then the rates are 10MB/s for read and 9MB/s for writes. The IO rates in the Cumulative compaction line uses the total uptime. The IO rates in the Interval compaction line uses the interval uptime. So these Cumalative & Interval compaction IO rates cannot be compared to the per-level IO rates. But both forms of the rates are useful for debugging perf. Task ID: # Blame Rev: Test Plan: run db_bench Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D38667 | 19 May 2015, 23:19:30 UTC |
d5de04d | Karthikeyan Radhakrishnan | 19 May 2015, 21:19:11 UTC | Update history for 3.11 Flipped the unreleased section to 3.11 | 19 May 2015, 21:19:11 UTC |
08b6b37 | Igor Canadi | 19 May 2015, 18:45:01 UTC | FORCE_GIT_SHA Summary: In third-party2 build we need to force git sha because we're compiling from a different git repositry. Test Plan: `FORCE_GIT_SHA=igor make` Reviewers: kradhakrishnan, sdong Reviewed By: kradhakrishnan Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38679 | 19 May 2015, 18:45:01 UTC |
04feaee | Igor Canadi | 19 May 2015, 17:59:30 UTC | Fix comparison between signed and usigned integers Summary: Not sure why this fails on some compilers and doesn't on others. Test Plan: none Reviewers: meyering, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38673 | 19 May 2015, 17:59:30 UTC |
4a855c0 | Igor Canadi | 19 May 2015, 00:03:59 UTC | Add an option wal_bytes_per_sync to control sync_file_range for WAL files Summary: sync_file_range is not always asyncronous and thus can block writes if we do this for WAL in the foreground thread. See more here: http://yoshinorimatsunobu.blogspot.com/2014/03/how-syncfilerange-really-works.html Some users don't want us to call sync_file_range on WALs. Some other do. Thus, I'm adding a separate option wal_bytes_per_sync to control calling sync_file_range on WAL files. bytes_per_sync will apply only to table files now. Test Plan: no more sync_file_range for WAL as evidenced by strace Reviewers: yhchiang, rven, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38253 | 19 May 2015, 00:03:59 UTC |
b0fdda4 | Igor Canadi | 18 May 2015, 22:34:33 UTC | Allow flushes to run in parallel with manual compaction Summary: As title. I spent some time thinking about it and I don't think there should be any issue with running manual compaction and flushes in parallel Test Plan: make check works Reviewers: rven, yhchiang, sdong Reviewed By: yhchiang, sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38355 | 18 May 2015, 22:34:33 UTC |
74f3832 | Yueh-Hsuan Chiang | 18 May 2015, 20:44:31 UTC | Fixed compile errors due to some gcc does not have std::map::emplace Summary: Fixed the following compile errors due to some gcc does not have std::map::emplace util/thread_status_impl.cc: In static member function ‘static std::map<std::basic_string<char>, long unsigned int> rocksdb::ThreadStatus::InterpretOperationProperties(rocksdb::ThreadStatus::OperationType, const uint64_t*)’: util/thread_status_impl.cc:88:20: error: ‘class std::map<std::basic_string<char>, long unsigned int>’ has no member named ‘emplace’ util/thread_status_impl.cc:90:20: error: ‘class std::map<std::basic_string<char>, long unsigned int>’ has no member named ‘emplace’ util/thread_status_impl.cc:94:20: error: ‘class std::map<std::basic_string<char>, long unsigned int>’ has no member named ‘emplace’ util/thread_status_impl.cc:96:20: error: ‘class std::map<std::basic_string<char>, long unsigned int>’ has no member named ‘emplace’ util/thread_status_impl.cc:98:20: error: ‘class std::map<std::basic_string<char>, long unsigned int>’ has no member named ‘emplace’ util/thread_status_impl.cc:101:20: error: ‘class std::map<std::basic_string<char>, long unsigned int>’ has no member named ‘emplace’ make: *** [util/thread_status_impl.o] Error 1 Test Plan: make db_bench Reviewers: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38643 | 18 May 2015, 20:48:56 UTC |
0c8017d | stash93 | 18 May 2015, 20:44:52 UTC | Remove duplicated code Summary: Call Flush() function instead Test Plan: make all check Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D38583 | 18 May 2015, 20:44:52 UTC |
fb5bdbf | sdong | 18 May 2015, 18:47:16 UTC | DBTest.DynamicLevelMaxBytesCompactRange: make sure L0 is not empty before running compact range Summary: DBTest.DynamicLevelMaxBytesCompactRange needs to make sure L0 is not empty to properly cover the code paths we want to cover. However, current codes have a bug that might leave the condition not held. Improve the test to ensure it. Test Plan: Run the test in an environment that is used to fail. Also run it many times. Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D38631 | 18 May 2015, 18:49:45 UTC |
6fa7085 | sdong | 15 April 2015, 04:45:20 UTC | CompactRange skips levels 1 to base_level -1 for dynamic level base size Summary: CompactRange() now is much more expensive for dynamic level base size as it goes through all the levels. Skip those not used levels between level 0 an base level. Test Plan: Run all unit tests Reviewers: yhchiang, rven, anthony, kradhakrishnan, igor Reviewed By: igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D37125 | 18 May 2015, 17:54:11 UTC |
3f0867c | Yueh-Hsuan Chiang | 16 May 2015, 06:22:22 UTC | Allow GetThreadList to report Flush properties. Summary: Allow GetThreadList to report Flush properties, which includes: * job id * number of bytes that has been written since flush started. * total size of input mem-tables Test Plan: ./db_bench --threads=30 --num=1000000 --benchmarks=fillrandom --thread_status_per_interval=100 --value_size=1000 Sample output from db_bench which tracks same flush job ThreadID ThreadType cfName Operation ElapsedTime Stage State OperationProperties 140213879898240 High Pri default Flush 5789 us FlushJob::WriteLevel0Table BytesMemtables 4112835 | BytesWritten 577104 | JobID 8 | ThreadID ThreadType cfName Operation ElapsedTime Stage State OperationProperties 140213879898240 High Pri default Flush 30.634 ms FlushJob::WriteLevel0Table BytesMemtables 4112835 | BytesWritten 1734865 | JobID 8 | Reviewers: rven, igor, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38505 | 16 May 2015, 06:22:22 UTC |
a66f643 | Yueh-Hsuan Chiang | 15 May 2015, 22:55:20 UTC | Use a better way to initialize ThreadStatus::kNumOperationProperties. Summary: Use a better way to initialize ThreadStatus::kNumOperationProperties. Test Plan: make Reviewers: sdong, rven, anthony, krishnanm86, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38547 | 15 May 2015, 22:55:20 UTC |
7413306 | Igor Canadi | 15 May 2015, 21:14:40 UTC | Take a chance on a random file when choosing compaction Summary: When trying to compact entire database with SuggestCompactRange(), we'll first try the left-most files. This is pretty bad, because: 1) the left part of LSM tree will be overly compacted, but right part will not be touched 2) First compaction will pick up the left-most file. Second compaction will try to pick up next left-most, but this will not be possible, because there's a big chance that second's file range on N+1 level is already being compacted. I observe both of those problems when running Mongo+RocksDB and trying to compact the DB to clean up tombstones. I'm unable to clean them up :( This diff adds a bit of randomness into choosing a file. First, it chooses a file at random and tries to compact that one. This should solve both problems specified here. Test Plan: make check Reviewers: yhchiang, rven, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38379 | 15 May 2015, 21:14:40 UTC |
8c52788 | sdong | 13 May 2015, 21:32:47 UTC | Use version defined in Makefile in rocksdb_build_git_sha Summary: Now rocksdb_build_git_sha is determined from "git sha". It is hard if the release is not from the repository directly but from a source code copy. Change to use the versions given in Makefile. Test Plan: Run "make util/build_version.cc" Reviewers: kradhakrishnan, rven, meyering, igor Reviewed By: igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D38451 | 15 May 2015, 20:51:57 UTC |
5aad881 | sdong | 13 May 2015, 22:26:02 UTC | DBTest.DynamicLevelMaxBytesBase2: remove an unnecesary check Summary: DBTest.DynamicLevelMaxBytesBase2 has a check that is not necessary and may fail. Remove it, and add two unrelated check. Test Plan: Run the test Reviewers: yhchiang, rven, kradhakrishnan, anthony, igor Reviewed By: igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D38457 | 14 May 2015, 16:22:43 UTC |
ec43a8b | sdong | 13 May 2015, 17:29:49 UTC | Universal Compaction with multiple levels won't allocate up to output size Summary: Universal compactions with multiple levels should use file preallocation size based on file size if output level is not level 0 Test Plan: Run all tests. Reviewers: igor Reviewed By: igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D38439 | 13 May 2015, 21:15:46 UTC |
714fcc0 | Yueh-Hsuan Chiang | 13 May 2015, 19:26:07 UTC | Make ThreadStatus::InterpretOperationProperties take const uint64_t* Summary: Make ThreadStatus::InterpretOperationProperties take const uint64_t* Test Plan: make make OPT=-DROCKSDB_LITE shared_lib Reviewers: igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38445 | 13 May 2015, 19:26:07 UTC |
bc68bd5 | sdong | 13 May 2015, 00:40:27 UTC | db_bench to support rate limiter Summary: Add --rate_limiter_bytes_per_sec to db_bench to allow rater limit to disk Test Plan: Run ./db_bench --benchmarks=fillseq --num=30000000 --rate_limiter_bytes_per_sec=3000000 --num_multi_db=8 -disable_wal And see io_stats to have the rate limited. Reviewers: yhchiang, rven, anthony, kradhakrishnan, igor Reviewed By: igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D38385 | 13 May 2015, 17:03:41 UTC |
df1f87a | Yueh-Hsuan Chiang | 12 May 2015, 23:18:58 UTC | Fixed compile error in db/column_family.cc Summary: Fixed the following compile error in db/column_family.cc db/column_family.cc:633:33: error: ‘ASSERT_GT’ was not declared in this scope 16:14:45 ASSERT_GT(listeners.size(), 0U); Test Plan: make db_test Reviewers: igor, sdong, rven Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38367 | 12 May 2015, 23:20:03 UTC |
14431e9 | Yueh-Hsuan Chiang | 12 May 2015, 23:10:23 UTC | Fixed a bug in EventListener::OnCompactionCompleted(). Summary: Fixed a bug in EventListener::OnCompactionCompleted() that returns incorrect list of input / output file names. Test Plan: Extend existing test in listener_test.cc Reviewers: sdong, rven, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38349 | 12 May 2015, 23:10:23 UTC |
dbd95b7 | Igor Canadi | 12 May 2015, 22:53:55 UTC | Add more table properties to EventLogger Summary: Example output: {"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}} Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter. Test Plan: make check + check out the output in the log Reviewers: sdong, rven, yhchiang Reviewed By: yhchiang Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38343 | 12 May 2015, 22:53:55 UTC |
b588176 | Igor Canadi | 12 May 2015, 18:16:25 UTC | Reset parent_index and base_index when picking files marked for compaction Summary: This caused a crash of our MongoDB + RocksDB instance. PickCompactionBySize() sets its own parent_index. We never reset this parent_index when picking PickFilesMarkedForCompactionExperimental(). So we might end up doing SetupOtherInputs() with parent_index that was set by PickCompactionBySize, although we're using compaction calculated using PickFilesMarkedForCompactionExperimental. Test Plan: Added a unit test that fails with assertion on master. Reviewers: yhchiang, rven, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38337 | 12 May 2015, 18:16:25 UTC |
711465c | agiardullo | 11 May 2015, 21:51:51 UTC | API to fetch from both a WriteBatchWithIndex and the db Summary: Added a couple functions to WriteBatchWithIndex to make it easier to query the value of a key including reading pending writes from a batch. (This is needed for transactions). I created write_batch_with_index_internal.h to use to store an internal-only helper function since there wasn't a good place in the existing class hierarchy to store this function (and it didn't seem right to stick this function inside WriteBatchInternal::Rep). Since I needed to access the WriteBatchEntryComparator, I moved some helper classes from write_batch_with_index.cc into write_batch_with_index_internal.h/.cc. WriteBatchIndexEntry, ReadableWriteBatch, and WriteBatchEntryComparator are all unchanged (just moved to a different file(s)). Test Plan: Added new unit tests. Reviewers: rven, yhchiang, sdong, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38037 | 11 May 2015, 21:51:51 UTC |
3996fff | Igor Canadi | 09 May 2015, 18:03:36 UTC | Fix clang build - add override Summary: In new clang we need to add override to every overriden function Test Plan: none Reviewers: rven Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D38259 | 09 May 2015, 18:04:14 UTC |
d978139 | Igor Canadi | 09 May 2015, 02:37:02 UTC | SuggestCompactRange() is manual compaction Summary: When reporting compaction that was started because of SuggestCompactRange() we should treat it as manual compaction. Test Plan: none Reviewers: yhchiang, rven Reviewed By: rven Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38139 | 09 May 2015, 02:37:02 UTC |
2fe24e4 | Yueh-Hsuan Chiang | 08 May 2015, 18:02:47 UTC | Don't treat warnings as error when building rocksdbjavastatic Summary: Don't treat warnings as error when building rocksdbjavastatic Test Plan: make rocksdbjavastatic -j32 Reviewers: rven, fyrz, adamretter, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38187 | 08 May 2015, 18:02:47 UTC |
beda81c | Igor Canadi | 08 May 2015, 00:47:19 UTC | rm -rf in make clean Summary: Without this I get bunch of questions when I run `make clean` Test Plan: no more questions! Reviewers: rven, yhchiang, meyering, anthony Reviewed By: meyering, anthony Subscribers: meyering, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38145 | 08 May 2015, 00:47:19 UTC |
50eab9c | Igor Canadi | 08 May 2015, 00:39:19 UTC | Fix BackupEngine Summary: In D28521 we removed GarbageCollect() from BackupEngine's constructor. The reason was that opening BackupEngine on HDFS was very slow and in most cases we didn't have any garbage. We allowed the user to call GarbageCollect() when it detects some garbage files in his backup directory. Unfortunately, this left us vulnerable to an interesting issue. Let's say we started a backup and copied files {1, 3} but the backup failed. On another host, we restore DB from backup and generate {1, 3, 5}. Since {1, 3} is already there, we will not overwrite. However, these files might be from a different database so their contents might be different. See internal task t6781803 for more info. Now, when we're copying files and we discover a file already there, we check: 1. if the file is not referenced from any backups, we overwrite the file. 2. if the file is referenced from other backups AND the checksums don't match, we fail the backup. This will only happen if user is using a single backup directory for backing up two different databases. 3. if the file is referenced from other backups AND the checksums match, it's all good. We skip the copy and go copy the next file. Test Plan: Added new test to backupable_db_test. The test fails before this patch. Reviewers: sdong, rven, yhchiang Reviewed By: yhchiang Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37599 | 08 May 2015, 00:39:19 UTC |
962f8ba | Igor Canadi | 07 May 2015, 20:26:18 UTC | Bump to 3.11 Summary: as title Test Plan: none Reviewers: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38175 | 07 May 2015, 20:27:14 UTC |
37bbd31 | Jörg Maier | 07 May 2015, 06:14:47 UTC | Merge pull request #601 from adamretter/jdb-bench Small fixes to Java benchmark | 07 May 2015, 06:14:47 UTC |
77a5a54 | Yueh-Hsuan Chiang | 07 May 2015, 05:50:35 UTC | Allow GetThreadList() to report basic compaction operation properties. Summary: Now we're able to show more details about a compaction in GetThreadList() :) This patch allows GetThreadList() to report basic compaction operation properties. Basic compaction properties include: 1. job id 2. compaction input / output level 3. compaction property flags (is_manual, is_deletion, .. etc) 4. total input bytes 5. the number of bytes has been read currently. 6. the number of bytes has been written currently. Flush operation properties will be done in a seperate diff. Test Plan: /db_bench --threads=30 --num=1000000 --benchmarks=fillrandom --thread_status_per_interval=1 Sample output of tracking same job: ThreadID ThreadType cfName Operation ElapsedTime Stage State OperationProperties 140664171987072 Low Pri default Compaction 31.357 ms CompactionJob::FinishCompactionOutputFile BaseInputLevel 1 | BytesRead 2264663 | BytesWritten 1934241 | IsDeletion 0 | IsManual 0 | IsTrivialMove 0 | JobID 277 | OutputLevel 2 | TotalInputBytes 3964158 | ThreadID ThreadType cfName Operation ElapsedTime Stage State OperationProperties 140664171987072 Low Pri default Compaction 59.440 ms CompactionJob::FinishCompactionOutputFile BaseInputLevel 1 | BytesRead 2264663 | BytesWritten 1934241 | IsDeletion 0 | IsManual 0 | IsTrivialMove 0 | JobID 277 | OutputLevel 2 | TotalInputBytes 3964158 | ThreadID ThreadType cfName Operation ElapsedTime Stage State OperationProperties 140664171987072 Low Pri default Compaction 226.375 ms CompactionJob::Install BaseInputLevel 1 | BytesRead 3958013 | BytesWritten 3621940 | IsDeletion 0 | IsManual 0 | IsTrivialMove 0 | JobID 277 | OutputLevel 2 | TotalInputBytes 3964158 | Reviewers: sdong, rven, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37653 | 07 May 2015, 05:51:06 UTC |
65fe1cf | Igor Canadi | 06 May 2015, 02:01:12 UTC | Cleanup CompactionJob Summary: Couple changes: 1. instead of SnapshotList, just take a vector of snapshots 2. don't take a separate parameter is_snapshots_supported. If there are snapshots in the list, that means they are supported. I actually think we should get rid of this notion of snapshots not being supported. 3. don't pass in mutable_cf_options as a parameter. Lifetime of mutable_cf_options is a bit tricky to maintain, so it's better to not pass it in for the whole compaction job. We only really need it when we install the compaction results. Test Plan: make check Reviewers: sdong, rven, yhchiang Reviewed By: yhchiang Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D36627 | 06 May 2015, 02:01:12 UTC |
df4130a | Laurent Demailly | 05 May 2015, 23:54:47 UTC | fix crashes in stats and compaction filter for db_ttl_impl Summary: fix crashes in stats and compaction filter for db_ttl_impl Test Plan: Ran build with lots of debugging https://reviews.facebook.net/differential/diff/194175/ Reviewers: yhchiang, igor, rven Reviewed By: igor Subscribers: rven, dhruba Differential Revision: https://reviews.facebook.net/D38001 | 05 May 2015, 23:54:47 UTC |
7ea7694 | Venkatesh Radhakrishnan | 05 May 2015, 22:59:02 UTC | Fix flakiness in column_family_test Summary: Fixes #6840824, running "make check" on centos6 hits a deadlock in column_family_test Test Plan: seq 10000 | parallel --gnu --eta 't=/dev/shm/rdb-{}; rm -rf $t; mkdir $t && export TEST_TMPDIR=$t; ./column_family_test > $t/log-{}' Made the test deterministic by narrrowing the window for the flush. Reviewers: igor, meyering Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38079 | 05 May 2015, 22:59:02 UTC |
a2c4cc7 | Igor Canadi | 05 May 2015, 18:54:13 UTC | Don't treat warnings as error when building release Summary: This will reduce errors reported by open source by 90%. Developers should have warnings break their compile. Users shouldn't. Test Plan: make dbg: g++ -g -W -Wextra -Wall -Wsign-compare -Wshadow -Wno-unused-parameter -Werror -I. -I./include -std=c++11 -DROCKSDB_PLATFORM_POSIX -DOS_LINUX -fno-builtin-memcmp -DROCKSDB_FALLOCATE_PRESENT -DSNAPPY -DGFLAGS=gflags -DZLIB -DBZIP2 -march=native -isystem ./third-party/gtest-1.7.0/fused-src -Woverloaded-virtual -Wn on-virtual-dtor -Wno-missing-field-initializers -c db/repair.cc -o db/repair.o make all: g++ -g -W -Wextra -Wall -Wsign-compare -Wshadow -Wno-unused-parameter -Werror -I. -I./include -std=c++11 -DROCKSDB_PLATFORM_POSIX -DOS_LINUX -fno-builtin-memcmp -DROCKSDB_FALLOCATE_PRESENT -DSNAPPY -DGFLAGS=gflags -DZLIB -DBZIP2 -march=native -isystem ./third-party/gtest-1.7.0/fused-src -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -c db/repair.cc -o db/repair.o make static_lib: g++ -g -W -Wextra -Wall -Wsign-compare -Wshadow -Wno-unused-parameter -I. -I./include -std=c++11 -DROCKSDB_PLATFORM_POSIX -DOS_LINUX -fno-builtin-memcmp -DROCKSDB_FALLOCATE_PRESENT -DSNAPPY -DGFLAGS=gflags -DZLIB -DBZIP2 -march=native -isystem ./third-party/gtest-1.7.0/fused-src -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer -DNDEBUG -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -c db/repair.cc -o db/repair.o Reviewers: sdong, yhchiang, rven Reviewed By: rven Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38031 | 05 May 2015, 18:54:13 UTC |
9aa011f | Liangjun Feng | 05 May 2015, 16:57:47 UTC | Optimize GetRange Function Summary: Optimize GetRange Function by checking the level of the files Test Plan: pass make all check Reviewers: rven, yhchiang, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37977 | 05 May 2015, 16:57:47 UTC |
36a7408 | Igor Canadi | 05 May 2015, 15:57:34 UTC | Fix UNLIKELY parenthesis Summary: Ooops :) status.ok() is acutally highly likely :) Test Plan: none Reviewers: rven, yhchiang, anthony Reviewed By: anthony Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D38043 | 05 May 2015, 15:57:34 UTC |
9bdbaab | Igor Canadi | 04 May 2015, 22:27:59 UTC | Update USERS.md Fix formatting | 04 May 2015, 22:27:59 UTC |
2ab7065 | Jim Meyering | 02 May 2015, 20:19:10 UTC | build: avoid unused-variable warning Summary: [noticed a new warning when building with the very latest gcc] * db/memtablerep_bench.cc (FLAGS_env): Remove declaration of unused varaible, to avoid this warning/error: db/memtablerep_bench.cc:135:22: error: ‘FLAGS_env’ defined but not\ used [-Werror=unused-variable] static rocksdb::Env* FLAGS_env = rocksdb::Env::Default(); ^ Test Plan: compile Reviewers: ljin, rven, igor.sugak, yhchiang, sdong, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37983 | 02 May 2015, 20:19:10 UTC |
8804434 | Mark Callaghan | 02 May 2015, 14:46:12 UTC | Add Size-GB column to benchmark reports Summary: See https://gist.github.com/mdcallag/b867ee051d765760be0d for a sample Task ID: # Blame Rev: Test Plan: Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37971 | 02 May 2015, 14:46:12 UTC |
d2346c2 | Venkatesh Radhakrishnan | 01 May 2015, 22:41:50 UTC | Fix hang with large write batches and column families. Summary: This diff fixes a hang reported by a Github user. https://www.facebook.com/l.php?u=https%3A%2F%2Fgithub.com%2Ffacebook%2Frocksdb%2Fissues%2F595%23issuecomment-96983273&h=9AQFYOWlo Multiple large write batches with column families cause a hang. The issue was caused by not doing flushes/compaction when the write controller was stopped. Test Plan: Create a DBTest from the user's test case Reviewers: igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37929 | 01 May 2015, 22:41:50 UTC |
b6b100f | Mark Callaghan | 01 May 2015, 21:17:45 UTC | Remove iter_refresh_interval_us Summary: The default, use one iter for the whole test, isn't good. This cost me a few hours of debugging and a few days of tessting. For readonly that isn't realistic and for read-write that keeps a lot of old sst files around. I remove the option because nothing uses it and not calling gettimeofday per loop iteration adds about 3% to QPS at 20 threads. Task ID: # Blame Rev: Test Plan: run db_bench Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37965 | 01 May 2015, 21:17:45 UTC |
197f01b | Adam Retter | 30 April 2015, 22:16:04 UTC | Bugfix remove deprecated option use which was removed in previous commit 019ecd19329ee895284e9b040df0ffe4c08b35d8 | 30 April 2015, 22:16:04 UTC |
aa094e8 | Adam Retter | 30 April 2015, 22:15:33 UTC | Fix conversion from nano-seconds to milli-seconds and seconds | 30 April 2015, 22:15:33 UTC |
dddceef | Igor Canadi | 30 April 2015, 18:11:04 UTC | Fix clang build Summary: fix build Test Plan: works Reviewers: kradhakrishnan Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37911 | 30 April 2015, 18:11:35 UTC |
d454065 | krad | 29 April 2015, 22:36:21 UTC | Optimize GetApproximateSizes() to use lesser CPU cycles. Summary: CPU profiling reveals GetApproximateSizes as a bottleneck for performance. The current implementation is sub-optimal, it scans every file in every level to compute the result. We can take advantage of the fact that all levels above 0 are sorted in the increasing order of key ranges and use binary search to locate the starting index. This can reduce the number of comparisons required to compute the result. Test Plan: We have good test coverage. Run the tests. Reviewers: sdong, igor, rven, dynamike Subscribers: dynamike, maykov, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37755 | 30 April 2015, 17:55:03 UTC |
fd96b55 | Igor Canadi | 30 April 2015, 16:29:51 UTC | Making GetOptions() comment better (#597) | 30 April 2015, 16:29:51 UTC |
7246ad3 | Igor Canadi | 29 April 2015, 20:35:48 UTC | Don't compact bottommost level in SuggestCompactRange Summary: Before the fix we also marked the bottommost level for compaction. This is wrong because then RocksDB has N+1 levels instead of N as before the compaction. Test Plan: SuggestCompactRangeTest in db_test Reviewers: yhchiang, rven Reviewed By: rven Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37869 | 29 April 2015, 20:35:48 UTC |
7f47ba0 | Igor Canadi | 29 April 2015, 17:52:31 UTC | Fix possible SIGSEGV in CompactRange (github issue #596) Summary: For very detailed explanation of what's happening read this: https://github.com/facebook/rocksdb/issues/596 Test Plan: make check + new unit test Reviewers: yhchiang, anthony, rven Reviewed By: rven Subscribers: adamretter, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37779 | 29 April 2015, 17:52:31 UTC |
aba005c | Yueh-Hsuan Chiang | 28 April 2015, 23:19:40 UTC | Merge pull request #585 from fyrz/RocksJava-RemoveDeprecatedStuff [RocksJava] Remove deprecated methods | 28 April 2015, 23:19:40 UTC |
d6f39c5 | agiardullo | 28 April 2015, 03:23:50 UTC | Helper function to time Merges Summary: Remove duplicate code. If this diff looks good, I will cleanup other call sites as well. Test Plan: unit tests Reviewers: rven, yhchiang, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37761 | 28 April 2015, 03:23:50 UTC |
a087f80 | Mark Callaghan | 28 April 2015, 02:32:56 UTC | Add scripts to run leveldb benchmark Summary: This runs a benchmark for LevelDB similar to what we have in tools/run_flash_bench.sh. It requires changes to db_bench that I published in a LevelDB fork on github. Some results are at: http://smalldatum.blogspot.com/2015/04/comparing-leveldb-and-rocksdb-take-2.html Sample output: ops/sec mb/sec usec/op avg p50 Test 525 16.4 1904.5 1904.5 111.0 fillseq.v32768 75187 15.5 13.3 13.3 4.4 fillseq.v200 28328 5.8 35.3 35.3 4.7 overwrite.t1.s0 175438 0.0 5.7 5.7 4.4 readrandom.t1 28490 5.9 35.1 35.1 4.7 overwrite.t1.s0 121951 0.0 8.2 8.2 5.7 readwhilewriting.t1 Task ID: # Blame Rev: Test Plan: Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37749 | 28 April 2015, 02:32:56 UTC |
1bb4928 | Igor Canadi | 27 April 2015, 22:20:02 UTC | Include bunch of more events into EventLogger Summary: Added these events: * Recovery start, finish and also when recovery creates a file * Trivial move * Compaction start, finish and when compaction creates a file * Flush start, finish Also includes small fix to EventLogger Also added option ROCKSDB_PRINT_EVENTS_TO_STDOUT which is useful when we debug things. I've spent far too much time chasing LOG files. Still didn't get sst table properties in JSON. They are written very deeply into the stack. I'll address in separate diff. TODO: * Write specification. Let's first use this for a while and figure out what's good data to put here, too. After that we'll write spec * Write tools that parse and analyze LOGs. This can be in python or go. Good intern task. Test Plan: Ran db_bench with ROCKSDB_PRINT_EVENTS_TO_STDOUT. Here's the output: https://phabricator.fb.com/P19811976 Reviewers: sdong, yhchiang, rven, MarkCallaghan, kradhakrishnan, anthony Reviewed By: anthony Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37521 | 27 April 2015, 22:20:02 UTC |
3db81d5 | Aashish Pant | 27 April 2015, 04:47:30 UTC | Fix memory leak in cache_test introduced in the previous commit Test Plan: Verified that valgrind build passes for cache_test Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37665 | 27 April 2015, 04:47:30 UTC |
4961a96 | Igor Canadi | 26 April 2015, 04:09:45 UTC | Fix build Summary: Build broken by https://github.com/facebook/rocksdb/commit/6ede020dc419c1621254f26060076ee6d2c2d792 Test Plan: make all Reviewers: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37689 | 26 April 2015, 04:12:52 UTC |
93ab147 | Igor Canadi | 26 April 2015, 03:47:11 UTC | Merge pull request #593 from charsyam/feature/type-1 fix typos | 26 April 2015, 03:47:11 UTC |
6ede020 | clark.kang | 25 April 2015, 09:14:27 UTC | fix typos | 25 April 2015, 09:14:27 UTC |
3d1af4a | Igor Canadi | 25 April 2015, 01:43:07 UTC | Don't preinstall jemalloc in Travis Summary: Since we enabled jemalloc for open source builds, Travis looks like it's dying. Don't install jemalloc when running in travis Test Plan: none Reviewers: yhchiang Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37659 | 25 April 2015, 01:43:07 UTC |
242f9b4 | Aashish Pant | 24 April 2015, 21:45:12 UTC | Fix CLANG build issue introduced in previous commit Summary: Added keyword override for SetCapacity() Test Plan: Fixes build Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37647 | 24 April 2015, 21:45:12 UTC |
794ccfd | Aashish Pant | 24 April 2015, 21:12:58 UTC | Task 6532943: Rocksdb - SetCapacity() can dynamically change cache capacity if feasible Summary: When new capacity is larger than existing capacity, simply update the capacity to the new valie When new capacity is less than existing capacity, but more than the usage, simply update the capacity to new value When new capacity is less than the existing capacity and existing usage both, try to purge entries in LRU if feasible to make usage < capacity Test Plan: Created unit tests in cache_test.cc Reviewers: sdong, rven, yhchiang, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37527 | 24 April 2015, 21:12:58 UTC |
98a4455 | sdong | 24 April 2015, 02:17:57 UTC | Build for CYGWIN Summary: Make it build for CYGWIN. Need to define "-std=gnu++11" instead of "-std=c++11" and use some replacement functions. Test Plan: Build it and run some unit tests in CYGWIN Reviewers: yhchiang, rven, anthony, kradhakrishnan, igor Reviewed By: igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D37605 | 24 April 2015, 04:33:44 UTC |
d01bbb5 | sdong | 22 April 2015, 23:55:22 UTC | Fix CompactRange for universal compaction with num_levels > 1 Summary: CompactRange for universal compaction with num_levels > 1 seems to have a bug. The unit test also has a bug so it doesn't capture the problem. Fix it. Revert the compact range to the logic equivalent to num_levels=1. Always compact all files together. It should also fix DBTest.IncreaseUniversalCompactionNumLevels. The issue was that options.write_buffer_size = 100 << 10 and options.write_buffer_size = 100 << 10 are not used in later test scenarios. So write_buffer_size of 4MB was used. The compaction trigger condition is not anymore obvious as expected. Test Plan: Run the new test and all test suites Reviewers: yhchiang, rven, kradhakrishnan, anthony, igor Reviewed By: igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D37551 | 24 April 2015, 02:12:31 UTC |
e003d38 | Igor Canadi | 24 April 2015, 01:08:37 UTC | Abstract out SetMaxPossibleForUserKey() and SetMinPossibleForUserKey Summary: Based on feedback from D37083. Are all of these correct? In some spaces it seems like we're doing SetMaxPossibleForUserKey() although we want the smallest possible internal key for user key. Test Plan: make check Reviewers: sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37341 | 24 April 2015, 01:08:37 UTC |
fd7a357 | Igor Canadi | 24 April 2015, 00:48:18 UTC | Enable open source users to use jemalloc (github issue #438) Summary: Currently open source rocksdb only builds with tcmalloc. This diff first checks if jemalloc is available. If it is, it compiles with jemalloc. If it isn't, it checks for tcmalloc. Test Plan: Tried this out on my Ubuntu virtual machine and confirms that jemalloc is correctly detected and compiled. Reviewers: MarkCallaghan, yhchiang, rven, sdong Reviewed By: sdong Subscribers: adamretter, meyering, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D36789 | 24 April 2015, 00:48:18 UTC |
aa14670 | Igor Canadi | 24 April 2015, 00:46:15 UTC | Add an assertion in CompactionPicker Summary: Reading CompactionPicker I noticed this dangerous substraction of two unsigned integers. We should assert to mark this as safe. Test Plan: make check Reviewers: anthony, rven, yhchiang, sdong Reviewed By: sdong Subscribers: kradhakrishnan, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37041 | 24 April 2015, 00:46:15 UTC |
2dc421d | Giuseppe Ottaviano | 23 April 2015, 19:10:36 UTC | Implement DB::PromoteL0 method Summary: This diff implements a new `DB` method `PromoteL0` which moves all files in L0 to a given level skipping compaction, provided that the files have disjoint ranges and all levels up to the target level are empty. This method provides finer-grain control for trivial compactions, and it is useful for bulk-loading pre-sorted keys. Compared to D34797, it does not change the semantics of an existing operation, which can impact existing code. PromoteL0 is designed to work well in combination with the proposed `GetSstFileWriter`/`AddFile` interface, enabling to "design" the level structure by populating one level at a time. Such fine-grained control can be very useful for static or mostly-static databases. Test Plan: `make check` Reviewers: IslamAbdelRahman, philipp, MarkCallaghan, yhchiang, igor, sdong Reviewed By: sdong Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37107 | 23 April 2015, 19:10:36 UTC |
9bf40b6 | sdong | 20 April 2015, 18:15:35 UTC | Print max score in level summary Summary: Add more logging to help debugging issues. Test Plan: Run test suites Reviewers: yhchiang, rven, igor Reviewed By: igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D37401 | 23 April 2015, 18:34:36 UTC |
397b658 | sdong | 17 April 2015, 22:26:50 UTC | options.paranoid_file_checks to read all rows after writing to a file. Summary: To further distinguish the corruption cases were caused by storage media or in memory states when writing it, add a paranoid check after writing the file to iterate all the rows. Test Plan: Add a new unit test for it Reviewers: rven, igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D37335 | 23 April 2015, 18:34:35 UTC |
283a042 | Mark Callaghan | 23 April 2015, 16:18:25 UTC | Set --seed per test Summary: This is done to avoid having each thread use the same seed between runs of db_bench. Without this we can inflate the OS filesystem cache hit rate on reads for read heavy tests and generally see the same key sequences get generated between teste runs. Task ID: # Blame Rev: Test Plan: Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37563 | 23 April 2015, 16:18:25 UTC |
618d07b | Venkatesh Radhakrishnan | 23 April 2015, 15:35:02 UTC | Making PreShutdown tests more reliable. Summary: A couple of times on Travis, we have had the thread status say that there were no compactions done and since we assert for it, the test failed. We now fix this by waiting till compaction started. Test Plan: run DBTEST::*PreShutdown* d=/tmp/j; rm -rf $d; seq 200 | parallel --gnu --eta 'd=/tmp/j/d-{}; mkdir -p $d; TEST_TMPDIR=$d ./db_test --gtest_filter=DBTest.PreShutdown* >& '$d'/log-{}' Reviewers: sdong, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37545 | 23 April 2015, 15:35:02 UTC |
0a91bca | Jim Meyering | 23 April 2015, 15:00:56 UTC | test: avoid vuln-inducing use of temporary directory Summary: Without this change, someone on the machine on which I run "make check" could cause me to overwrite arbitrary files owned by me, via a symlink attack. Instead of using a predictable temporary directory and accepting to use a preexisting one, always create a new one using mkdtemp. If $TEST_IOCTL_FRIENDLY_TMPDIR is set and usable, attempt first to find a usable temporary directory therein. If not, or if unusable, then try /var/tmp and /tmp. If none of those is usable abort with a diagnostic. To do that, I added a new class. Its constructor finds a suitable directory or aborts, the sole member prints that directory's name, and the destructor unlinks what should be an empty directory. Note that while the code before this did not remove its temporary directory, there was only one per $UID. Now, there would be at least one per run or one per test, depending on implementation, so it is important to remove them. Test Plan: Run this on a fedora rawhide system, where /tmp is a tmpfs file system, and /var/tmp is ext4. # This gives a diagnostic that /dev/shm is not suitable # and ends up using /var/tmp. TEST_IOCTL_FRIENDLY_TMPDIR=/dev/shm ./env_test # Uses /var/tmp; same as when envvar not set. TEST_IOCTL_FRIENDLY_TMPDIR=/var/tmp ./env_test # Uses /tmp unless it's tmpfs, in which case it gives # a diagnostic and uses /var/tmp. TEST_IOCTL_FRIENDLY_TMPDIR=/tmp ./env_test Reviewers: ljin, rven, igor.sugak, yhchiang, sdong, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37287 | 23 April 2015, 15:00:56 UTC |
6e35941 | Mark Callaghan | 23 April 2015, 00:17:50 UTC | Add rpath for production builds Summary: This lets the production toolchain libraries get used on devservers and in production. Task ID: #6849362 Blame Rev: Test Plan: Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37533 | 23 April 2015, 00:17:50 UTC |
78dbd08 | Mark Callaghan | 22 April 2015, 20:23:08 UTC | Improve benchmark scripts Summary: This adds: 1) use of --level_compaction_dynamic_level_bytes=true 2) use of --bytes_per_sync=2M The second is a big win for disks. The first helps in general. This also adds a new test, fillseq with 32kb values to increase the peak ingest and make it more likely that storage limits throughput. Sample outpout from the first 3 tests - https://gist.github.com/mdcallag/e793bd3038e367b05d6f Task ID: # Blame Rev: Test Plan: Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37509 | 22 April 2015, 20:23:08 UTC |
6a5ffee | Igor Canadi | 22 April 2015, 19:47:50 UTC | Fix gflags Makefile Summary: `echo` correctly interpretes \n on mac, but not on linux. On linux you have to give it `-e` to interpret \n. Unfortunately, `-e` options is not available on Mac. Go back to old way of checking gflags Test Plan: build_tools/build_detect_platform on mac and linux Reviewers: sdong Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37515 | 22 April 2015, 19:50:28 UTC |
108a927 | Siying Dong | 21 April 2015, 21:57:36 UTC | Merge pull request #589 from coderplay/patch-1 Add "Tango Me" section in USERS.md | 21 April 2015, 21:57:36 UTC |
a58fd74 | Min Zhou | 21 April 2015, 21:53:09 UTC | Update USERS.md | 21 April 2015, 21:53:09 UTC |
d85d08c | Igor Canadi | 21 April 2015, 03:45:25 UTC | One last fix to Makefile Summary: Based on comment from D37455 Test Plan: make install after make static_lib Reviewers: meyering Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37461 | 21 April 2015, 03:46:08 UTC |
2db96dc | Igor Canadi | 21 April 2015, 03:30:04 UTC | Fix make install when there is no shared lib Summary: make install fails when there is no shared lib. We need to revert the conditions, which will have the same effect, but without the failure Test Plan: make install after only compiling static library Reviewers: meyering Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37455 | 21 April 2015, 03:39:54 UTC |
7d13699 | Igor Canadi | 21 April 2015, 02:44:02 UTC | Get rid of error output Summary: We should send error output to /dev/null Test Plan: none Reviewers: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37449 | 21 April 2015, 02:44:32 UTC |
79c1b02 | Igor Canadi | 21 April 2015, 02:39:51 UTC | Fix Makefile Summary: The current version tries to install librocksdb.so even though it doesn't exist. This version will install librocksdb.so.3.10.0 and then create soft links in place Test Plan: `make static_lib; sudo make install` does not try to install librocksdb.so `make shared_lib; sudo make install` installs one library and 3 symlinks. Before, four libraries were installed Reviewers: sdong, meyering Reviewed By: meyering Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37425 | 21 April 2015, 02:39:51 UTC |
742fa9e | Igor Canadi | 20 April 2015, 17:55:17 UTC | Fix compile with two gflags Summary: If the system has gflags with both `google` and `gflags` namespaces installed, we try to define GFLAGS as two things. This breaks the compile. Fix: Use `else if` -- try compiling with `google` namespace only if compile with `gflags` failed Test Plan: build_tools/build_detect_platform correctly identifies gflags Reviewers: lgalanis Reviewed By: lgalanis Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37389 | 20 April 2015, 17:55:17 UTC |
79c21ec | Jim Meyering | 18 April 2015, 03:39:02 UTC | skip ioctl-using tests when not supported Summary: [NB: this is a prerequisite for the /tmp-abuse-fixing patch] This avoids spurious test failure on Linux systems like Fedora for which /tmp is a tmpfs file system. On a devtmpfs file system, ioctl(fd, FS_IOC_GETVERSION, &version) returns -1 with errno == ENOTTTY, indicating that that ioctl is not supported on such a file system. Do not let this cause test failures, e.g., where env_test would assert that file->GetUniqueId(...) > 0. Before this change, ./env_test would fail these three tests on a fedora rawhide system: [ FAILED ] 3 tests, listed below: [ FAILED ] EnvPosixTest.RandomAccessUniqueID [ FAILED ] EnvPosixTest.RandomAccessUniqueIDConcurrent [ FAILED ] EnvPosixTest.RandomAccessUniqueIDDeletes 3 FAILED TESTS The fix: When support for that ioctl is lacking, skip each affected test. Could be improved by noting which sub-tests are being skipped. Test Plan: run these on F21 and note that they now pass. TEST_TMPDIR=/dev/shm/rdb ./env_test ./env_test Reviewers: ljin, rven, igor.sugak, yhchiang, sdong, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37323 | 18 April 2015, 03:39:02 UTC |
6059bdf | Igor Canadi | 17 April 2015, 23:44:45 UTC | Add experimental API MarkForCompaction() Summary: Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys). All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart. MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction. Test Plan: added a new unit test Reviewers: yhchiang, rven, MarkCallaghan, sdong Reviewed By: sdong Subscribers: yoshinorim, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37083 | 17 April 2015, 23:44:45 UTC |
acf8a41 | Jim Meyering | 17 April 2015, 21:54:17 UTC | maint: use ASSERT_TRUE, not ASSERT_EQ(true; same for false Summary: The usage I'm fixing here caused trouble on Fedora 21 when compiling with the current gcc version 4.9.2 20150212 (Red Hat 4.9.2-6) (GCC): db/write_controller_test.cc: In member function ‘virtual void rocksdb::WriteControllerTest_SanityTest_Test::TestBody()’: db/write_controller_test.cc:23:165: error: converting ‘false’ to pointer type for argument 1 of ‘char testing::internal::IsNullLiteralHelper(testing::internal::Secret*)’ [-Werror=conversion-null] ASSERT_EQ(false, controller.IsStopped()); ^ This change was induced mechanically via: git grep -l -E 'ASSERT_EQ\(false'|xargs perl -pi -e 's/ASSERT_EQ\(false, /ASSERT_FALSE(/' git grep -l -E 'ASSERT_EQ\(true'|xargs perl -pi -e 's/ASSERT_EQ\(true, /ASSERT_TRUE(/' Except for the three in utilities/backupable/backupable_db_test.cc for which I ended up reformatting (joining lines) in the result. As for why this problem is exhibited with that version of gcc, and none of the others I've used (from 4.8.1 through gcc-5.0.0 and newer), I suspect it's a bug in F21's gcc that has been fixed in gcc-5.0.0. Test Plan: "make" now succeed on Fedora 21 Reviewers: ljin, rven, igor.sugak, yhchiang, sdong, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37329 | 17 April 2015, 21:54:17 UTC |
b5400f9 | Igor Canadi | 17 April 2015, 19:07:47 UTC | Kill dead code Summary: this is not used anywhere Test Plan: compiles Reviewers: yhchiang, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37053 | 17 April 2015, 19:07:47 UTC |
48b0a04 | Igor Canadi | 17 April 2015, 02:31:34 UTC | Speed up reduce_levels_test Summary: For some reason reduce_levels is opening the databse with 65.000 levels. This makes ComputeCompactionScore() function terribly slow and the tests is also very slow (20seconds). Test Plan: mr reduce_levels_test now takes 20ms Reviewers: sdong, rven, kradhakrishnan, yhchiang Reviewed By: yhchiang Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37059 | 17 April 2015, 02:31:34 UTC |
00c2afc | Igor Canadi | 17 April 2015, 02:31:10 UTC | Fix bug in ExpandWhileOverlapping() Summary: If ExpandWhileOverlapping() we don't clear inputs. That's a bug introduced by my recent patch https://reviews.facebook.net/D36687. However, we have no tests covering ExpandWhileOverlapping(). I created a task t6771252 to add ExpandWhileOverlapping() tests. Test Plan: make check Reviewers: sdong, rven, yhchiang Reviewed By: yhchiang Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37077 | 17 April 2015, 02:31:10 UTC |
019ecd1 | fyrz | 15 April 2015, 21:43:30 UTC | [RocksJava] Remove deprecated methods Summary: - Removed deprecated ColumnFamilyDescript constructor methods - Removed deprecated skipLogErrorOnRecovery methods - Removed deprecated tableCacheRemoveScanCountLimit methods Test Plan: make rocksdbjava make jtest Reviewers: yhchiang, adamretter, ankgup87 Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D37191 | 15 April 2015, 22:31:43 UTC |
98ef21d | Igor Canadi | 15 April 2015, 21:56:36 UTC | Merge pull request #584 from pshareghi/rocksdb-3.10-falloch Added falloc.h in build_detect_platform | 15 April 2015, 21:56:36 UTC |
5b7131c | fyrz | 15 April 2015, 21:35:09 UTC | [RocksJava] Removed deprecated skipLogErrorOnRecovery methods. As annonunced these options are not used anymore so these are removed from code. | 15 April 2015, 21:37:24 UTC |
566f652 | fyrz | 15 April 2015, 21:31:59 UTC | [RocksJava] Removed deprecated ColumnFamilyDescriptor methods As announced previously removed methods are obsolete and will be replaced by its byte array pendants. | 15 April 2015, 21:31:59 UTC |
6997aa0 | Igor Canadi | 15 April 2015, 21:23:29 UTC | Merge pull request #582 from fyrz/RocksJava-Fix-RateLimiter [RocksJava] Fix RateLimiter Tests in 3.10 | 15 April 2015, 21:23:29 UTC |
582c4b0 | fyrz | 15 April 2015, 21:19:41 UTC | [RocksJava] Fix RateLimiter Tests in 3.10 | 15 April 2015, 21:19:41 UTC |
6cfb215 | Igor Canadi | 15 April 2015, 20:36:27 UTC | Merge pull request #581 from vladb38/patch-3 Update USERS.md | 15 April 2015, 20:36:27 UTC |
d71e8f7 | Vlad Balan | 15 April 2015, 20:25:52 UTC | Update USERS.md | 15 April 2015, 20:25:52 UTC |
debaf85 | sdong | 15 April 2015, 02:58:52 UTC | Bug of trivial move of dynamic level Summary: D36669 introduces a bug that trivial moved data is not going to specific level but the next level, which will incorrectly be level 1 for level 0 compaciton if base level is not level 1. Fixing it by appreciating the output level Test Plan: Run all tests Reviewers: MarkCallaghan, rven, yhchiang, igor Reviewed By: igor Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D37119 | 15 April 2015, 04:42:08 UTC |
12d7d3d | sdong | 14 April 2015, 23:53:19 UTC | Fix and Improve DBTest.DynamicLevelCompressionPerLevel2 Summary: Recent change of DBTest.DynamicLevelCompressionPerLevel2 has a bug that the second sync point is not enabled. Fix it. Also add an assert for that. Also, flush compression is not tracked in the test. Add it. Test Plan: Build everything Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D37101 | 15 April 2015, 04:42:08 UTC |
a1271c6 | sdong | 14 April 2015, 23:39:23 UTC | Fix build break introduced by new SyncPoint interface change Summary: When commiting the sync point interface change, didn't resolve the new occurance of the old interface in rebase. Fix it. Test Plan: Build and see it pass Reviewers: igor, yhchiang, rven, anthony, kradhakrishnan Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D37095 | 14 April 2015, 23:42:37 UTC |