https://github.com/voldemort/voldemort
Tip revision: 7df008c7c2189e71ca0796fefee3e5172dc7f4d1 authored by Yan Yan on 11 August 2016, 02:03:42 UTC
Releasing Voldemort 1.10.19.1
Releasing Voldemort 1.10.19.1
Tip revision: 7df008c
build.gradle
apply plugin: 'java'
apply plugin: 'war'
buildscript {
repositories { jcenter() }
dependencies {
classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.1'
}
}
apply plugin: 'com.github.johnrengelman.shadow'
def String getProjectProperty(String propertyName) {
String propertyValue = "null"
if (hasProperty(propertyName)) {
propertyValue = this.properties[propertyName]
}
else {
throw new GradleScriptException("PropertyName " + propertyName + " is not defined in properties file")
}
return propertyValue
}
def projectName = project.name
def sourceDir = getProjectProperty('src.dir')
def distDir = getProjectProperty('dist.dir')
def classesDir = getProjectProperty('classes.dir')
def javaDir = getProjectProperty('java.dir')
def resourcesDir = getProjectProperty('resources.dir')
def javaDocDir = getProjectProperty('javadoc.dir')
def voldTestClassesDir = getProjectProperty('testclasses.dir')
def commonTestSrcDir = getProjectProperty('commontestsrc.dir')
def unitTestSrcDir = getProjectProperty('unittestsrc.dir')
def intTestSrcDir = getProjectProperty('inttestsrc.dir')
def longTestSrcDir = getProjectProperty('longtestsrc.dir')
def voldVersion = getProjectProperty('curr.release')
def javacVersion = getProjectProperty('javac.version')
//This is the javaCompile variable version. Directly defining 'def version' will override this and cause nightmare
version = voldVersion
def archiveDirectoryName = projectName + '-' + version
def archiveDirectoryPath = distDir + "/" + archiveDirectoryName
def javadocEnabled = getProjectProperty('javadoc.enabled').toBoolean()
def deleteDirectoryContents(directory) {
project.file(directory).deleteDir()
project.file(directory).mkdirs()
}
allprojects {
sourceCompatibility = javacVersion
targetCompatibility = javacVersion
compileJava.options.debug = true
repositories {
mavenCentral()
maven {
// For Hadoop dependencies
url "https://repository.cloudera.com/artifactory/cloudera-repos/"
}
maven {
// For BDB-Je dependencies
url "http://download.oracle.com/maven/"
}
}
// http://blog.joda.org/2014/02/turning-off-doclint-in-jdk-8-javadoc.html
if (JavaVersion.current().java8Compatible) {
tasks.withType(Javadoc) {
options.addStringOption('Xdoclint:none', '-quiet')
}
}
tasks.withType(Javadoc) {
enabled = javadocEnabled
}
}
sourceSets {
main {
java { srcDirs = [javaDir] }
resources {
srcDirs = [javaDir]
include '**/*.xsd'
}
output.classesDir = classesDir
output.resourcesDir = resourcesDir
}
test {
java {
srcDirs = [
commonTestSrcDir,
unitTestSrcDir,
intTestSrcDir,
longTestSrcDir
]
}
output.classesDir = voldTestClassesDir
}
}
compileJava.doLast {
project.copy {
from (javaDir) { exclude '**/*.java','**/*.html','**/log4j.properties' }
into classesDir
}
project.copy {
// Theoretically this block can be replaced by including the log4j.properties in main resources.
// But that causes the log4j.properties to be present in the voldJar . Not sure what is the
// implication of this change, so avoiding it for now.
from (javaDir) { include 'log4j.properties' }
into resourcesDir
}
}
javadoc {
destinationDir = file(javaDocDir)
}
compileTestJava.doLast {
project.copy {
from (commonTestSrcDir) { exclude '**/*.java','**/*.html' }
from (unitTestSrcDir) { exclude '**/*.java','**/*.html' }
into voldTestClassesDir
}
}
task testJar(type: Jar) {
baseName = projectName + "-test"
from sourceSets.test.output
destinationDir = project.file(distDir)
}
jar {
manifest {
attributes 'Voldemort-Implementation-Version' : version,
'Implementation-Title': 'Voldemort',
'Implementation-Version': version,
'Implementation-Vendor' :'LinkedIn'
}
destinationDir = project.file(distDir)
}
task contribJar(type:Jar) {
baseName = projectName + "-contrib"
from {subprojects*.sourceSets.main.output}
destinationDir = project.file(distDir)
}
task srcJar(type: Jar, dependsOn: classes) {
classifier = 'sources'
from sourceSets.main.java.srcDirs
destinationDir = project.file(distDir)
}
task javadocJar(type: Jar) {
enabled = javadocEnabled
classifier = 'javadoc'
from javadoc
destinationDir = file(distDir)
}
task bnpJar(dependsOn: shadowJar) {
// Just a nicer more self-explanatory name than "shadowJar"
}
artifacts {
archives jar
archives testJar
archives contribJar
archives srcJar
archives javadocJar
}
clean {
delete(distDir)
delete('lib')
doLast { deleteDirectoryContents(javaDocDir) }
}
// Dependencies used by both BnP and Voldemort
// TODO: Decide if we want to do that for all dependencies, even if they're used just in Voldemort...
def depAvro = 'org.apache.avro:avro:1.4.0'
def depProtoBuf = 'com.google.protobuf:protobuf-java:2.3.0'
def depJdom = 'org.jdom:jdom:1.1'
def depAzkaban = 'com.linkedin.azkaban:azkaban:2.5.0'
def depGuava = 'com.google.guava:guava:14.0.1'
def depLog4j = 'log4j:log4j:1.2.15'
def depJacksonMapper = 'org.codehaus.jackson:jackson-mapper-asl:1.9.13'
def depJoda = 'joda-time:joda-time:1.6'
def depTehuti = 'io.tehuti:tehuti:0.7.0'
shadowJar {
classifier "bnp"
from sourceSets.main.output, sourceSets.test.output, sourceSets.main.resources
from {subprojects*.sourceSets.main.output}
from {subprojects*.sourceSets.test.output}
// Required when working in an Hadoop 2.x environment
dependencies {
include(dependency(depAvro))
include(dependency(depProtoBuf))
include(dependency(depJdom))
include(dependency(depAzkaban))
include(dependency(depGuava))
include(dependency(depLog4j))
include(dependency(depJacksonMapper))
include(dependency(depJoda))
include(dependency(depTehuti))
}
relocate 'com.google.protobuf', 'voldemort.shadow.2.3.0.com.google.protobuf'
relocate 'org.apache.avro', 'voldemort.shadow.1.4.0.org.apache.avro'
// TODO: find a way to exclude private lib's BDB-JE which gets pulled into the fat jar...
}
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
task protobufJar(type: ShadowJar) {
baseName = projectName + "-protobuf"
manifest {
attributes 'Voldemort-Implementation-Version' : version,
'Implementation-Title': 'Voldemort',
'Implementation-Version': version,
'Implementation-Vendor' :'LinkedIn'
}
dependencies {
include(dependency(depProtoBuf))
configurations.runtime.each { exclude(dependency(it)) }
}
configurations = [ project.configurations.runtime ]
from sourceSets.main.output
destinationDir = project.file(distDir)
relocate 'com.google.protobuf', 'voldemort.shadow.2.3.0.com.google.protobuf'
// Since BDB is included via compile files option, shadow does not support
// excluding them. https://github.com/johnrengelman/shadow/issues/142
// Once shadow supports excluding them, it can be excluded. There are
// Complex ways to exclude it, but not worth it.
relocate 'com.sleepycat' , 'voldemort.shadow.5.0.88.com.sleepycat'
}
task copySources (type: Copy) {
from ('.') { include 'bin/*.sh', 'bin/*.bat' , 'bin/*.py' }
from ('.') { include distDir + '/*.jar'}
from ('.') { exclude distDir + '/**' ,'bin/**' , 'build/**', '.git/**' , '.gradle/**', 'config/**/data/**' }
into archiveDirectoryPath
}
task copyDeps(type: Copy) {
// note this only copies the dependencies of the root
// project into /lib if we start adding compile deps to
// subprojects will have to rethink
from { configurations.compile }
into "lib"
}
task zip (type: Zip) {
dependsOn copySources, copyDeps, contribJar, protobufJar
baseName = projectName
from(distDir) {
include archiveDirectoryName + '/bin/**'
fileMode = 0755
}
from(distDir) {
include archiveDirectoryName + '/**'
exclude archiveDirectoryName + '/bin/**'
}
destinationDir = project.file(distDir)
}
task tar (type: Tar) {
dependsOn copySources, copyDeps, contribJar, protobufJar
compression = Compression.GZIP
baseName = projectName
extension = "tar.gz"
from(distDir) {
include archiveDirectoryName + '/bin/**'
fileMode = 0755
}
from(distDir) {
include archiveDirectoryName + '/**'
exclude archiveDirectoryName + '/bin/**'
}
destinationDir = project.file(distDir)
}
war {
dependsOn copyDeps
from sourceSets.main.output
webXml = project.file('web.xml')
destinationDir = project.file(distDir)
}
assemble.dependsOn copyDeps
copySources.dependsOn jar
allprojects {
tasks.withType(Test) {
maxHeapSize = "8g"
// If ignoreFailures is not set, then merged reports will not be generated
// Gradle aborts further tasks on test failure. so if you run junitAll
// which runs 3 tests, reports task will never be run on failure cases.
ignoreFailures = true
useJUnit()
testLogging {
events "started", "passed", "skipped", "failed"
exceptionFormat = 'full'
// showStandardStreams = true
doFirst {
def classesSize = candidateClassFiles.files.size()
logger.lifecycle("{} starts executing {} test classes {}",
path, classesSize, classesSize > 0? "(" + candidateClassFiles.files*.name[0] + ", ...)" : "")
}
//Set reasonable defaults for reports location
reports.html.destination = file("$buildDir/reports/$name")
reports.junitXml.destination = file("$buildDir/$name-results")
}
// Makes sure tests aren't marked "UP-TO-DATE" after running
outputs.upToDateWhen { false }
}
}
tasks.withType(Test) {
// note only the root project's tests fork for very test class
forkEvery = 1
// Do not set the max parallelism as there are tests that uses the same port and will
// run into bind exceptions.
//ignoreFailures = gradle.startParameter.continueOnFailure
//all standard error messages from tests will get routed to 'DEBUG' level messages.
//logging.captureStandardError(LogLevel.DEBUG)
//all standard output messages from tests will get routed to 'DEBUG' level messages.
//logging.captureStandardOutput(LogLevel.DEBUG)
//Set reasonable defaults classpath and classes dir. They can be reconfigured in an individual task.
// it.testClassesDir = sourceSets.test.output.classesDir
// classpath = sourceSets.test.runtimeClasspath
}
task resetConfig() {
doLast {
def DirsToDelete = [".temp", ".version", "data"]
def deleteRecursively
deleteRecursively = { file ->
file.eachFile() {f ->
if(f.directory) {
if( DirsToDelete.contains(f.getName()) )
{
println "deleting ${f.getAbsolutePath()}"
delete f
}
else
{
deleteRecursively(f)
}
}
}
}
deleteRecursively (new File("config"))
}
}
task junit(dependsOn: test)
Collection<String> testClassesFrom(String dir, String include = '**/*Test.*') {
//take all *Test.java files found in given dir, make the path relative and replace .java with .class
fileTree(dir: dir, includes: [include]).collect { it.absolutePath.replace("\\", "/").replaceAll(file(dir).absolutePath.replace("\\", "/") + "/", "").replaceAll(".java\$", ".class")}
}
test {
description = "Runs acceptance tests"
include testClassesFrom(unitTestSrcDir)
}
task junitLong(type: Test) {
description = "Runs long junit tests"
include testClassesFrom(longTestSrcDir)
}
task junitInt(type: Test) {
description = "Runs integration tests"
include testClassesFrom(intTestSrcDir)
}
task junitRebalance(type: Test) {
include testClassesFrom(unitTestSrcDir, '**/*Rebalance*Test.java')
}
task junitRebalanceLong(type: Test) {
include testClassesFrom(longTestSrcDir, '**/*Rebalance*Test.java')
}
task contribJunit(type:TestReport) {
// this populated below by depending on all the test tasks found
// in the subprojects.
destinationDir = file("$buildDir/reports/$name")
}
subprojects {
tasks.withType(Test) {
// hook up the report to teh contrib task and junitAll
rootProject.contribJunit.reportOn it
rootProject.junitAll.reportOn it
}
}
task junitAll(type: TestReport) {
reportOn test, junitLong
destinationDir = file("$project.buildDir/reports/$name")
}
task aggregatedJunit(type: TestReport) {
destinationDir = file("$project.buildDir/reports/$name")
}
allprojects {
tasks.withType(Test) {
finalizedBy rootProject.aggregatedJunit
doLast { rootProject.aggregatedJunit.reportOn it }
}
}
task wrapper(type: Wrapper) { gradleVersion = '2.9' }
dependencies {
// Avro serialization format
compile depAvro
// INTERNAL_LIBS azkaban version not found
// azkaban-common-0.05.jar
// INTERNAL_LIBS Used for tomcat deployment, not sure if anyone uses it
// catalina-ant.jar , version not found in maven central
// coders decoders containing the Base64,binary encoding
compile 'commons-codec:commons-codec:1.4'
// TRANSITIVE_DEPENDENCY Contrib jar depends on commons-configuration-1.6.jar
// commons-configuration instead depends on commons-collection
//compile 'commons-collections:commons-collections:3.2.1'
// Used by MySql storage engine classes
// The jar supports database connection pooling
compile 'commons-dbcp:commons-dbcp:1.2.2'
// commons io is used at many places
// IOUtils, FileUtils and ByteArrayOutputStream
compile 'commons-io:commons-io:2.1'
// LZF compression strategy for store and tests.
compile 'com.ning:compress-lzf:0.9.1'
// Used all over the place for collections
compile depGuava
// used for readonly store hdfs fetcher.
compile 'org.apache.hadoop:hadoop-auth:2.3.0-cdh5.1.5'
// used at lots of places. Seems like there is some overlap between httpclient and core, but not clear
compile 'org.apache.httpcomponents:httpclient:4.1.2'
// contains both http server and client functionalities. Used for HttpResponse but could be used at more places.
compile 'org.apache.httpcomponents:httpcore:4.1.2'
// JSON mapping library from Java Objects to JSON
compile depJacksonMapper
// JSON processing library
compile 'org.codehaus.jackson:jackson-core-asl:1.9.13'
// Used for reading XML files and Document.
compile depJdom
// Jetty is used for HttpService and tests. Jetty Util is used for QueuedThreadPool class.
compile 'org.mortbay.jetty:jetty-util:6.1.18'
compile 'org.mortbay.jetty:jetty:6.1.18'
// A line processing library for command line. No compile time dependency
// Used by Voldemort shell
compile 'jline:jline:0.9.94'
// jna is library for invoking native functions
// used in the readonly store
compile 'net.java.dev.jna:jna:3.2.7'
// joda time is replacement for Java Date and Time
// used in readonly store code.
compile depJoda
// Used for argument command line parsing
compile 'net.sf.jopt-simple:jopt-simple:4.6'
// log4j - logger used in almost all files
compile depLog4j
// used in readonly store and Co-ordinator
compile 'javax.mail:mail:1.4.1'
// Used in co-ordinator and rest services
compile 'io.netty:netty:3.5.8.Final'
// TRANSITIVE_DEPENDENCY Paranamer is a library that allows the parameter names of non-private methods and constructors to be accessed at runtime
// Avro has a dependency on paranamer
// compile 'com.thoughtworks.paranamer:paranamer:2.1'
// protobuf is a supported protocol format between voldemort client and server
compile depProtoBuf
// Servlet
compile 'javax.servlet:servlet-api:2.5'
// slf4j is another logging abstraction framework.
// It is used by the apache.avro, apache.hadoop and r2 clients
compile 'org.slf4j:slf4j-api:1.5.6'
compile 'org.slf4j:slf4j-log4j12:1.5.6'
// snappy is one of the supported compression strategies in voldemort
compile 'org.iq80.snappy:snappy:0.2'
// Velocity is a simple yet powerful Java-based template engine that renders data
// from plain Java objects to text, xml, email, SQL, Post Script, HTML etc
// Velocity is used for Http Server GUI
compile 'org.apache.velocity:velocity:1.6.2'
// TRANSITIVE_DEPENDENCY Apache XML Parser
// used by jdom
// compile 'xerces:xercesImpl:2.9.1'
// BDB-JE from Oracle
compile 'com.sleepycat:je:5.0.104'
// cern library containing high performance Maps for int and double
// Currently only used in the tests
testCompile 'colt:colt:1.2.0'
// Used in resource pool perf testing class
testCompile 'commons-pool:commons-pool:1.5.2'
testRuntime 'mysql:mysql-connector-java:5.1.31'
// Used for unit tests and other automated testing
testCompile 'junit:junit:4.6'
// Mockito is written by our beloved friend Szczepan Faber :)
// Mocking framework used in some tests
testCompile 'org.mockito:mockito-all:1.8.5'
// contribCompile sourceSets.main.output
// contribCompile sourceSets.test.output
// declaring contribCompile dependencies as compile dependencies
// otherwise while copying dependencies to lib directory
// conflict resolution is not done properly across sourceSets
// and we end up with 2 versions of few jars like ( log4j, servlet etc. )
compile 'commons-configuration:commons-configuration:1.6'
compile('org.apache.hadoop:hadoop-core:2.3.0-mr1-cdh5.1.5') {
exclude group: 'com.google.protobuf'
exclude group: 'org.apache.avro'
}
compile('org.apache.hadoop:hadoop-common:2.3.0-cdh5.1.5') {
exclude group: 'com.google.protobuf'
exclude group: 'org.apache.avro'
}
compile('org.apache.hadoop:hadoop-hdfs:2.3.0-cdh5.1.5') {
exclude group: 'com.google.protobuf'
exclude group: 'org.apache.avro'
}
compile 'com.linkedin.pegasus:r2:1.8.3'
compile 'com.linkedin.pegasus:data:1.8.3'
compile 'com.linkedin.pegasus:pegasus-common:1.8.3'
compile depAzkaban
compile 'com.google.code.typica:typica:1.7.2'
compile 'com.sna-projects.krati:krati:0.4.9'
// Metrics
compile depTehuti
testCompile 'io.tehuti:tehuti:0.7.0:test'
// Other libs...
compile 'org.apache.tomcat:catalina-ant:6.0.43'
compile 'org.apache.hadoop:libthrift:0.5.0.0'
// rocksdb from maven
compile 'org.rocksdb:rocksdbjni:3.13.1'
// Bouncy Castle Libaray
compile 'org.bouncycastle:bcprov-jdk15on:1.48'
}
subprojects {
// this configures all the contrib subprojects
// note at the moment there dependencies are still
// declared in the dependency block above.
apply plugin:'java'
sourceSets {
// note that the contrib projects don't currently have resource
// directories so below is actually just a to keep idea & gradle from
// thinking that the test resources dir is src/test/resources
main {
java { srcDirs = ['src/java'] }
resources { srcDirs = ['src/resources'] }
}
test {
java { srcDirs = ['test'] }
resources { srcDirs = ['testResources'] }
}
}
dependencies {
compile rootProject
// Used for unit tests and other automated testing
testCompile rootProject.sourceSets.test.runtimeClasspath
}
tasks.withType(Test) {
// this is required as the test utils expect the config directory
// to be at the root of the process working directory.
workingDir = rootProject.projectDir
}
}
allprojects {
apply plugin: 'idea'
apply plugin: 'eclipse'
eclipse {
jdt {
// Currently the javac Version of jar is set to 1.5
// But @Override works differently between both, so overriding here
sourceCompatibility = targetCompatibility = 1.6
}
classpath {
defaultOutputDir = project.file('classes') // overrides the default of /bin which is where our scripts are
downloadSources = true
file {
whenMerged { classpath ->
// so for some reason the generated .classpath for the contrib projects includes
// two copies of *most* (maybe all) of the libraries from the parent project
// the following de-dupes these
def duplicateLibs = classpath.entries
.findAll { it.kind == 'lib' } // only library entries
.groupBy { it.library } // index by the library
.findAll { it.value.size() > 1 } // only where there is more that a single entry
duplicateLibs.each { k, v ->
// pick one from the list of dupes ..prefering the one without a sourcelib path
// else just take the first
def toRemove = v.find { !it.sourcePath } ?: v.first()
classpath.entries.remove toRemove
}
// also with for no reason contrib projects gets a source path configured
// with the following path.. since this path doesn't exist eclipse complains
classpath.entries.removeAll {
it.kind == 'lib' && it.library.path.endsWith('build/resources/test')
}
}
}
}
}
idea {
module {
downloadJavadoc = true
downloadSources = true
}
}
}