swh:1:snp:764524290233376de64947417267228741ce2485
Tip revision: 95cd5e7bcf4ed2af2e1d5563cc48d73731f37770 authored by Felix GV on 09 August 2014, 00:16:19 UTC
Removed old parent delegating code in StoreStats which would cause double counting.
Removed old parent delegating code in StoreStats which would cause double counting.
Tip revision: 95cd5e7
build.gradle
import java.util.jar.JarEntry;
apply plugin: 'java'
apply plugin: 'scala'
apply plugin: 'idea'
apply plugin: 'eclipse'
apply plugin: 'war'
def String getProjectProperty(String propertyName) {
String propertyValue = "null"
if (hasProperty(propertyName)) {
propertyValue = this.properties[propertyName]
}
else {
throw GradleScriptException("PropertyName " + propertyName + " is not defined in properties file")
}
return propertyValue
}
def projectName = "voldemort"
def sourceDir = getProjectProperty('src.dir')
def distDir = getProjectProperty('dist.dir')
def classesDir = getProjectProperty('classes.dir')
def javaDir = getProjectProperty('java.dir')
def privateLibDir = getProjectProperty('private.lib.dir')
def resourcesDir = getProjectProperty('resources.dir')
def javaDocDir = getProjectProperty('javadoc.dir')
def voldTestClassesDir = getProjectProperty('testclasses.dir')
def commonTestSrcDir = getProjectProperty('commontestsrc.dir')
def unitTestSrcDir = getProjectProperty('unittestsrc.dir')
def intTestSrcDir = getProjectProperty('inttestsrc.dir')
def longTestSrcDir = getProjectProperty('longtestsrc.dir')
def contribClassesDir = getProjectProperty('contrib.classes.dir')
def contribRootDir = getProjectProperty('contrib.root.dir')
def voldVersion = getProjectProperty('curr.release')
def javacVersion = getProjectProperty('javac.version')
def scalaVersion = getProjectProperty('scalac.version')
//This is the javaCompile variable version. Directly defining 'def version' will override this and cause nightmare
version = voldVersion
def archiveDirectoryName = projectName + '-' + version
def archiveDirectoryPath = distDir + "/" + archiveDirectoryName
def deleteDirectoryContents(directory) {
project.file(directory).deleteDir()
project.file(directory).mkdirs()
}
println 'java source target compatibility version ' + javacVersion
sourceCompatibility = javacVersion
targetCompatibility = javacVersion
compileJava.options.debug = true
// Using zinc scala compiler fails intermittently because of missing dependencies
// Scala is not main stream so incremeental compilation is not worth the problem
// it is causing.
// tasks.withType(ScalaCompile) { scalaCompileOptions.useAnt = false }
repositories {
mavenCentral()
flatDir { dirs privateLibDir }
flatDir { dirs contribRootDir }
}
sourceSets {
main {
java { srcDirs = [javaDir]}
scala {
srcDirs = [sourceDir]
include '**/*.scala'
}
resources {
srcDirs = [javaDir]
include '**/*.xsd'
}
output.classesDir = classesDir
output.resourcesDir = resourcesDir
}
test {
java {
srcDirs = [
commonTestSrcDir ,
unitTestSrcDir,
intTestSrcDir ,
longTestSrcDir
]
}
output.classesDir = voldTestClassesDir
}
contrib {
java { srcDirs = [contribRootDir]}
compileClasspath += sourceSets.main.runtimeClasspath + sourceSets.test.runtimeClasspath
output.classesDir = contribClassesDir
}
}
compileJava.doLast {
project.copy {
from (javaDir) { exclude '**/*.java','**/*.html','**/*.scala', '**/log4j.properties' }
into classesDir
}
project.copy {
// Theoretically this block can be replaced by including the log4j.properties in main resources.
// But that causes the log4j.properties to be present in the voldJar . Not sure what is the
// implication of this change, so avoiding it for now.
from (javaDir) { include 'log4j.properties' }
into resourcesDir
}
}
compileTestJava.doLast {
project.copy {
from (commonTestSrcDir) { exclude '**/*.java','**/*.html' }
from (unitTestSrcDir) { exclude '**/*.java','**/*.html' }
into voldTestClassesDir
}
}
compileContribJava.doLast {
project.copy {
from (contribRootDir + '/ec2-testing/resources')
into contribClassesDir
}
}
task testJar(type: Jar) {
baseName = projectName + "-test"
from sourceSets.test.output
destinationDir = project.file(distDir)
}
task voldJar(type:Jar) {
baseName = projectName
manifest {
attributes 'Voldemort-Implementation-Version' : version,
'Implementation-Title': 'Voldemort',
'Implementation-Version': version,
'Implementation-Vendor' :'LinkedIn'
}
from sourceSets.main.output
destinationDir = project.file(distDir)
}
war {
from sourceSets.main.output
webXml = project.file('web.xml')
destinationDir = project.file(distDir)
}
task contribJar(type:Jar) {
dependsOn voldJar, testJar, sourceSets.contrib.output
baseName = projectName + "-contrib"
from sourceSets.contrib.output
destinationDir = project.file(distDir)
}
task srcJar(type: Jar, dependsOn: classes) {
classifier = 'src'
from sourceSets.main.java.srcDirs
destinationDir = project.file(distDir)
}
artifacts {
archives voldJar
archives testJar
archives contribJar
archives srcJar
}
clean {
delete(distDir)
delete('lib')
doLast { deleteDirectoryContents(javaDocDir) }
}
task copySources (type: Copy) {
from ('.') { include 'bin/**' }
from ('.') { include distDir + '/*.jar'}
from ('.') { exclude distDir + '/**' ,'bin/**' , 'build/**', '.git/**' , '.gradle/**' }
into archiveDirectoryPath
}
task zip (type: Zip) {
dependsOn copySources
baseName = projectName
from(distDir) {
include archiveDirectoryName + '/bin/**'
fileMode = 0755
}
from(distDir) {
include archiveDirectoryName + '/**'
exclude archiveDirectoryName + '/bin/**'
}
destinationDir = project.file(distDir)
}
task tar (type: Tar) {
dependsOn copySources
compression = Compression.GZIP
baseName = projectName
extension = "tar.gz"
from(distDir) {
include archiveDirectoryName + '/bin/**'
fileMode = 0755
}
from(distDir) {
include archiveDirectoryName + '/**'
exclude archiveDirectoryName + '/bin/**'
}
destinationDir = project.file(distDir)
}
task copyDeps(type: Copy) {
from {configurations.compile }
into "lib"
}
jar.dependsOn contribJar,srcJar, copyDeps
compileContribJava.dependsOn voldJar
copySources.dependsOn jar
tasks.withType(Test) {
// ant restarts jvm for each tests, If not restarted the test runs into outOfMemory even
// if you set the JVM to 8gb. On inspecting most of the space is consumed by int[] of
// Histogram in the NioSelectorManager. I believe this could be explained by
// creating lots of client factory which creates lot of NIO threads. Did not proceed
// further as I will be maintaining compatbility with ant. Also if you dont fork for each
// tests JMX bean related tests will fail.
// Do not set the max parallelism as there are tests that uses the same port and will
// run into bind exceptions.
maxHeapSize = "2g"
forkEvery = 1
// If ignoreFailures is not set, then merged reports will not be generated
// Gradle aborts further tasks on test failure. so if you run junitAll
// which runs 3 tests, reports task will never be run on failure cases.
ignoreFailures = true
//ignoreFailures = gradle.startParameter.continueOnFailure
useJUnit()
testLogging {
events "passed", "skipped", "failed"
exceptionFormat = 'full'
}
afterTest { test, result ->
logger.lifecycle("testFinished: $test, result: $result.resultType")
}
doFirst {
def classesSize = candidateClassFiles.files.size()
logger.lifecycle("{} starts executing {} test classes {}",
path, classesSize, classesSize > 0? "(" + candidateClassFiles.files*.name[0] + ", ...)" : "")
}
//all standard error messages from tests will get routed to 'DEBUG' level messages.
//logging.captureStandardError(LogLevel.DEBUG)
//all standard output messages from tests will get routed to 'DEBUG' level messages.
//logging.captureStandardOutput(LogLevel.DEBUG)
//Set reasonable defaults for reports location
reports.html.destination = file("$project.buildDir/reports/$name")
reports.junitXml.destination = file("$project.buildDir/$name-results")
//Set reasonable defaults classpath and classes dir. They can be reconfigured in an individual task.
it.testClassesDir = sourceSets.test.output.classesDir
classpath = sourceSets.test.runtimeClasspath
}
task resetConfig() {
doLast {
def DirsToDelete = ["STORES" , ".temp", ".version", "data"]
def deleteRecursively
deleteRecursively = { file ->
file.eachFile() {f ->
if(f.directory) {
if( DirsToDelete.contains(f.getName()) )
{
println "deleting ${f.getAbsolutePath()}"
delete f
}
else
{
deleteRecursively(f)
}
}
}
}
deleteRecursively (new File("config"))
}
}
task junit(dependsOn: test)
Collection<String> testClassesFrom(String dir, String include = '**/*Test.*') {
//take all *Test.java files found in given dir, make the path relative and replace .java with .class
fileTree(dir: dir, includes: [include]).collect { it.absolutePath.replaceAll(file(dir).absolutePath + "/", "").replaceAll(".java\$", ".class")}
}
test {
description = "Runs acceptance tests"
include testClassesFrom(unitTestSrcDir)
}
task junitLong(type: Test) {
description = "Runs long junit tests"
include testClassesFrom(longTestSrcDir)
}
task junitRebalance(type: Test) {
include testClassesFrom(unitTestSrcDir, '**/*Rebalance*Test.java')
}
task junitRebalanceLong(type: Test) {
include testClassesFrom(longTestSrcDir, '**/*Rebalance*Test.java')
}
task contribJunit(type: Test) {
description = "Run contrib junit tests except EC2 and Krati tests."
it.testClassesDir = file(contribClassesDir)
exclude '**/*PerformanceTest.class'
exclude '**/*RemoteTest.class'
exclude '**/Ec2*Test.class'
exclude '**/Krati*Test.class'
exclude '**/HadoopStoreBuilder*Test.class'
classpath += sourceSets.contrib.runtimeClasspath + sourceSets.contrib.output
}
task junitAll(type: TestReport) {
reportOn test, junitLong, contribJunit
destinationDir = file("$project.buildDir/reports/$name")
}
task aggregatedJunit(type: TestReport) {
destinationDir = file("$project.buildDir/reports/$name")
}
tasks.withType(Test) {
finalizedBy aggregatedJunit
doLast { aggregatedJunit.reportOn it.binResultsDir }
}
task wrapper(type: Wrapper) { gradleVersion = '2.0' }
dependencies {
// Avro serialization format
compile 'org.apache.avro:avro:1.4.0'
// INTERNAL_LIBS azkaban version not found
// azkaban-common-0.05.jar
// INTERNAL_LIBS Used for tomcat deployment, not sure if anyone uses it
// catalina-ant.jar , version not found in maven central
// coders decoders containing the Base64,binary encoding
compile 'commons-codec:commons-codec:1.4'
// TRANSITIVE_DEPENDENCY Contrib jar depends on commons-configuration-1.6.jar
// commons-configuration instead depends on commons-collection
//compile 'commons-collections:commons-collections:3.2.1'
// Used by MySql storage engine classes
// The jar supports database connection pooling
compile 'commons-dbcp:commons-dbcp:1.2.2'
// commons io is used at many places
// IOUtils, FileUtils and ByteArrayOutputStream
compile 'commons-io:commons-io:1.4'
// LZF compression strategy for store and tests.
compile 'com.ning:compress-lzf:0.9.1'
// TRANSITIVE_DEPENDENCY tusk jar internally uses gson.
compile 'com.google.code.gson:gson:2.2.4'
// Used all over the place for collections
compile 'com.google.guava:guava:14.0.1'
// used for readonly store hdfs fetcher.
compile 'org.apache.hadoop:hadoop-auth:2.0.5-alpha'
// used at lots of places. Seems like there is some overlap between httpclient and core, but not clear
compile 'org.apache.httpcomponents:httpclient:4.1.2'
// contains both http server and client functionalities. Used for HttpResponse but could be used at more places.
compile 'org.apache.httpcomponents:httpcore:4.1.2'
// JSON mapping library from Java Objects to JSON
compile 'org.codehaus.jackson:jackson-mapper-asl:1.4.0'
// JSON processing library
compile 'org.codehaus.jackson:jackson-core-asl:1.4.0'
// Used for reading XML files and Document.
compile 'jdom:jdom:1.1'
// je- BDB je 5.0.88 is not available in maven central repository.
// Could be possibly found in oracle maven repository, but not sure
/*
groupId>com.sleepycat</groupId>
<artifactId>je</artifactId>
<version>3.3.75</version>
groupId>com.sleepycat</groupId>
<artifactId>je</artifactId>
<version>3.3.75</version>
*/
// Jetty is used for HttpService and tests. Jetty Util is used for QueuedThreadPool class.
compile 'org.mortbay.jetty:jetty-util:6.1.18'
compile 'org.mortbay.jetty:jetty:6.1.18'
// A line processing library for command line. No compile time dependency
// Used by Voldemort shell
compile 'jline:jline:0.9.94'
// jna is library for invoking native functions
// used in the readonly store
compile 'net.java.dev.jna:jna:3.2.7'
// joda time is replacement for Java Date and Time
// used in readonly store code.
compile 'joda-time:joda-time:1.6'
// Used for argument command line parsing
compile 'net.sf.jopt-simple:jopt-simple:4.6'
// INTERNAL_LIBS thrift is one of the supported serialization format.
// libthrift 0.5 version is not found in the maven central repository.
// The closest version available is 0.6.1
// libthrift-0.5.0.jar
// log4j - logger used in almost all files
compile 'log4j:log4j:1.2.15'
// used in readonly store and Co-ordinator
compile 'javax.mail:mail:1.4.1'
// Used in co-ordinator and rest services
compile 'io.netty:netty:3.5.8.Final'
// TRANSITIVE_DEPENDENCY Paranamer is a library that allows the parameter names of non-private methods and constructors to be accessed at runtime
// Avro has a dependency on paranamer
// compile 'com.thoughtworks.paranamer:paranamer:2.1'
// protobuf is a supported protocol format between voldemort client and server
compile 'com.google.protobuf:protobuf-java:2.3.0'
// Command line shell is also supported in scala
compile 'org.scala-lang:scala-compiler:2.10.4'
compile 'org.scala-lang:scala-library:2.10.4'
compile 'org.scala-lang:scala-reflect:2.10.4'
// Servlet
compile 'javax.servlet:servlet-api:2.5'
// slf4j is another logging abstraction framework.
// It is used by the apache.avro, apache.hadoop and r2 clients
compile 'org.slf4j:slf4j-api:1.5.6'
compile 'org.slf4j:slf4j-log4j12:1.5.6'
// snappy is one of the supported compression strategies in voldemort
compile 'org.iq80.snappy:snappy:0.2'
// could not locate tusk in maven central repository
// tusk-0.0.2.jar
// Velocity is a simple yet powerful Java-based template engine that renders data
// from plain Java objects to text, xml, email, SQL, Post Script, HTML etc
// Velocity is used for Http Server GUI
compile 'org.apache.velocity:velocity:1.6.2'
// TRANSITIVE_DEPENDENCY Apache XML Parser
// used by jdom
// compile 'xerces:xercesImpl:2.9.1'
compile fileTree(dir: privateLibDir, includes: ['**/*.jar'])
// cern library containing high performance Maps for int and double
// Currently only used in the tests
testCompile 'colt:colt:1.2.0'
// Used in ec2 testing , is a wrapper over any logging framework
testCompile 'commons-logging:commons-logging:1.1.1'
// Used in resource pool perf testing class
testCompile 'commons-pool:commons-pool:1.5.2'
testRuntime 'mysql:mysql-connector-java:5.1.31'
// Used for unit tests and other automated testing
testCompile 'junit:junit:4.6'
// Mockito is written by our beloved friend Szczepan Faber :)
// Mocking framework used in some tests
testCompile 'org.mockito:mockito-all:1.8.5'
contribCompile sourceSets.main.output
contribCompile sourceSets.test.output
// declaring contribCompile dependencies as compile dependencies
// otherwise while copying dependencies to lib directory
// conflict resolution is not done properly across sourceSets
// and we end up with 2 versions of few jars like ( log4j, servlet etc. )
compile 'commons-configuration:commons-configuration:1.6'
compile 'org.apache.hadoop:hadoop-core:1.0.4'
compile 'com.linkedin.pegasus:r2:1.8.3'
compile 'com.linkedin.pegasus:data:1.8.3'
compile 'com.linkedin.pegasus:pegasus-common:1.8.3'
compile 'com.google.code.typica:typica:1.7.2'
compile 'com.sna-projects.krati:krati:0.4.9'
}
eclipse {
project {
buildCommand 'org.eclipse.jdt.core.javabuilder'
}
jdt {
// Currently the javac Version of jar is set to 1.5
// But @Override works differently between both, so overriding here
sourceCompatibility = targetCompatibility = 1.6
}
classpath {
defaultOutputDir = project.file('classes')
file {
// SourceSets creates multiple src/java in .classpath with varying includes
// But eclipse 4.3 is complaining about duplicate classpaths
// If contrib root is included instead of the seperate projects, eclipse
// expects the package to be different. ( contrib.restclient.test.voldemort.client
// versus voldemort.client ). So removing all the src entries and adding the previous
// src entries which used to work.
whenMerged { classpath ->
classpath.entries.removeAll { entry ->
(entry.kind == 'src' )
}
}
withXml {
def node = it.asNode()
[
"src/java",
"contrib/ec2-testing/resources",
"contrib/ec2-testing/src/java",
"contrib/ec2-testing/test",
"contrib/hadoop-store-builder/test",
"contrib/hadoop-store-builder/src/java",
"test/unit",
"test/integration",
"test/common",
"test/long",
"example/java",
"contrib/krati/src/java",
"contrib/krati/test",
"contrib/collections/src/java",
"contrib/collections/test",
"contrib/restclient/src/java",
"contrib/restclient/test"
]
.each{
node.appendNode('classpathentry', [kind: 'src', path: "$it"])
}
}
}
}
}