buildscript { repositories { mavenLocal() mavenCentral() // jcenter() maven { url 'https://plugins.gradle.org/m2/' } } dependencies { // classpath group: 'net.researchgate', name: 'gradle-release', version: '2.8.1' classpath 'gradle.plugin.org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.10.1' // classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.5' classpath 'com.github.jengelman.gradle.plugins:shadow:6.0.0' classpath 'com.adarshr:gradle-test-logger-plugin:2.0.0' } } repositories { mavenLocal() mavenCentral() maven { url "https://jitpack.io" } } //apply plugin: 'net.researchgate.release' apply plugin: 'jacoco' jacoco { toolVersion = '0.8.5' } allprojects { apply plugin: 'java-library' apply plugin: 'base' apply plugin: 'com.github.kt3k.coveralls' apply plugin: 'com.adarshr.test-logger' group = "org.grobid" tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } } subprojects { apply plugin: 'java' apply plugin: 'maven-publish' // apply plugin: 'com.jfrog.bintray' publishing { publications { mavenJava(MavenPublication) { from components.java } } repositories { mavenLocal() } } task install(dependsOn: publishToMavenLocal) sourceCompatibility = 1.8 targetCompatibility = 1.8 repositories { mavenCentral() // jcenter() maven { url new File(rootProject.rootDir, "grobid-core/localLibs") } maven { url "https://jitpack.io" } } configurations { all*.exclude group: 'org.slf4j', module: "slf4j-log4j12" //all*.exclude group: 'log4j', module: "log4j" // implementation.setCanBeResolved(true) } ext { // treating them separately, these jars will be flattened into grobid-core.jar on installing, // to avoid missing dependencies from the projects that include grobid-core (see 'jar' task in grobid-core) localLibs = ['crfpp-1.0.2.jar', 'langdetect-1.1-20120112.jar', 'wipo-analysers-0.0.2.jar', 'imageio-pnm-1.0.jar', 'wapiti-1.5.0.jar'] } dependencies { // packaging local libs inside grobid-core.jar implementation fileTree(dir: new File(rootProject.rootDir, 'grobid-core/localLibs'), include: localLibs) testImplementation "junit:junit:4.12" testImplementation "org.easymock:easymock:3.4" testImplementation "org.powermock:powermock-api-easymock:2.0.7" testImplementation "org.powermock:powermock-module-junit4:2.0.7" testImplementation "xmlunit:xmlunit:1.6" testImplementation "org.hamcrest:hamcrest-all:1.3" implementation "com.cybozu.labs:langdetect:1.1-20120112" implementation "com.rockymadden.stringmetric:stringmetric-core_2.10:0.27.3" implementation "commons-pool:commons-pool:1.6" implementation "commons-io:commons-io:2.5" implementation "org.apache.commons:commons-lang3:3.6" implementation "org.apache.commons:commons-collections4:4.1" implementation "commons-dbutils:commons-dbutils:1.7" implementation "com.google.guava:guava:28.2-jre" implementation "org.apache.httpcomponents:httpclient:4.5.3" implementation 'org.apache.commons:commons-text:1.8' implementation "com.fasterxml.jackson.core:jackson-core:2.10.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.10.1" implementation "com.fasterxml.jackson.module:jackson-module-afterburner:2.10.1" //implementation 'com.fasterxml.jackson.core:jackson-annotations:2.10.1' implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.10.1" } task sourceJar(type: Jar) { description = 'A jar that contains source code' archiveClassifier = 'sources' from project.sourceSets.main.java duplicatesStrategy = DuplicatesStrategy.EXCLUDE } artifacts { archives sourceJar archives jar } /* // removing - this currently breaks the release task gitRev(type: Exec) { commandLine 'git', 'rev-parse', '--short', 'HEAD' standardOutput = new ByteArrayOutputStream() ext.hash = { standardOutput.toString() } } // allows to upload non-snapshot versions that are tight to git revision (substitutes the version if it's a snapshot one) // usage "gradle uploadArchives -PversionWithGitRev" task changeVersionIfNeeded(dependsOn: gitRev) { doLast { def suffix = '-SNAPSHOT' if (project.hasProperty('versionWithGitRev') && project.version.endsWith(suffix)) { println project.version project.version = project.version.substring(0, project.version.length() - suffix.length()) + "-" + tasks.gitRev.hash() println "Version to be uploaded: " + project.version } } }*/ //compileJava.dependsOn(changeVersionIfNeeded) // uploadArchives { // // if you want to enable uploading to some maven repo, add those properties to ~/.gradle/gradle.properties, e.g.: // /* // mavenRepoUserName=maven_username // mavenRepoPassword=super_secret // mavenRepoReleasesUrl=https://nexus3.example.org/repository/maven-releases/ // mavenRepoSnapshotsUrl=https://nexus3.example.org/repository/maven-snapshots/ // */ // def user = project.hasProperty('mavenRepoUserName') ? project.findProperty('mavenRepoUserName') : '' // def password = project.hasProperty('mavenRepoPassword') ? project.findProperty('mavenRepoPassword') : '' // def rurl = project.hasProperty('mavenRepoReleasesUrl') ? project.findProperty('mavenRepoReleasesUrl') : '' // def surl = project.hasProperty('mavenRepoSnapshotsUrl') ? project.findProperty('mavenRepoSnapshotsUrl') : '' // // repositories.mavenDeployer { // repository(url: rurl) { // authentication(userName: user, password: password) // } // snapshotRepository(url: surl) { // authentication(userName: user, password: password) // } // // } // } /* bintray { dryRun = false // Whether to run this as dry-run, without deploying publish = true // If version should be auto published after an upload // read from ~/.gradle/gradle.properties. Set those up before releasing user = project.hasProperty('bintrayUser') ? bintrayUser : '' key = project.hasProperty('bintrayApiKey') ? bintrayApiKey : '' configurations = ['archives'] pkg { name = 'grobid' repo = 'maven' userOrg = 'rookies' licenses = ['Apache-2.0'] vcsUrl = 'https://github.com/kermitt2/grobid' issueTrackerUrl = 'https://github.com/kermitt2/grobid/issues' githubRepo = 'kermitt2/grobid' version { name = rootProject.version vcsTag = rootProject.version released = new Date() } } } */ test { testLogging.showStandardStreams = true // enable for having separate test executor for different tests forkEvery = 1 maxHeapSize = "1024m" } //rootProject.afterReleaseBuild.dependsOn subprojects.bintrayUpload } /** SUBPROJECTS **/ project("grobid-core") { apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'jacoco' configurations { shadedLib } dependencies { implementation(group: 'xml-apis', name: 'xml-apis') { // otherwise xml-apis 2.0.1 will come from XOM and will result in // java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal //TODO: sort out this problem better version { strictly '1.4.01' } } // Logs api 'org.slf4j:slf4j-api:1.7.25' api 'org.slf4j:slf4j-log4j12:1.7.25' runtimeOnly 'org.slf4j:slf4j-jdk14:1.7.25' implementation "org.apache.pdfbox:pdfbox:2.0.18" api "xerces:xercesImpl:2.12.0" api "net.arnx:jsonic:1.3.10" api "net.sf.saxon:Saxon-HE:9.6.0-9" api "xom:xom:1.3.2" api 'javax.xml.bind:jaxb-api:2.3.0' implementation "joda-time:joda-time:2.9.9" implementation "org.apache.lucene:lucene-analyzers-common:4.5.1" implementation 'black.ninia:jep:3.9.1' implementation 'org.apache.opennlp:opennlp-tools:1.9.1' implementation group: 'org.jruby', name: 'jruby-complete', version: '9.2.13.0' shadedLib "org.apache.lucene:lucene-analyzers-common:4.5.1" } jar { from { // project.configurations.implementation.collect { project.configurations.runtimeClasspath.collect { it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : [] } } // duplicatesStrategy = DuplicatesStrategy.EXCLUDE } shadowJar { archiveClassifier = 'onejar' mergeServiceFiles() zip64 true manifest { attributes 'Main-Class': 'org.grobid.core.main.batch.GrobidMain' } from sourceSets.main.output from { // project.configurations.implementation.collect { project.configurations.runtimeClasspath.collect { it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : [] } } configurations = [project.configurations.shadedLib, project.configurations.runtimeClasspath] relocate 'org.apache.lucene', 'org.grobid.shaded.org.apache.lucene' } artifacts { archives shadowJar archives jar } processResources { filesMatching('grobid-version.txt') { filter { it.replace('project.version', project.property('version')) } } } } project("grobid-home") { task packageGrobidHome(type: Zip) { from('.') { include("config/*") include("language-detection/**") include("sentence-segmentation/**") include("lib/**") include("pdfalto/**") include("models/**") include("lexicon/**") include("schemas/**") include("scripts/**") exclude("models/**/*.old") } into("grobid-home") } artifacts { archives packageGrobidHome } } project(":grobid-service") { apply plugin: 'application' apply plugin: 'jacoco' apply plugin: 'com.github.johnrengelman.shadow' mainClassName = 'org.grobid.service.main.GrobidServiceApplication' tasks.run.workingDir = rootProject.rootDir configurations { all*.exclude group: 'org.slf4j', module: "slf4j-jdk14" all*.exclude group: 'org.slf4j', module: "slf4j-log4j12" all*.exclude group: 'log4j', module: "log4j" } tasks.distZip.enabled = true tasks.distTar.enabled = false tasks.shadowDistZip.enabled = false tasks.shadowDistTar.enabled = false dependencies { implementation project(':grobid-core') implementation "io.dropwizard:dropwizard-core:1.3.23" implementation "io.dropwizard:dropwizard-assets:1.3.23" implementation "com.hubspot.dropwizard:dropwizard-guicier:1.3.5.0" implementation "io.dropwizard:dropwizard-testing:1.3.23" implementation "io.dropwizard:dropwizard-forms:1.3.23" implementation "io.dropwizard:dropwizard-client:1.3.23" implementation "io.dropwizard:dropwizard-auth:1.3.23" implementation "org.apache.pdfbox:pdfbox:2.0.3" implementation "javax.activation:activation:1.1.1" implementation "io.prometheus:simpleclient_dropwizard:0.11.0" implementation "io.prometheus:simpleclient_servlet:0.11.0" testImplementation "io.dropwizard:dropwizard-testing:1.3.17" } shadowJar { archiveClassifier = 'onejar' mergeServiceFiles() zip64 true manifest { attributes 'Main-Class': 'org.grobid.core.main.batch.GrobidMain' } duplicatesStrategy = DuplicatesStrategy.EXCLUDE } artifacts { archives shadowJar } distributions { main { contents { from(new File(rootProject.rootDir, "grobid-service/README.md")) { into "doc" } //from(new File(rootProject.rootDir, "../grobid-home/config/grobid.yaml")) { // into "config" //} } } } } project(":grobid-trainer") { apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'jacoco' dependencies { implementation(group: 'xml-apis', name: 'xml-apis') { // otherwise xml-apis 2.0.1 will come from XOM and will result in // java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal //TODO: sort out this problem better version { strictly '1.4.01' } } implementation project(':grobid-core') implementation "com.rockymadden.stringmetric:stringmetric-core_2.10:0.27.3" implementation "me.tongfei:progressbar:0.9.0" implementation 'org.slf4j:slf4j-log4j12:1.7.25' implementation 'org.slf4j:slf4j-api:1.7.25' implementation 'org.slf4j:slf4j-jdk14:1.7.25' } configurations { all*.exclude group: 'org.slf4j', module: "slf4j-jdk14" all*.exclude group: 'org.slf4j', module: "slf4j-log4j12" //all*.exclude group: 'log4j', module: "log4j" } jar { from { // project.configurations.implementation.collect { project.configurations.runtimeClasspath.collect { it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : [] } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE } shadowJar { archiveClassifier = 'onejar' mergeServiceFiles() zip64 true manifest { attributes 'Main-Class': 'org.grobid.trainer.TrainerRunner' } duplicatesStrategy = DuplicatesStrategy.EXCLUDE } artifacts { archives shadowJar archives jar } def trainerTasks = [ "train_name_header" : "org.grobid.trainer.NameHeaderTrainer", "train_name_citation" : "org.grobid.trainer.NameCitationTrainer", "train_affiliation_address" : "org.grobid.trainer.AffiliationAddressTrainer", "train_header" : "org.grobid.trainer.HeaderTrainer", "train_fulltext" : "org.grobid.trainer.FulltextTrainer", "train_shorttext" : "org.grobid.trainer.ShorttextTrainer", "train_figure" : "org.grobid.trainer.FigureTrainer", "train_table" : "org.grobid.trainer.TableTrainer", "train_citation" : "org.grobid.trainer.CitationTrainer", "train_date" : "org.grobid.trainer.DateTrainer", "train_segmentation" : "org.grobid.trainer.SegmentationTrainer", "train_reference_segmentation": "org.grobid.trainer.ReferenceSegmenterTrainer", "train_ebook_model" : "org.grobid.trainer.EbookTrainer", "train_patent_citation" : "org.grobid.trainer.PatentParserTrainer" ] trainerTasks.each { taskName, mainClassName -> tasks.create(name: taskName, type: JavaExec, group: 'modeltraining') { main = mainClassName classpath = sourceSets.main.runtimeClasspath } } // evaluation tasks ext.getArg = { propName, defaultVal -> return project.hasProperty(propName) ? project.getProperty(propName) : defaultVal; } // run like this: // ./gradlew jatsEval -Pp2t=/path/to/goldenSet // ./gradlew jatsEval -Pp2t=/path/to/goldenSet -Prun=1 -PfileRatio=0.1 // ./gradlew teiEval -Pp2t=/path/to/goldenSet // ./gradlew PrepareDOIMatching -Pp2t=ABS_PATH_TO_PMC/PMC_sample_1943 // ./gradlew EvaluateDOIMatching -Pp2t=ABS_PATH_TO_PMC/PMC_sample_1943 task(jatsEval, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') { main = 'org.grobid.trainer.evaluation.EndToEndEvaluation' classpath = sourceSets.main.runtimeClasspath args 'nlm', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0') jvmArgs '-Xmx3072m' } task(teiEval, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') { main = 'org.grobid.trainer.evaluation.EndToEndEvaluation' classpath = sourceSets.main.runtimeClasspath args 'tei', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0') jvmArgs '-Xmx3072m' } task(PrepareDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') { main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching' classpath = sourceSets.main.runtimeClasspath args 'data', getArg('p2t', '.') jvmArgs '-Xmx3072m' } task(EvaluateDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') { main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching' classpath = sourceSets.main.runtimeClasspath args 'eval', getArg('p2t', '.') jvmArgs '-Xmx3072m' } } /** JACOCO **/ tasks.register("codeCoverageReport", JacocoReport) { // If a subproject applies the 'jacoco' plugin, add the result it to the report subprojects { subproject -> subproject.plugins.withType(JacocoPlugin).configureEach { subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).configureEach { testTask -> sourceSets subproject.sourceSets.main executionData(testTask) } // To automatically run `test` every time `./gradlew codeCoverageReport` is called, // you may want to set up a task dependency between them as shown below. // Note that this requires the `test` tasks to be resolved eagerly (see `forEach`) which // may have a negative effect on the configuration time of your build. subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).forEach { rootProject.tasks.codeCoverageReport.dependsOn(it) } } } // XML -> coveralls, // HTML -> for manual check reports { xml.enabled true html.enabled true csv.enabled true } } /** COVERALLS **/ coveralls { sourceDirs = files(subprojects.sourceSets.main.allSource.srcDirs).files.absolutePath jacocoReportPath = "build/reports/jacoco/codeCoverageReport/codeCoverageReport.xml" } tasks.coveralls { dependsOn codeCoverageReport } wrapper { gradleVersion "6.5.1" } build.dependsOn project.getSubprojects().collect({ it.getTasks().getByName("build") })