https://github.com/kermitt2/grobid
Tip revision: a20d6cea4f0f00a743dec2bef262c0a9e81cef97 authored by lopez on 23 October 2020, 05:22:23 UTC
experiment to control spurious figure captions
experiment to control spurious figure captions
Tip revision: a20d6ce
build.gradle
buildscript {
repositories {
mavenLocal()
mavenCentral()
jcenter()
maven {
url 'https://plugins.gradle.org/m2/'
}
}
dependencies {
classpath group: 'net.researchgate', name: 'gradle-release', version: '2.8.1'
classpath 'gradle.plugin.org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.10.1'
classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.5'
classpath 'com.github.jengelman.gradle.plugins:shadow:6.0.0'
classpath 'com.adarshr:gradle-test-logger-plugin:2.0.0'
}
}
repositories {
mavenCentral()
}
apply plugin: 'net.researchgate.release'
apply plugin: 'jacoco'
jacoco {
toolVersion = '0.8.5'
}
allprojects {
apply plugin: 'java-library'
apply plugin: 'base'
apply plugin: 'com.github.kt3k.coveralls'
apply plugin: 'com.adarshr.test-logger'
group = "org.grobid"
tasks.withType(JavaCompile) {
options.encoding = 'UTF-8'
}
}
subprojects {
apply plugin: 'java'
apply plugin: 'maven-publish'
apply plugin: 'com.jfrog.bintray'
publishing {
publications {
mavenJava(MavenPublication) {
from components.java
}
}
repositories {
mavenLocal()
}
}
task install(dependsOn: publishToMavenLocal)
sourceCompatibility = 1.8
targetCompatibility = 1.8
repositories {
mavenCentral()
jcenter()
maven {
url new File(rootProject.rootDir, "grobid-core/localLibs")
}
}
configurations {
all*.exclude group: 'org.slf4j', module: "slf4j-log4j12"
//all*.exclude group: 'log4j', module: "log4j"
// implementation.setCanBeResolved(true)
}
ext {
// treating them separately, these jars will be flattened into grobid-core.jar on installing,
// to avoid missing dependencies from the projects that include grobid-core (see 'jar' task in grobid-core)
localLibs = ['crfpp-1.0.2.jar',
'langdetect-1.1-20120112.jar',
'wipo-analysers-0.0.2.jar',
'imageio-pnm-1.0.jar',
'wapiti-1.5.0.jar']
}
dependencies {
// packaging local libs inside grobid-core.jar
implementation fileTree(dir: new File(rootProject.rootDir, 'grobid-core/localLibs'), include: localLibs)
testImplementation "junit:junit:4.12"
testImplementation "org.easymock:easymock:3.4"
testImplementation "org.powermock:powermock-api-easymock:2.0.7"
testImplementation "org.powermock:powermock-module-junit4:2.0.7"
testImplementation "xmlunit:xmlunit:1.6"
testImplementation "org.hamcrest:hamcrest-all:1.3"
implementation "com.cybozu.labs:langdetect:1.1-20120112"
implementation "com.rockymadden.stringmetric:stringmetric-core_2.10:0.27.3"
implementation "commons-pool:commons-pool:1.6"
implementation "commons-io:commons-io:2.5"
implementation "org.apache.commons:commons-lang3:3.6"
implementation "org.apache.commons:commons-collections4:4.1"
implementation "commons-dbutils:commons-dbutils:1.7"
implementation "com.google.guava:guava:28.2-jre"
implementation "org.apache.httpcomponents:httpclient:4.5.3"
implementation 'org.apache.commons:commons-text:1.8'
implementation "com.fasterxml.jackson.core:jackson-core:2.10.1"
implementation "com.fasterxml.jackson.core:jackson-databind:2.10.1"
implementation "com.fasterxml.jackson.module:jackson-module-afterburner:2.10.1"
}
task sourceJar(type: Jar) {
description = 'A jar that contains source code'
archiveClassifier = 'sources'
from project.sourceSets.main.java
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
artifacts {
archives sourceJar
archives jar
}
/*
// removing - this currently breaks the release
task gitRev(type: Exec) {
commandLine 'git', 'rev-parse', '--short', 'HEAD'
standardOutput = new ByteArrayOutputStream()
ext.hash = {
standardOutput.toString()
}
}
// allows to upload non-snapshot versions that are tight to git revision (substitutes the version if it's a snapshot one)
// usage "gradle uploadArchives -PversionWithGitRev"
task changeVersionIfNeeded(dependsOn: gitRev) {
doLast {
def suffix = '-SNAPSHOT'
if (project.hasProperty('versionWithGitRev') && project.version.endsWith(suffix)) {
println project.version
project.version = project.version.substring(0, project.version.length() - suffix.length()) + "-" + tasks.gitRev.hash()
println "Version to be uploaded: " + project.version
}
}
}*/
//compileJava.dependsOn(changeVersionIfNeeded)
// uploadArchives {
// // if you want to enable uploading to some maven repo, add those properties to ~/.gradle/gradle.properties, e.g.:
// /*
// mavenRepoUserName=maven_username
// mavenRepoPassword=super_secret
// mavenRepoReleasesUrl=https://nexus3.example.org/repository/maven-releases/
// mavenRepoSnapshotsUrl=https://nexus3.example.org/repository/maven-snapshots/
// */
// def user = project.hasProperty('mavenRepoUserName') ? project.findProperty('mavenRepoUserName') : ''
// def password = project.hasProperty('mavenRepoPassword') ? project.findProperty('mavenRepoPassword') : ''
// def rurl = project.hasProperty('mavenRepoReleasesUrl') ? project.findProperty('mavenRepoReleasesUrl') : ''
// def surl = project.hasProperty('mavenRepoSnapshotsUrl') ? project.findProperty('mavenRepoSnapshotsUrl') : ''
//
// repositories.mavenDeployer {
// repository(url: rurl) {
// authentication(userName: user, password: password)
// }
// snapshotRepository(url: surl) {
// authentication(userName: user, password: password)
// }
//
// }
// }
bintray {
dryRun = false // Whether to run this as dry-run, without deploying
publish = true // If version should be auto published after an upload
// read from ~/.gradle/gradle.properties. Set those up before releasing
user = project.hasProperty('bintrayUser') ? bintrayUser : ''
key = project.hasProperty('bintrayApiKey') ? bintrayApiKey : ''
configurations = ['archives']
pkg {
name = 'grobid'
repo = 'maven'
userOrg = 'rookies'
licenses = ['Apache-2.0']
vcsUrl = 'https://github.com/kermitt2/grobid'
issueTrackerUrl = 'https://github.com/kermitt2/grobid/issues'
githubRepo = 'kermitt2/grobid'
version {
name = rootProject.version
vcsTag = rootProject.version
released = new Date()
}
}
}
test {
testLogging.showStandardStreams = true
// enable for having separate test executor for different tests
// forkEvery = 1
maxHeapSize = "1024m"
}
rootProject.afterReleaseBuild.dependsOn subprojects.bintrayUpload
}
/** SUBPROJECTS **/
project("grobid-core") {
apply plugin: 'com.github.johnrengelman.shadow'
apply plugin: 'jacoco'
configurations {
shadedLib
}
dependencies {
implementation(group: 'xml-apis', name: 'xml-apis') {
// otherwise xml-apis 2.0.1 will come from XOM and will result in
// java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
//TODO: sort out this problem better
version {
strictly '1.4.01'
}
}
// Logs
api 'org.slf4j:slf4j-api:1.7.25'
api 'org.slf4j:slf4j-log4j12:1.7.25'
runtimeOnly 'org.slf4j:slf4j-jdk14:1.7.25'
implementation "org.apache.pdfbox:pdfbox:2.0.18"
api "xerces:xercesImpl:2.12.0"
api "net.arnx:jsonic:1.3.10"
api "net.sf.saxon:Saxon-HE:9.6.0-9"
api "xom:xom:1.3.2"
api 'javax.xml.bind:jaxb-api:2.3.0'
implementation "joda-time:joda-time:2.9.9"
implementation "org.apache.lucene:lucene-analyzers-common:4.5.1"
implementation 'black.ninia:jep:3.9.1'
implementation 'org.apache.opennlp:opennlp-tools:1.9.1'
implementation group: 'org.jruby', name: 'jruby-complete', version: '9.2.13.0'
shadedLib "org.apache.lucene:lucene-analyzers-common:4.5.1"
}
jar {
from {
// project.configurations.implementation.collect {
project.configurations.runtimeClasspath.collect {
it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
}
}
// duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
shadowJar {
archiveClassifier = 'onejar'
mergeServiceFiles()
zip64 true
manifest {
attributes 'Main-Class': 'org.grobid.core.main.batch.GrobidMain'
}
from sourceSets.main.output
from {
// project.configurations.implementation.collect {
project.configurations.runtimeClasspath.collect {
it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
}
}
configurations = [project.configurations.shadedLib, project.configurations.runtimeClasspath]
relocate 'org.apache.lucene', 'org.grobid.shaded.org.apache.lucene'
}
artifacts {
archives shadowJar
archives jar
}
processResources {
filesMatching('grobid-version.txt') {
filter {
it.replace('project.version', project.property('version'))
}
}
}
}
project("grobid-home") {
task packageGrobidHome(type: Zip) {
from('.') {
include("config/*")
include("language-detection/**")
include("sentence-segmentation/**")
include("lib/**")
include("pdf2xml/**")
include("models/**")
include("lexicon/**")
include("schemas/**")
exclude("models/**/*.old")
}
into("grobid-home")
}
artifacts {
archives packageGrobidHome
}
}
project(":grobid-service") {
apply plugin: 'application'
apply plugin: 'jacoco'
apply plugin: 'com.github.johnrengelman.shadow'
mainClassName = 'org.grobid.service.main.GrobidServiceApplication'
tasks.run.workingDir = rootProject.rootDir
configurations {
all*.exclude group: 'org.slf4j', module: "slf4j-jdk14"
all*.exclude group: 'org.slf4j', module: "slf4j-log4j12"
all*.exclude group: 'log4j', module: "log4j"
}
tasks.distZip.enabled = true
tasks.distTar.enabled = false
tasks.shadowDistZip.enabled = false
tasks.shadowDistTar.enabled = false
dependencies {
implementation project(':grobid-core')
implementation "io.dropwizard:dropwizard-core:1.3.23"
implementation "io.dropwizard:dropwizard-assets:1.3.23"
implementation "com.hubspot.dropwizard:dropwizard-guicier:1.3.5.0"
implementation "io.dropwizard:dropwizard-testing:1.3.23"
implementation "io.dropwizard:dropwizard-forms:1.3.23"
implementation "io.dropwizard:dropwizard-client:1.3.23"
implementation "io.dropwizard:dropwizard-auth:1.3.23"
implementation "org.apache.pdfbox:pdfbox:2.0.3"
implementation "javax.activation:activation:1.1.1"
testImplementation "io.dropwizard:dropwizard-testing:1.3.17"
}
shadowJar {
archiveClassifier = 'onejar'
mergeServiceFiles()
zip64 true
manifest {
attributes 'Main-Class': 'org.grobid.core.main.batch.GrobidMain'
}
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
artifacts {
archives shadowJar
}
distributions {
main {
contents {
from(new File(rootProject.rootDir, "grobid-service/README.md")) {
into "doc"
}
from(new File(rootProject.rootDir, "grobid-service/config/config.yaml")) {
into "config"
}
//from(new File(rootProject.rootDir, "grobid-service/config/grobid_service.properties")) {
// into "config"
//}
}
}
}
}
project(":grobid-trainer") {
apply plugin: 'com.github.johnrengelman.shadow'
apply plugin: 'jacoco'
dependencies {
implementation(group: 'xml-apis', name: 'xml-apis') {
// otherwise xml-apis 2.0.1 will come from XOM and will result in
// java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
//TODO: sort out this problem better
version {
strictly '1.4.01'
}
}
implementation project(':grobid-core')
implementation "com.rockymadden.stringmetric:stringmetric-core_2.10:0.27.3"
}
jar {
from {
// project.configurations.implementation.collect {
project.configurations.runtimeClasspath.collect {
it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
}
}
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
shadowJar {
archiveClassifier = 'onejar'
mergeServiceFiles()
zip64 true
manifest {
attributes 'Main-Class': 'org.grobid.trainer.TrainerRunner'
}
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
artifacts {
archives shadowJar
archives jar
}
def trainerTasks = [
"train_name_header" : "org.grobid.trainer.NameHeaderTrainer",
"train_name_citation" : "org.grobid.trainer.NameCitationTrainer",
"train_affiliation_address" : "org.grobid.trainer.AffiliationAddressTrainer",
"train_header" : "org.grobid.trainer.HeaderTrainer",
"train_fulltext" : "org.grobid.trainer.FulltextTrainer",
"train_shorttext" : "org.grobid.trainer.ShorttextTrainer",
"train_figure" : "org.grobid.trainer.FigureTrainer",
"train_table" : "org.grobid.trainer.TableTrainer",
"train_citation" : "org.grobid.trainer.CitationTrainer",
"train_date" : "org.grobid.trainer.DateTrainer",
"train_segmentation" : "org.grobid.trainer.SegmentationTrainer",
"train_reference_segmentation": "org.grobid.trainer.ReferenceSegmenterTrainer",
"train_ebook_model" : "org.grobid.trainer.EbookTrainer",
"train_patent_citation" : "org.grobid.trainer.PatentParserTrainer"
]
trainerTasks.each { taskName, mainClassName ->
tasks.create(name: taskName, type: JavaExec, group: 'modeltraining') {
main = mainClassName
classpath = sourceSets.main.runtimeClasspath
}
}
// evaluation tasks
ext.getArg = { propName, defaultVal ->
return project.hasProperty(propName) ? project.getProperty(propName) : defaultVal;
}
// run like this:
// ./gradlew jatsEval -Pp2t=/path/to/goldenSet
// ./gradlew jatsEval -Pp2t=/path/to/goldenSet -Prun=1 -PfileRatio=0.1
// ./gradlew teiEval -Pp2t=/path/to/goldenSet
// ./gradlew PrepareDOIMatching -Pp2t=ABS_PATH_TO_PMC/PMC_sample_1943
// ./gradlew EvaluateDOIMatching -Pp2t=ABS_PATH_TO_PMC/PMC_sample_1943
task(jatsEval, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EndToEndEvaluation'
classpath = sourceSets.main.runtimeClasspath
args 'nlm', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0')
jvmArgs '-Xmx3072m'
}
task(teiEval, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EndToEndEvaluation'
classpath = sourceSets.main.runtimeClasspath
args 'tei', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0')
jvmArgs '-Xmx3072m'
}
task(PrepareDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching'
classpath = sourceSets.main.runtimeClasspath
args 'data', getArg('p2t', '.')
jvmArgs '-Xmx3072m'
}
task(EvaluateDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching'
classpath = sourceSets.main.runtimeClasspath
args 'eval', getArg('p2t', '.')
jvmArgs '-Xmx3072m'
}
}
/** JACOCO **/
tasks.register("codeCoverageReport", JacocoReport) {
// If a subproject applies the 'jacoco' plugin, add the result it to the report
subprojects { subproject ->
subproject.plugins.withType(JacocoPlugin).configureEach {
subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).configureEach { testTask ->
sourceSets subproject.sourceSets.main
executionData(testTask)
}
// To automatically run `test` every time `./gradlew codeCoverageReport` is called,
// you may want to set up a task dependency between them as shown below.
// Note that this requires the `test` tasks to be resolved eagerly (see `forEach`) which
// may have a negative effect on the configuration time of your build.
subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).forEach {
rootProject.tasks.codeCoverageReport.dependsOn(it)
}
}
}
// XML -> coveralls,
// HTML -> for manual check
reports {
xml.enabled true
html.enabled true
csv.enabled true
}
}
/** COVERALLS **/
coveralls {
sourceDirs = files(subprojects.sourceSets.main.allSource.srcDirs).files.absolutePath
jacocoReportPath = "build/reports/jacoco/codeCoverageReport/codeCoverageReport.xml"
}
tasks.coveralls {
dependsOn codeCoverageReport
}
wrapper {
gradleVersion "6.5.1"
}
build.dependsOn project.getSubprojects().collect({ it.getTasks().getByName("build") })