https://github.com/kermitt2/grobid
Tip revision: 5749816fce0ea6cd267901c61eb973fbf56186e4 authored by Luca Foppiano on 07 October 2023, 06:55:29 UTC
proposal for serialize copyright statement in TEI
proposal for serialize copyright statement in TEI
Tip revision: 5749816
build.gradle
buildscript {
repositories {
mavenLocal()
mavenCentral()
maven {
url 'https://plugins.gradle.org/m2/'
}
}
dependencies {
classpath 'gradle.plugin.org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.12.0'
classpath "gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0"
classpath 'com.adarshr:gradle-test-logger-plugin:2.0.0'
}
}
repositories {
mavenLocal()
mavenCentral()
}
apply plugin: 'jacoco'
jacoco {
toolVersion = '0.8.8'
}
allprojects {
apply plugin: 'java-library'
apply plugin: 'base'
apply plugin: 'com.github.kt3k.coveralls'
apply plugin: 'com.adarshr.test-logger'
group = "org.grobid"
tasks.withType(JavaCompile) {
options.encoding = 'UTF-8'
}
}
subprojects {
apply plugin: 'java'
apply plugin: 'maven-publish'
publishing {
publications {
mavenJava(MavenPublication) {
//from components.java
artifact jar
}
}
repositories {
mavenLocal()
}
}
sourceCompatibility = 1.8
targetCompatibility = 1.8
repositories {
mavenCentral()
maven {
url new File(rootProject.rootDir, "grobid-core/localLibs")
}
maven { url "https://jitpack.io" }
}
/*configurations {
all*.exclude group: 'org.slf4j', module: "slf4j-log4j12"
//all*.exclude group: 'log4j', module: "log4j"
// implementation.setCanBeResolved(true)
}*/
ext {
// treating them separately, these jars will be flattened into grobid-core.jar on installing,
// to avoid missing dependencies from the projects that include grobid-core (see 'jar' task in grobid-core)
localLibs = ['crfpp-1.0.2.jar',
'langdetect-1.1-20120112.jar',
'wipo-analysers-0.0.2.jar',
'imageio-pnm-1.0.jar',
'wapiti-1.5.0.jar']
}
dependencies {
// packaging local libs inside grobid-core.jar
implementation fileTree(dir: new File(rootProject.rootDir, 'grobid-core/localLibs'), include: localLibs)
testImplementation "junit:junit:4.12"
testImplementation "org.easymock:easymock:3.4"
testImplementation "org.powermock:powermock-api-easymock:2.0.7"
testImplementation "org.powermock:powermock-module-junit4:2.0.7"
testImplementation "xmlunit:xmlunit:1.6"
testImplementation "org.hamcrest:hamcrest-all:1.3"
implementation "com.cybozu.labs:langdetect:1.1-20120112"
implementation "com.rockymadden.stringmetric:stringmetric-core_2.10:0.27.3"
implementation "commons-pool:commons-pool:1.6"
implementation "commons-io:commons-io:2.5"
implementation "org.apache.commons:commons-lang3:3.6"
implementation "org.apache.commons:commons-collections4:4.1"
implementation 'org.apache.commons:commons-text:1.8'
implementation "commons-dbutils:commons-dbutils:1.7"
implementation "com.google.guava:guava:28.2-jre"
implementation "org.apache.httpcomponents:httpclient:4.5.3"
implementation "black.ninia:jep:4.0.2"
implementation "com.fasterxml.jackson.core:jackson-core:2.10.1"
implementation "com.fasterxml.jackson.core:jackson-databind:2.10.1"
implementation "com.fasterxml.jackson.module:jackson-module-afterburner:2.10.1"
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.10.1"
}
task sourceJar(type: Jar) {
description = 'A jar that contains source code'
archiveClassifier = 'sources'
from project.sourceSets.main.java
}
artifacts {
archives sourceJar
archives jar
}
//compileJava.dependsOn(changeVersionIfNeeded)
// uploadArchives {
// // if you want to enable uploading to some maven repo, add those properties to ~/.gradle/gradle.properties, e.g.:
// /*
// mavenRepoUserName=maven_username
// mavenRepoPassword=super_secret
// mavenRepoReleasesUrl=https://nexus3.example.org/repository/maven-releases/
// mavenRepoSnapshotsUrl=https://nexus3.example.org/repository/maven-snapshots/
// */
// def user = project.hasProperty('mavenRepoUserName') ? project.findProperty('mavenRepoUserName') : ''
// def password = project.hasProperty('mavenRepoPassword') ? project.findProperty('mavenRepoPassword') : ''
// def rurl = project.hasProperty('mavenRepoReleasesUrl') ? project.findProperty('mavenRepoReleasesUrl') : ''
// def surl = project.hasProperty('mavenRepoSnapshotsUrl') ? project.findProperty('mavenRepoSnapshotsUrl') : ''
//
// repositories.mavenDeployer {
// repository(url: rurl) {
// authentication(userName: user, password: password)
// }
// snapshotRepository(url: surl) {
// authentication(userName: user, password: password)
// }
//
// }
// }
test {
testLogging.showStandardStreams = true
// enable for having separate test executor for different tests
forkEvery = 1
maxHeapSize = "1024m"
def libraries = ""
if (Os.isFamily(Os.FAMILY_MAC)) {
if (Os.OS_ARCH.equals("aarch64")) {
libraries = "${file("./grobid-home/lib/mac_arm-64").absolutePath}"
} else {
libraries = "${file("./grobid-home/lib/mac-64").absolutePath}"
}
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
def jepDir = rootProject.rootDir.getAbsolutePath() + "/grobid-home/lib/lin-64/jep"
libraries = jepDir
jepDir = rootProject.rootDir.getAbsolutePath() + "/grobid-home/lib/lin-64"
libraries += ":"+jepDir
} else {
throw new RuntimeException("Unsupported platform!")
}
if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs "--add-opens", "java.base/java.util.stream=ALL-UNNAMED",
"--add-opens", "java.base/java.io=ALL-UNNAMED"
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}
}
/** SUBPROJECTS **/
project("grobid-core") {
apply plugin: 'com.github.johnrengelman.shadow'
apply plugin: 'jacoco'
configurations {
shadedLib
}
dependencies {
implementation(group: 'xml-apis', name: 'xml-apis') {
// otherwise xml-apis 2.0.1 will come from XOM and will result in
// java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
//TODO: sort out this problem better
version {
strictly '1.4.01'
}
}
// Logs
api 'org.slf4j:slf4j-api:1.7.25'
//api 'org.slf4j:slf4j-log4j12:1.7.25'
runtimeOnly 'org.slf4j:slf4j-jdk14:1.7.25'
implementation "org.apache.pdfbox:pdfbox:2.0.18"
api "xerces:xercesImpl:2.12.0"
api "net.arnx:jsonic:1.3.10"
api "net.sf.saxon:Saxon-HE:9.6.0-9"
api "xom:xom:1.3.2"
api 'javax.xml.bind:jaxb-api:2.3.0'
implementation "joda-time:joda-time:2.9.9"
implementation "org.apache.lucene:lucene-analyzers-common:4.5.1"
implementation 'black.ninia:jep:4.0.2'
implementation 'org.apache.opennlp:opennlp-tools:1.9.1'
implementation group: 'org.jruby', name: 'jruby-complete', version: '9.2.13.0'
shadedLib "org.apache.lucene:lucene-analyzers-common:4.5.1"
}
jar {
from {
project.configurations.runtimeClasspath.collect {
it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
}
}
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
shadowJar {
archiveClassifier = 'onejar'
mergeServiceFiles()
zip64 true
manifest {
attributes 'Main-Class': 'org.grobid.core.main.batch.GrobidMain'
}
from sourceSets.main.output
from {
project.configurations.runtimeClasspath.collect {
it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
}
}
configurations = [project.configurations.shadedLib, project.configurations.runtimeClasspath]
relocate 'org.apache.lucene', 'org.grobid.shaded.org.apache.lucene'
}
artifacts {
archives jar
archives shadowJar
}
processResources {
filesMatching('grobid-version.txt') {
filter {
it.replace('project.version', project.property('version'))
}
}
}
task install {
dependsOn publishToMavenLocal
dependsOn 'shadowJar'
}
}
project("grobid-home") {
task packageGrobidHome(type: Zip) {
from('.') {
include("config/*")
include("language-detection/**")
include("sentence-segmentation/**")
include("lib/**")
include("pdfalto/**")
include("models/**")
include("lexicon/**")
include("schemas/**")
include("scripts/**")
exclude("models/**/*.old")
}
into("grobid-home")
}
artifacts {
archives packageGrobidHome
}
}
import org.apache.tools.ant.taskdefs.condition.Os
project(":grobid-service") {
apply plugin: 'application'
apply plugin: 'jacoco'
apply plugin: 'com.github.johnrengelman.shadow'
mainClassName = 'org.grobid.service.main.GrobidServiceApplication'
tasks.run {
def libraries = ""
if (Os.isFamily(Os.FAMILY_MAC)) {
if (Os.OS_ARCH.equals("aarch64")) {
libraries = "${file("../grobid-home/lib/mac_arm-64").absolutePath}"
} else {
libraries = "${file("../grobid-home/lib/mac-64").absolutePath}"
}
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
libraries = "${file("../grobid-home/lib/lin-64/jep").absolutePath}:" +
"${file("../grobid-home/lib/lin-64").absolutePath}:"
} else {
throw new RuntimeException("Unsupported platform!")
}
if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs "--add-opens", "java.base/java.lang=ALL-UNNAMED"
}
workingDir = rootProject.rootDir
def javaLibraryPath = "${System.getProperty('java.library.path')}:" + libraries
// if (System.env.CONDA_PREFIX) {
// def condaEnv = "${System.env.CONDA_PREFIX}/lib"
// def pythonDirectory = file(condaEnv).listFiles({ it.toString().contains("/lib/python") } as FileFilter)?.first()
// def pythonVersion = (pythonDirectory =~ /python([0-9]\.[0-9]+)/)[0][1]
//
// javaLibraryPath = "${System.getProperty('java.library.path')}:" +
// libraries + ":" +
// "${System.env.CONDA_PREFIX}/lib:" +
// "${System.env.CONDA_PREFIX}/lib/python${pythonVersion}/site-packages/jep"
// }
systemProperty "java.library.path", javaLibraryPath
}
configurations {
all*.exclude group: 'org.slf4j', module: "slf4j-jdk14"
all*.exclude group: 'org.slf4j', module: "slf4j-log4j12"
all*.exclude group: 'log4j', module: "log4j"
}
tasks.distZip.enabled = true
tasks.distTar.enabled = false
tasks.shadowDistZip.enabled = false
tasks.shadowDistTar.enabled = false
distZip { duplicatesStrategy = DuplicatesStrategy.EXCLUDE }
distTar { duplicatesStrategy = DuplicatesStrategy.EXCLUDE }
dependencies {
implementation project(':grobid-core')
implementation project(':grobid-trainer')
implementation "io.dropwizard:dropwizard-core:1.3.23"
implementation "io.dropwizard:dropwizard-assets:1.3.23"
implementation "com.hubspot.dropwizard:dropwizard-guicier:1.3.5.0"
implementation "io.dropwizard:dropwizard-testing:1.3.23"
implementation "io.dropwizard:dropwizard-forms:1.3.23"
implementation "io.dropwizard:dropwizard-client:1.3.23"
implementation "io.dropwizard:dropwizard-auth:1.3.23"
implementation "org.apache.pdfbox:pdfbox:2.0.3"
implementation "javax.activation:activation:1.1.1"
implementation "io.prometheus:simpleclient_dropwizard:0.11.0"
implementation "io.prometheus:simpleclient_servlet:0.11.0"
testImplementation "io.dropwizard:dropwizard-testing:1.3.17"
}
shadowJar {
archiveClassifier = 'onejar'
mergeServiceFiles()
zip64 true
manifest {
attributes 'Main-Class': 'org.grobid.core.main.batch.GrobidMain'
}
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
artifacts {
archives shadowJar
}
distributions {
main {
contents {
from(new File(rootProject.rootDir, "grobid-service/README.md")) {
into "doc"
}
//from(new File(rootProject.rootDir, "../grobid-home/config/grobid.yaml")) {
// into "config"
//}
}
}
}
}
project(":grobid-trainer") {
apply plugin: 'com.github.johnrengelman.shadow'
apply plugin: 'jacoco'
dependencies {
implementation(group: 'xml-apis', name: 'xml-apis') {
// otherwise xml-apis 2.0.1 will come from XOM and will result in
// java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
//TODO: sort out this problem better
version {
strictly '1.4.01'
}
}
implementation project(':grobid-core')
implementation "com.rockymadden.stringmetric:stringmetric-core_2.10:0.27.3"
implementation "me.tongfei:progressbar:0.9.0"
//implementation 'org.slf4j:slf4j-log4j12:1.7.25'
implementation 'org.slf4j:slf4j-api:1.7.25'
//implementation 'org.slf4j:slf4j-jdk14:1.7.25'
}
configurations {
//all*.exclude group: 'org.slf4j', module: "slf4j-jdk14"
//all*.exclude group: 'org.slf4j', module: "slf4j-log4j12"
//all*.exclude group: 'log4j', module: "log4j"
}
jar {
from {
project.configurations.runtimeClasspath.collect {
it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
}
}
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
shadowJar {
archiveClassifier = 'onejar'
mergeServiceFiles()
zip64 true
manifest {
attributes 'Main-Class': 'org.grobid.trainer.TrainerRunner'
}
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
artifacts {
archives shadowJar
archives jar
}
task install {
dependsOn publishToMavenLocal
dependsOn 'shadowJar'
}
def trainerTasks = [
"train_name_header" : "org.grobid.trainer.NameHeaderTrainer",
"train_name_citation" : "org.grobid.trainer.NameCitationTrainer",
"train_affiliation_address" : "org.grobid.trainer.AffiliationAddressTrainer",
"train_header" : "org.grobid.trainer.HeaderTrainer",
"train_fulltext" : "org.grobid.trainer.FulltextTrainer",
"train_shorttext" : "org.grobid.trainer.ShorttextTrainer",
"train_figure" : "org.grobid.trainer.FigureTrainer",
"train_table" : "org.grobid.trainer.TableTrainer",
"train_citation" : "org.grobid.trainer.CitationTrainer",
"train_date" : "org.grobid.trainer.DateTrainer",
"train_segmentation" : "org.grobid.trainer.SegmentationTrainer",
"train_reference_segmentation": "org.grobid.trainer.ReferenceSegmenterTrainer",
"train_ebook_model" : "org.grobid.trainer.EbookTrainer",
"train_patent_citation" : "org.grobid.trainer.PatentParserTrainer",
"train_funding_acknowledgement" : "org.grobid.trainer.FundingAcknowledgementTrainer"
]
def libraries = ""
if (Os.isFamily(Os.FAMILY_MAC)) {
if (Os.OS_ARCH.equals("aarch64")) {
libraries = "${file("../grobid-home/lib/mac_arm-64").absolutePath}"
} else {
libraries = "${file("../grobid-home/lib/mac-64").absolutePath}"
}
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
libraries = "${file("../grobid-home/lib/lin-64/jep").absolutePath}:" +
"${file("../grobid-home/lib/lin-64").absolutePath}:"
} else {
throw new RuntimeException("Unsupported platform!")
}
trainerTasks.each { taskName, mainClassName ->
tasks.create(name: taskName, type: JavaExec, group: 'modeltraining') {
main = mainClassName
classpath = sourceSets.main.runtimeClasspath
if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0)
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}
}
// evaluation tasks
ext.getArg = { propName, defaultVal ->
return project.hasProperty(propName) ? project.getProperty(propName) : defaultVal;
}
// run like this:
// ./gradlew jatsEval -Pp2t=/path/to/goldenSet
// ./gradlew jatsEval -Pp2t=/path/to/goldenSet -Prun=1 -PfileRatio=0.1
// ./gradlew teiEval -Pp2t=/path/to/goldenSet
// ./gradlew PrepareDOIMatching -Pp2t=ABS_PATH_TO_PMC/PMC_sample_1943
// ./gradlew EvaluateDOIMatching -Pp2t=ABS_PATH_TO_PMC/PMC_sample_1943
task(jatsEval, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EndToEndEvaluation'
classpath = sourceSets.main.runtimeClasspath
args 'nlm', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0')
if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
} else {
jvmArgs '-Xmx3072m'
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}
task(teiEval, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EndToEndEvaluation'
classpath = sourceSets.main.runtimeClasspath
args 'tei', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0')
if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
} else {
jvmArgs '-Xmx3072m'
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}
task(PrepareDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching'
classpath = sourceSets.main.runtimeClasspath
args 'data', getArg('p2t', '.')
if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
} else {
jvmArgs '-Xmx3072m'
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}
task(EvaluateDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching'
classpath = sourceSets.main.runtimeClasspath
args 'eval', getArg('p2t', '.')
if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
} else {
jvmArgs '-Xmx3072m'
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}
}
/** JACOCO **/
tasks.register("codeCoverageReport", JacocoReport) {
// If a subproject applies the 'jacoco' plugin, add the result it to the report
subprojects { subproject ->
subproject.plugins.withType(JacocoPlugin).configureEach {
subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).configureEach { testTask ->
sourceSets subproject.sourceSets.main
executionData(testTask)
}
// To automatically run `test` every time `./gradlew codeCoverageReport` is called,
// you may want to set up a task dependency between them as shown below.
// Note that this requires the `test` tasks to be resolved eagerly (see `forEach`) which
// may have a negative effect on the configuration time of your build.
subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).forEach {
rootProject.tasks.codeCoverageReport.dependsOn(it)
}
}
}
// XML -> coveralls,
// HTML -> for manual check
reports {
xml.enabled true
html.enabled true
csv.enabled true
}
}
/** COVERALLS **/
coveralls {
sourceDirs = files(subprojects.sourceSets.main.allSource.srcDirs).files.absolutePath
}
tasks.coveralls {
dependsOn codeCoverageReport
}
wrapper {
gradleVersion "7.2"
}
build.dependsOn project.getSubprojects().collect({ it.getTasks().getByName("build") })