https://github.com/flora-phenotype-ontology/flopoontology
Revision 5904c1375215ff22c29da28b2ce9257eb6a24fbc authored by leechuck on 06 April 2014, 13:43:25 UTC, committed by leechuck on 06 April 2014, 13:43:25 UTC
1 parent c686b9c
Tip revision: 5904c1375215ff22c29da28b2ce9257eb6a24fbc authored by leechuck on 06 April 2014, 13:43:25 UTC
added parameters for ontology generation, generate annotation file
added parameters for ontology generation, generate annotation file
Tip revision: 5904c13
GenerateLuceneIndex.groovy
import org.apache.lucene.analysis.*
import org.apache.lucene.analysis.standard.*
import org.apache.lucene.document.*
import org.apache.lucene.index.*
import org.apache.lucene.store.*
import org.apache.lucene.util.*
import org.apache.lucene.search.*
import org.apache.lucene.queryparser.classic.*
import com.aliasi.medline.*
import org.apache.lucene.analysis.fr.*
import opennlp.tools.sentdetect.*
import opennlp.tools.dictionary.*
import opennlp.tools.tokenize.*
import opennlp.tools.util.*
import opennlp.tools.chunker.*
import opennlp.tools.postag.*
import opennlp.tools.namefind.*
import java.util.concurrent.*
String indexPath = "lucene-index"
String ontologyIndexPath = "lucene-index-ontology"
Directory dir = FSDirectory.open(new File(indexPath)) // RAMDirectory()
Directory ontologyIndexDir = FSDirectory.open(new File(ontologyIndexPath)) // RAMDirectory()
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47)
Analyzer frenchAnalyzer = new FrenchAnalyzer(Version.LUCENE_47)
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, frenchAnalyzer)
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
iwc.setRAMBufferSizeMB(32768.0)
IndexWriter writer = new IndexWriter(dir, iwc)
IndexWriterConfig iwcEnglish = new IndexWriterConfig(Version.LUCENE_47, analyzer)
iwcEnglish.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
iwcEnglish.setRAMBufferSizeMB(32768.0)
IndexWriter englishWriter = new IndexWriter(ontologyIndexDir, iwcEnglish)
def author = null
def year = null
List<String> rankOrder = ["order", "family", "subfamily", "tribe", "subtribe", "genus", "subgenus", "species", "subspecies", "variety"]
Map<String, Set<String>> previousCharacters = [:] // this maps taxonomic rank name to EQs
Map<String, String> previousNames = [:] // this keeps the previously encountered taxon names; ordername -> value
XmlSlurper slurper = new XmlSlurper(false, false)
slurper.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
new File("flora-gabon").eachFile { florafile ->
def flora = slurper.parse(florafile)
flora.treatment.each { treatment ->
treatment.taxon.each { taxon ->
// first try to get the taxon name
def taxonString = ""
def name = null
taxon.nomenclature.homotypes.nom.each { nom ->
if (nom.@class.text() == "accepted") {
name = nom
def lastOrderRank = ""
nom.name.each { nomname -> /* first we determine the level of the tree in which we currently are; we will reuse information from the higher orders
// mentioned before */
def cname = nomname.@class.text()
if (cname == "author") { author = nomname.text() }
if (cname == "year") { year = nomname.text() }
if (cname in rankOrder) {
lastOrderRank = cname
def cvalue = nomname.text()
// we delete everything from the end of the list to the current rank from previousCharacters map
/* rankOrder[-1..rankOrder.indexOf(cname)].each {
previousCharacters[it] = null
previousNames[it] = null
}*/
previousNames[cname] = cvalue
}
}
rankOrder[0..rankOrder.indexOf(lastOrderRank)].each {
if (previousNames[it] != null) {
taxonString += "$it: "+previousNames[it]+"; "
}
}
taxonString += "$author; $year"
}
}
// now index the descriptions: first we assemble the description text, then we sentencize, then we create a new document for each sentence
String description = ""
taxon.feature.each { feature ->
if (feature.@class.text() == "description") {
feature.char.each { character ->
def cclass = character.@class.text().toLowerCase()
def ctext = character.text()
description += ctext + " "
//
}
}
}
/* now split the description in sentences */
SentenceModel sentenceModel = new SentenceModel(new FileInputStream("models/fr-sent.bin"))
SentenceDetectorME sentenceDetector = new SentenceDetectorME(sentenceModel)
def sentences = sentenceDetector.sentDetect(description)
sentences.each { sentence ->
Document doc = new Document()
doc.add(new Field("taxon", taxonString, Field.Store.YES, Field.Index.NO))
doc.add(new Field("description", sentence, TextField.TYPE_STORED))
writer.addDocument(doc)
}
}
}
}
/* Final part: we also add all the ontology terms to the index so that we can easier search for them */
def ontologyDirectory = "ont/"
new File("ont").eachFile { ontfile ->
def id = ""
ontfile.eachLine { line ->
if (line.startsWith("id:")) {
id = line.substring(3).trim()
}
if (line.startsWith("name:")) {
def name = line.substring(5).trim()
Document doc = new Document()
doc.add(new Field("id", id, Field.Store.YES, Field.Index.NO))
doc.add(new Field("label", name, TextField.TYPE_STORED))
englishWriter.addDocument(doc)
}
if (line.startsWith("synonym:")) {
def syn = line.substring(line.indexOf("\"")+1, line.lastIndexOf("\"")).trim()
Document doc = new Document()
doc.add(new Field("id", id, Field.Store.YES, Field.Index.NO))
doc.add(new Field("label", syn, TextField.TYPE_STORED))
englishWriter.addDocument(doc)
}
}
}
writer.close()
englishWriter.close()
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...