https://github.com/recski/HunTag
Tip revision: ac681ac97f8677f25c29060fc8a168d68aca22d9 authored by Gábor Recski on 18 January 2016, 08:52:12 UTC
Update README.md
Update README.md
Tip revision: ac681ac
huntag.py
from collections import defaultdict
from optparse import OptionParser
from feature import Feature
from trainer import Trainer
from tagger import Tagger
from bigram import Bigram
from tools import BookKeeper, writeSentence, sentenceIterator
from os.path import isdir, join
import sys
import os
def main_train(featureSet, options, input=sys.stdin):
optionsDict = vars(options)
if options.usedFeats:
optionsDict['usedFeats'] = file(options.usedFeats)
trainer = Trainer(featureSet, optionsDict)
if options.inFeatFile:
trainer.getEventsFromFile(options.inFeatFile)
else:
trainer.getEvents(input, options.outFeatFile)
trainer.cutoffFeats()
trainer.train()
trainer.save()
def main_bigramTrain(options, input):
bigramModel = Bigram(0.000000000000001)
for sen, _ in sentenceIterator(input):
tags = [tok[options.tagField] for tok in sen]
bigramModel.obsSequence(tags)
bigramModel.count()
bigramModel.writeToFile(options.bigramModelFile)
def main_tag(featureSet, options):
labelCounter, featCounter = BookKeeper(), BookKeeper()
labelCounter.readFromFile('{0}.labelNumbers'.format(options.modelName))
featCounter.readFromFile('{0}.featureNumbers'.format(options.modelName))
optionsDict = vars(options)
optionsDict['labelCounter'] = labelCounter
optionsDict['featCounter'] = featCounter
optionsDict['modelFile'] = '{0}.model'.format(options.modelName)
tagger = Tagger(featureSet, optionsDict)
if options.inFeatFile:
tagger_func = lambda: tagger.tag_features(options.inFeatFile)
writer_func = lambda s, c: writeSentence(s, comment=c)
elif options.input_dir:
assert isdir(options.input_dir), "--input-dir must be a directory"
out_dir = "{}_out".format(options.input_dir)
os.mkdir(out_dir)
tagger_func = lambda: tagger.tag_dir(options.input_dir)
writer_func = lambda s, c: writeSentence(
s, out=open(join(out_dir, '{}.tagged'.format(c)), 'a'))
else:
tagger_func = lambda: tagger.tag_corp(sys.stdin)
writer_func = lambda s, c: writeSentence(s, comment=c)
for sen, other in tagger_func():
writer_func(sen, other)
def getFeatureSet(cfgFile):
features = {}
optsByFeature = defaultdict(dict)
defaultRadius = -1
defaultCutoff = 1
for line in open(cfgFile):
if line == "\n" or line[0] == "#":
continue
feature = line.strip().split()
if feature[0] == 'let':
featName, key, value = feature[1:4]
optsByFeature[featName][key] = value
continue
if feature[0] == "!defaultRadius":
defaultRadius = int(feature[1])
continue
if feature[0] == "!defaultCutoff":
defaultCutoff = int(feature[1])
continue
type, name, actionName = feature[:3]
fields = [int(field) for field in feature[3].split(',')]
if len(feature) > 4:
radius = int(feature[4])
else:
radius = defaultRadius
cutoff = defaultCutoff
options = optsByFeature[name]
feat = Feature(type, name, actionName, fields, radius, cutoff, options)
features[name] = feat
return features
def getParser():
parser = OptionParser()
parser.add_option('-c', '--config-file', dest='cfgFile',
help='read feature configuration from FILE',
metavar='FILE')
parser.add_option('-m', '--model', dest='modelName',
help='name of model to be read/written',
metavar='NAME')
parser.add_option('-b', '--bigram-model', dest='bigramModelFile',
help='name of bigram model file to be read/written',
metavar='FILE')
parser.add_option('-l', '--language-model-weight', dest='lmw',
type='float', default=1,
help='set relative weight of the language model to L',
metavar='L')
parser.add_option('-o', '--cutoff', dest='cutoff', type='int', default=1,
help='set global cutoff to C', metavar='C')
parser.add_option('-p', '--parameters', dest='trainParams',
help='pass PARAMS to trainer', metavar='PARAMS')
parser.add_option('-u', '--used-feats', dest='usedFeats',
help='limit used features to those in FILE',
metavar='FILE')
parser.add_option('-d', '--input-dir', dest='input_dir',
help='process all files in DIR (instead of stdin)',
metavar='DIR')
parser.add_option('-i', '--input-feature-file', dest='inFeatFile',
help='use training events in FILE', metavar='FILE')
parser.add_option('-f', '--feature-file', dest='outFeatFile',
help='write training events to FILE', metavar='FILE')
parser.add_option('-t', '--tag-field', dest='tagField', type='int',
help="""specify FIELD containing the tags to build bigram
model from""", metavar='FIELD')
return parser
def main():
parser = getParser()
options, args = parser.parse_args()
task = args[0]
if task == 'bigram-train':
main_bigramTrain(options, sys.stdin)
elif task == 'train':
featureSet = getFeatureSet(options.cfgFile)
main_train(featureSet, options)
elif task == 'tag':
if options.inFeatFile:
assert not options.input_dir, 'at most one of input-feature-file\
and input-dir can be specified'
featureSet = None
else:
featureSet = getFeatureSet(options.cfgFile)
main_tag(featureSet, options)
else:
sys.stderr.write("""invalid task: %s\nRun huntag.py --help
for more information\n""" % task)
return
if __name__ == '__main__':
main()