https://github.com/annotation/text-fabric
Tip revision: e25528157d329db0c86204e9dfa403ffdc378e19 authored by Dirk Roorda on 01 December 2016, 21:46:41 UTC
New minor release 0.0.9
New minor release 0.0.9
Tip revision: e255281
data.py
import os,sys,array,pickle,json,gzip,collections,time
from datetime import datetime
from .timestamp import Timestamp
from .helpers import *
ERROR_CUTOFF = 20
GZIP_LEVEL = 2
PICKLE_PROTOCOL = 4
GRID = (
'otype',
'oslots',
'otext',
)
SECTIONS = (
'book',
'chapter',
'verse',
) # the only place in the code where these words occur (hopefully)
class Data(object):
def __init__(self, path, tm, edgeValues=False, data=None, isEdge=None, isConfig=None, metaData={}, method=None, dependencies=None):
(dirName, baseName) = os.path.split(path)
(fileName, extension) = os.path.splitext(baseName)
self.path = path
self.tm = tm
self.dirName = dirName
self.fileName = fileName
self.extension = extension
self.binDir = '{}/.tf'.format(dirName)
self.binPath = '{}/{}.tfx'.format(self.binDir, self.fileName)
self.edgeValues = edgeValues
self.isEdge = isEdge
self.isConfig = isConfig
self.metaData = metaData
self.method = method
self.dependencies = dependencies
self.data = data
self.dataLoaded = False
self.dataError = False
def load(self, silent=False):
self.tm.indent(level=1, reset=True)
origTime = self._getModified()
binTime = self._getModified(bin=True)
sourceRep = ', '.join(dep.fileName for dep in self.dependencies) if self.method else self.dirName
msgFormat = '{:<1} {:<20} from {}'
actionRep = ''
good = True
if self.dataError:
actionRep = 'E' # there has been an error in an earlier computation/compiling/loading of this feature
good = False
elif self.dataLoaded and (
self.isConfig or
(not origTime or self.dataLoaded >= origTime) and (not binTime or self.dataLoaded >= binTime)
):
actionRep = '=' # loaded and up to date
elif not origTime and not binTime:
actionRep = 'X' # no source and no binary present
good = False
else:
if not origTime:
actionRep = 'b'
good = self._readDataBin(silent=silent)
elif not binTime or origTime > binTime:
actionRep = 'C' if self.method else 'T'
good = self._compute(silent=silent) if self.method else self._readTf()
if good: self._writeDataBin(silent=silent)
else:
actionRep = 'B'
good = True if self.method else self._readTf(metaOnly=True)
if good:
if self.isConfig:
actionRep = 'M'
else:
good = self._readDataBin(silent=silent)
if good:
self.dataLoaded = time.time()
if actionRep != '=' and not silent:
self.tm.info(msgFormat.format(actionRep, self.fileName, sourceRep))
else:
self.dataError = True
self.tm.error(msgFormat.format(actionRep, self.fileName, sourceRep))
return good
def unload(self):
self.data = None
self.dataLoaded = False
def save(self, overwrite=False, nodeRanges=False):
return self._writeTf(overwrite=overwrite, nodeRanges=nodeRanges)
def _readTf(self, metaOnly=False):
path = self.path
if not os.path.exists(path):
self.tm.error('TF reading: feature file "{}" does not exist'.format(path))
return False
fh = open(path)
i = 0
self.metaData = {}
self.isConfig = False
for line in fh:
i += 1
if i == 1:
text = line.rstrip()
if text == '@edge': self.isEdge = True
elif text == '@node': self.isEdge = False
elif text == '@config': self.isConfig = True
else:
self.tm.error('Line {}: missing @node/@edge'.format(i))
fh.close()
return False
continue
text = line.rstrip('\n')
if len(text) and text[0] == '@':
fields = text[1:].split('=', 1)
self.metaData[fields[0]] = fields[1] if len(fields) == 2 else None
continue
else:
if text != '':
self.tm.error('Line {}: missing blank line after metadata'.format(i))
fh.close()
return False
else:
break
good = True
if not metaOnly and not self.isConfig:
good = self._readDataTf(fh, i)
fh.close()
return good
def _readDataTf(self, fh, firstI):
errors=collections.defaultdict(list)
first = True
i = firstI
implicit_node = 0
data = {}
isEdge = self.isEdge
edgeValues = self.edgeValues
normFields = 3 if isEdge and edgeValues else 2
for line in fh:
i += 1
fields = line.rstrip('\n').split('\t')
lfields = len(fields)
if lfields > normFields:
errors['wrongFields'].append(i)
continue
if lfields == normFields:
nodes = setFromSpec(fields[0])
if isEdge:
if fields[1] == '':
errors['emptyNode2Spec'].append(i)
continue
nodes2 = setFromSpec(fields[1])
if not isEdge or edgeValues:
valTf = fields[-1]
else:
if isEdge:
if edgeValues:
valTf = ''
if lfields == normFields - 1:
nodes = setFromSpec(fields[0])
if fields[1] == '':
errors['emptyNode2Spec'].append(i)
continue
nodes2 = setFromSpec(fields[1])
elif lfields == normFields - 2:
nodes = {implicit_node}
if fields[0] == '':
errors['emptyNode2Spec'].append(i)
continue
nodes2 = setFromSpec(fields[0])
else:
nodes = {implicit_node}
errors['emptyNode2Spec'].append(i)
continue
else:
if lfields == normFields - 1:
nodes = {implicit_node}
if fields[0] == '':
errors['emptyNode2Spec'].append(i)
continue
nodes2 = setFromSpec(fields[0])
else:
nodes = {implicit_node}
errors['emptyNode2Spec'].append(i)
continue
else:
nodes = {implicit_node}
if lfields == 1:
valTf = fields[0]
else:
valTf = ''
implicit_node = max(nodes) + 1
if not isEdge or edgeValues:
value = '' if valTf == '' else valueFromTf(valTf)
if isEdge:
for n in nodes:
for m in nodes2:
if not edgeValues:
data.setdefault(n, set()).add(m)
else:
data.setdefault(n, {})[m] = value
else:
for n in nodes: data[n] = value
for kind in errors:
lnk = len(errors[kind])
self.tm.error('{} in lines {}'.format(kind, ','.join(str(ln) for ln in errors[kind][0:ERROR_CUTOFF])))
if lnk > ERROR_CUTOFF:
self.tm.error('\t and {} more cases'.format(lnk - ERROR_CUTOFF), tm=False)
self.data = data
if not errors:
if self.fileName == GRID[0]:
slotType = data[0]
otype = []
maxSlot = -1
for n in sorted(data):
if data[n] == slotType:
maxSlot += 1
continue
otype.append(data[n])
otype.append(slotType)
otype.append(maxSlot)
self.data = tuple(otype)
elif self.fileName == GRID[1]:
slotsList = sorted(data)
maxSlot = min(data.keys()) - 1
slots = []
for n in slotsList:
slots.append(tuple(sorted(data[n])))
slots.append(maxSlot)
self.data = tuple(slots)
return not errors
def _compute(self, silent=False):
good = True
for feature in self.dependencies:
if not feature.load():
good = False
if not good: return False
if silent:
def info(msg, tm=True): pass
else:
def info(msg, tm=True): self.tm.info(cmpFormat.format(msg), tm=tm)
cmpFormat = 'c {:<20} {{}}'.format(self.fileName)
self.tm.indent(level=2, reset=True)
def error(msg, tm=True): self.tm.error(cmpFormat.format(msg), tm=tm)
self.data = self.method(info, error, *[
dep.metaData if dep.fileName == GRID[2] else dep.data for dep in self.dependencies
])
return self.data != None
def _writeTf(self, dirName=None, fileName=None, overwrite=True, extension=None, metaOnly=False, nodeRanges=False):
self.tm.indent(level=1, reset=True)
data = self.data
metaOnly = metaOnly or self.isConfig
dirName = dirName or self.dirName
fileName = fileName or self.fileName
extension = extension or self.extension
fpath = '{}/{}{}'.format(dirName, fileName, extension)
if fpath == self.path:
if os.path.exists(fpath):
if not overwrite:
self.tm.error('Feature file "{}" already exists, feature will not be written'.format(fpath))
return False
try:
fh = open(fpath, 'w')
except:
self.tm.error('Cannot write to feature file "{}"'.format(fpath))
return False
fh.write('@{}\n'.format('config' if self.isConfig else 'edge' if self.isEdge else 'node'))
for meta in sorted(self.metaData):
fh.write('@{}={}\n'.format(meta, self.metaData[meta]))
fh.write('@writtenBy=Text-Fabric\n')
fh.write('@dateWritten={}\n'.format(datetime.utcnow().replace(microsecond=0).isoformat()+'Z'))
fh.write('\n')
good = True
if not metaOnly:
good = self._writeDataTf(fh, nodeRanges=nodeRanges)
fh.close()
msgFormat = '{:<1} {:<20} to {}'
if good:
self.tm.info(msgFormat.format('M' if metaOnly else 'T', fileName, dirName))
else:
self.tm.error(msgFormat.format('M' if metaOnly else 'T', fileName, dirName))
return good
def _writeDataTf(self, fh, nodeRanges=False):
data = self.data
edgeValues = self.edgeValues
if self.isEdge:
implicitNode = 0
for n in sorted(data):
thisData = data[n]
sets = {}
if edgeValues:
for m in thisData:
sets.setdefault(thisData[m], set()).add(m)
for (value, mset) in sorted(sets.items()):
nodeSpec2 = specFromRanges(rangesFromSet(mset))
nodeSpec = '' if n == implicitNode else n
implicitNode = n + 1
fh.write('{}{}{}\t{}\n'.format(
nodeSpec, '\t' if nodeSpec else '', nodeSpec2, tfFromValue(value),
))
else:
nodeSpec2 = specFromRanges(rangesFromSet(thisData))
nodeSpec = '' if n == implicitNode else n
implicitNode = n + 1
fh.write('{}{}{}\n'.format(nodeSpec, '\t' if nodeSpec else '', nodeSpec2))
else:
sets = {}
if nodeRanges:
for n in sorted(data):
sets.setdefault(data[n], []).append(n)
implicitNode = 0
for (value, nset) in sorted(sets.items(), key=lambda x: (x[1][0], x[1][-1])):
if len(nset) == 1 and nset[0] == implicitNode:
nodeSpec = ''
else:
nodeSpec = specFromRanges(rangesFromSet(nset))
implicitNode = nset[-1]
fh.write('{}{}{}\n'.format(
nodeSpec, '\t' if nodeSpec else '', tfFromValue(value),
))
else:
implicitNode = 0
for n in sorted(data):
nodeSpec = '' if n == implicitNode else n
implicitNode = n + 1
fh.write('{}{}{}\n'.format(
nodeSpec, '\t' if nodeSpec else '', tfFromValue(data[n]),
))
return True
def _readDataBin(self, silent=False):
if not os.path.exists(self.binPath):
self.tm.error('TF reading: feature file "{}" does not exist'.format(self.binPath))
return False
with gzip.open(self.binPath, "rb") as f: self.data = pickle.load(f)
return True
def _writeDataBin(self, silent=False):
good = True
if not os.path.exists(self.binDir):
try:
os.makedirs(self.binDir, exist_ok=True)
except:
self.tm.error('Cannot create directory "{}"'.format(self.binDir))
good = False
if not good: return False
try:
with gzip.open(self.binPath, "wb", compresslevel=GZIP_LEVEL) as f:
pickle.dump(self.data, f, protocol=PICKLE_PROTOCOL)
except:
self.tm.error('Cannot write to file "{}"'.format(self.binPath))
good = False
return True
def _getModified(self, bin=False):
if bin:
return os.path.getmtime(self.binPath) if os.path.exists(self.binPath) else None
else:
if self.method:
depsInfo = [dep._getModified() for dep in self.dependencies]
depsModifieds = [d for d in depsInfo if d != None]
depsModified = None if len(depsModifieds) == 0 else max(depsModifieds)
if depsModified != None: return depsModified
elif os.path.exists(self.binPath): return os.path.getmtime(self.binPath)
else: return None
else:
if os.path.exists(self.path): return os.path.getmtime(self.path)
elif os.path.exists(self.binPath): return os.path.getmtime(self.binPath)
else: return None