https://github.com/annotation/text-fabric
Tip revision: 70a184b40d7c2bab9ab5736214dbb314df1fc073 authored by Dirk Roorda on 14 November 2018, 15:48:32 UTC
major update - sharing data
major update - sharing data
Tip revision: 70a184b
apphelpers.py
from functools import reduce
from IPython.display import display, Markdown, HTML
LIMIT_SHOW = 100
LIMIT_TABLE = 2000
RESULT = 'result'
FONT_BASE = 'https://github.com/Dans-labs/text-fabric/blob/master/tf/server/static/fonts'
CSS_FONT = '''
<link rel="stylesheet" href="/server/static/fonts.css"/>
'''
CSS_FONT_API = f'''
<style>
@font-face {{{{
font-family: "{{fontName}}";
src: url('{FONT_BASE}/{{font}}?raw=true');
src: url('{FONT_BASE}/{{fontw}}?raw=true') format('woff');
}}}}
</style>
'''
def search(app, query, silent=False, sets=None, shallow=False):
api = app.api
info = api.info
S = api.S
results = S.search(query, sets=sets, shallow=shallow)
if not shallow:
results = sorted(results)
nResults = len(results)
plural = '' if nResults == 1 else 's'
if not silent:
info(f'{nResults} result{plural}')
return results
def runSearch(api, query, cache):
S = api.S
plainSearch = S.search
cacheKey = (query, False)
if cacheKey in cache:
return cache[cacheKey]
(queryResults, messages) = plainSearch(query, msgCache=True)
features = {}
exe = getattr(S, 'exe', None)
if exe:
qnodes = getattr(exe, 'qnodes', [])
features = tuple((i, tuple(sorted(q[1].keys()))) for (i, q) in enumerate(qnodes))
queryResults = tuple(sorted(queryResults))
cache[cacheKey] = (queryResults, messages, features)
return (queryResults, messages, features)
def runSearchCondensed(api, query, cache, condenseType):
cacheKey = (query, True, condenseType)
if cacheKey in cache:
return cache[cacheKey]
(queryResults, messages, features) = runSearch(api, query, cache)
queryResults = _condense(api, queryResults, condenseType, multiple=True)
cache[cacheKey] = (queryResults, messages, features)
return (queryResults, messages, features)
def composeP(
app,
items,
textFormat,
opened,
sec2,
withNodes=False,
**options,
):
passageHtml = []
for item in items:
passageHtml.append(
_plainTextS2(
app,
item,
opened,
sec2,
fmt=textFormat,
withNodes=withNodes,
**options,
)
)
return '\n'.join(passageHtml)
def compose(
app,
tuples,
features,
start,
position,
opened,
condensed,
condenseType,
textFormat,
withNodes=False,
linked=1,
**options,
):
api = app.api
F = api.F
if condenseType is None:
condenseType = app.condenseType
item = condenseType if condensed else RESULT
features = set(reduce(
set.union,
(x[1] for x in features),
set(),
))
if features:
app.prettySetup(features)
tuplesHtml = []
doHeader = False
for (i, tup) in tuples:
if i is None:
if tup == 'results':
doHeader = True
else:
tuplesHtml.append(f'''
<div class="dtheadrow">
<span>n</span><span>{tup}</span>
</div>
''')
continue
if doHeader:
doHeader = False
tuplesHtml.append(f'''
<div class="dtheadrow">
<span>n</span><span>{"</span><span>".join(F.otype.v(n) for n in tup)}</span>
</div>
''')
tuplesHtml.append(
plainTuple(
app,
tup,
i,
isCondensed=condensed,
condenseType=condenseType,
item=item,
linked=linked,
fmt=textFormat,
withNodes=withNodes,
position=position,
opened=i in opened,
asString=True,
**options,
)
)
if features:
app.prettySetup()
return '\n'.join(tuplesHtml)
def table(
app,
tuples,
condensed=False,
condenseType=None,
start=None,
end=None,
linked=1,
fmt=None,
withNodes=False,
asString=False,
**options,
):
api = app.api
F = api.F
if condenseType is None:
condenseType = app.condenseType
item = condenseType if condensed else RESULT
if condensed:
tuples = _condense(api, tuples, condenseType, multiple=True)
markdown = []
one = True
for (i, tup) in _tupleEnum(tuples, start, end, LIMIT_TABLE, item):
if one:
nColumns = len(tup)
markdown.append('n | ' + (' | '.join(F.otype.v(n) for n in tup)))
markdown.append(' | '.join('---' for n in range(nColumns + 1)))
one = False
markdown.append(plainTuple(
app,
tup,
i,
isCondensed=condensed,
condenseType=condenseType,
item=item,
linked=linked,
fmt=fmt,
withNodes=withNodes,
position=None,
opened=False,
asString=True,
**options,
))
markdown = '\n'.join(markdown)
if asString:
return markdown
dm(markdown)
def plainTuple(
app,
tup,
seqNumber,
isCondensed=False,
condenseType=None,
item=RESULT,
linked=1,
fmt=None,
withNodes=False,
position=None,
opened=False,
asString=False,
**options,
):
asApp = app.asApp
api = app.api
F = api.F
T = api.T
if asApp:
prettyRep = prettyTuple(
app,
tup,
seqNumber,
isCondensed=isCondensed,
condenseType=condenseType,
fmt=fmt,
withNodes=withNodes,
**options,
) if opened else ''
current = ' focus' if seqNumber == position else ''
attOpen = ' open ' if opened else ''
refColumn = 1 if isCondensed else linked
refNode = tup[refColumn - 1] if refColumn <= len(tup) else None
refRef = '' if refNode is None else app.sectionLink(refNode)
tupSeq = ','.join(str(n) for n in tup)
sParts = T.sectionFromNode(refNode)
passageAtt = ' '.join(
f'sec{i}="{sParts[i] if i < len(sParts) else ""}"'
for i in range(3)
)
plainRep = ''.join(
f'''<span>{mdEsc(app.plain(
n,
linked=i == linked - 1,
fmt=fmt,
withNodes=withNodes,
**options,
))
}
</span>
'''
for (i, n) in enumerate(tup)
)
html = (
f'''
<details
class="pretty dtrow{current}"
seq="{seqNumber}"
{attOpen}
>
<summary>
<a href="#" class="pq fa fa-solar-panel fa-xs" title="show in context" {passageAtt}></a>
<a href="#" class="sq" tup="{tupSeq}">{seqNumber}</a>
{refRef}
{plainRep}
</summary>
<div class="pretty">
{prettyRep}
</div>
</details>
'''
)
return html
markdown = [str(seqNumber)]
for (i, n) in enumerate(tup):
markdown.append(
mdEsc(app.plain(
n,
linked=i == linked - 1,
fmt=fmt,
withNodes=withNodes,
asString=True,
**options,
))
)
markdown = '|'.join(markdown)
if asString:
return markdown
head = ['n | ' + (' | '.join(F.otype.v(n) for n in tup))]
head.append(' | '.join('---' for n in range(len(tup) + 1)))
head.append(markdown)
dm('\n'.join(head))
def _plainTextS2(
app,
sNode,
opened,
sec2,
fmt=None,
withNodes=False,
**options,
):
api = app.api
T = api.T
seqNumber = T.sectionFromNode(sNode)[2]
itemType = T.sectionTypes[2]
isOpened = str(seqNumber) in opened
tClass = '' if fmt is None else app.formatClass[fmt]
prettyRep = prettyTuple(
app,
(sNode,),
seqNumber,
isCondensed=False,
condenseType=itemType,
fmt=fmt,
withNodes=withNodes,
**options,
) if isOpened else ''
current = ' focus' if str(seqNumber) == str(sec2) else ''
attOpen = ' open ' if isOpened else ''
textRep = T.text(sNode, fmt=fmt, descend=True)
html = (
f'''
<details
class="pretty {current}"
seq="{seqNumber}"
{attOpen}
>
<summary class="{tClass}">
{app.sectionLink(sNode, text=seqNumber)}
{textRep}
</summary>
<div class="pretty">
{prettyRep}
</div>
</details>
'''
)
return html
def show(
app,
tuples,
condensed=True,
condenseType=None,
start=None,
end=None,
fmt=None,
withNodes=False,
suppress=set(),
colorMap=None,
highlights={},
**options,
):
api = app.api
F = api.F
if condenseType is None:
condenseType = app.condenseType
item = condenseType if condensed else RESULT
if condensed:
rawHighlights = _getHighlights(
api, tuples, highlights, colorMap, condenseType, multiple=True
)
highlights = {}
colorMap = None
tuples = _condense(api, tuples, condenseType, multiple=True)
else:
rawHighlights = None
for (i, tup) in _tupleEnum(tuples, start, end, LIMIT_SHOW, item):
item = F.otype.v(tup[0]) if condenseType else RESULT
prettyTuple(
app,
tup,
i,
isCondensed=condensed,
condenseType=condenseType,
item=item,
fmt=fmt,
withNodes=withNodes,
suppress=suppress,
colorMap=colorMap,
highlights=highlights,
rawHighlights=rawHighlights,
**options,
)
def prettyTuple(
app,
tup,
seqNumber,
item='Result',
condenseType=None,
isCondensed=False,
fmt=None,
withNodes=False,
suppress=set(),
colorMap=None,
highlights={},
rawHighlights=None,
**options,
):
asApp = app.asApp
if len(tup) == 0:
if asApp:
return ''
else:
return
api = app.api
if condenseType is None:
condenseType = app.condenseType
containers = {tup[0]} if isCondensed else _condenseSet(api, tup, condenseType)
newHighlights = (
_getHighlights(api, tup, highlights, colorMap, condenseType)
if rawHighlights is None else
rawHighlights
)
if not asApp:
dm(f'''
**{item}** *{seqNumber}*
''')
if asApp:
html = []
for t in containers:
h = app.pretty(
t,
condenseType=condenseType,
fmt=fmt,
withNodes=withNodes,
suppress=suppress,
highlights=newHighlights,
**options,
)
if asApp:
html.append(h)
if asApp:
return '\n'.join(html)
def pretty(
app,
n,
condenseType=None,
fmt=None,
withNodes=False,
suppress=set(),
highlights={},
**options,
):
asApp = app.asApp
api = app.api
F = api.F
L = api.L
otypeRank = api.otypeRank
containerN = None
if condenseType:
nType = F.otype.v(n)
if nType == condenseType:
containerN = n
elif otypeRank[nType] < otypeRank[condenseType]:
ups = L.u(n, otype=condenseType)
if ups:
containerN = ups[0]
(firstSlot, lastSlot) = (
getBoundary(api, n)
if condenseType is None else
(None, None)
if containerN is None else
getBoundary(api, containerN)
)
html = []
if type(highlights) is set:
highlights = {m: '' for m in highlights}
app._pretty(
n,
True,
html,
firstSlot,
lastSlot,
condenseType=condenseType,
fmt=fmt,
withNodes=withNodes,
suppress=suppress,
highlights=highlights,
**options,
)
htmlStr = '\n'.join(html)
if asApp:
return htmlStr
dh(htmlStr)
def prettyPre(
app,
n,
firstSlot,
lastSlot,
withNodes,
highlights,
):
api = app.api
F = api.F
slotType = F.otype.slotType
nType = F.otype.v(n)
boundaryClass = ''
myStart = None
myEnd = None
(myStart, myEnd) = getBoundary(api, n)
if firstSlot is not None:
if myEnd < firstSlot:
return False
if myStart < firstSlot:
boundaryClass += ' R'
if lastSlot is not None:
if myStart > lastSlot:
return False
if myEnd > lastSlot:
boundaryClass += ' L'
hl = highlights.get(n, None)
hlClass = ''
hlStyle = ''
if hl == '':
hlClass = ' hl'
else:
hlStyle = f' style="background-color: {hl};"'
nodePart = (f'<a href="#" class="nd">{n}</a>' if withNodes else '')
className = app.classNames.get(nType, None)
return (
slotType, nType,
className, boundaryClass, hlClass, hlStyle,
nodePart,
myStart, myEnd,
)
def prettySetup(app, features=None, noneValues=None):
if features is None:
app.prettyFeatures = ()
else:
featuresRequested = tuple(
features.strip().split()
) if type(features) is str else tuple(sorted(features))
tobeLoaded = set(featuresRequested) - app.prettyFeaturesLoaded
if tobeLoaded:
app.api.TF.load(tobeLoaded, add=True, silent=True)
app.prettyFeaturesLoaded |= tobeLoaded
app.prettyFeatures = featuresRequested
if noneValues is None:
app.noneValues = app.noneValues
else:
app.noneValues = noneValues
def getBoundary(api, n):
F = api.F
slotType = F.otype.slotType
if F.otype.v(n) == slotType:
return (n, n)
L = api.L
slots = L.d(n, otype=slotType)
return (slots[0], slots[-1])
def getFeatures(
app, n, suppress, features,
withName=set(),
givenValue={},
plain=False,
):
api = app.api
Fs = api.Fs
featurePartB = '<div class="features">'
featurePartE = '</div>'
givenFeatureSet = set(features)
extraFeatures = (
tuple(f for f in app.prettyFeatures if f not in givenFeatureSet)
)
extraSet = set(extraFeatures)
featureList = tuple(features) + extraFeatures
nFeatures = len(features)
withName |= set(app.prettyFeatures)
if not plain:
featurePart = featurePartB
hasB = True
else:
featurePart = ''
hasB = False
for (i, name) in enumerate(featureList):
if name not in suppress:
value = (
givenValue[name]
if name in givenValue else
Fs(name).v(n)
)
if value not in app.noneValues:
if name not in givenValue:
value = htmlEsc(value)
nameRep = f'<span class="f">{name}=</span>' if name in withName else ''
xClass = ' xft' if name in extraSet else ''
featureRep = f' <span class="{name}{xClass}">{nameRep}{value}</span>'
if i >= nFeatures:
if not hasB:
featurePart += featurePartB
hasB = True
featurePart += featureRep
if hasB:
featurePart += featurePartE
return featurePart
def getContext(api, nodes):
F = api.F
Fs = api.Fs
Fall = api.Fall
T = api.T
# L = api.L
# slotType = F.otype.slotType
sectionTypes = set(T.sectionTypes)
rows = []
feats = tuple(sorted(Fall()))
rows.append(('node',) + tuple(T.sectionTypes) + feats + ('text',))
for n in sorted(nodes):
nType = F.otype.v(n)
sParts = T.sectionFromNode(n)
nParts = len(sParts)
section = sParts + ((None,) * (3 - nParts))
if nType in sectionTypes:
text = ''
else:
# sns = [n] if nType == slotType else L.d(n, otype=slotType)
# text = T.text(sns)
text = T.text(n)
rows.append((n,) + section + tuple(Fs(f).v(n) for f in feats) + (text,))
return tuple(rows)
def getResultsX(api, results, features, noDescendTypes, fmt=None):
F = api.F
Fs = api.Fs
T = api.T
# L = api.L
# slotType = F.otype.slotType
sectionTypes = set(T.sectionTypes)
if len(results) == 0:
return ()
firstResult = results[0]
nTuple = len(firstResult)
refColumns = [
i for (i, n) in enumerate(firstResult) if F.otype.v(n) not in sectionTypes
]
refColumn = refColumns[0] if refColumns else nTuple - 1
header = ['R', 'S1', 'S2', 'S3']
emptyA = []
featureDict = dict(features)
for j in range(nTuple):
i = j + 1
n = firstResult[j]
nType = F.otype.v(n)
header.extend([f'NODE{i}', f'TYPE{i}'])
if nType not in sectionTypes:
header.append(f'TEXT{i}')
header.extend(f'{feature}{i}' for feature in featureDict.get(j, emptyA))
rows = [tuple(header)]
for (rm, r) in enumerate(results):
rn = rm + 1
row = [rn]
refN = r[refColumn]
sParts = T.sectionFromNode(refN)
nParts = len(sParts)
section = sParts + ((None,) * (3 - nParts))
row.extend(section)
for j in range(nTuple):
n = r[j]
nType = F.otype.v(n)
# sns = [n] if nType == slotType else L.d(n, otype=slotType)
row.extend((n, nType))
if nType not in sectionTypes:
text = T.text(n, fmt=fmt, descend=nType not in noDescendTypes)
# text = T.text(sns)
row.append(text)
row.extend(Fs(feature).v(n) for feature in featureDict.get(j, emptyA))
rows.append(tuple(row))
return tuple(rows)
def header(app):
return (
f'''
<div class="hdlinks">
{app.dataLink}
{app.charLink}
{app.featureLink}
{app.tfsLink}
{app.tutLink}
</div>
''',
f'<img class="hdlogo" src="/data/static/logo.png"/>',
f'<img class="hdlogo" src="/server/static/icon.png"/>',
)
def nodeFromDefaultSection(app, sectionStr):
api = app.api
T = api.T
sep1 = app.sectionSep1
sep2 = app.sectionSep2
msg = f'Not a valid passage: "{sectionStr}"'
msgi = '{} "{}" is not a number'
section = sectionStr.split(sep1)
if len(section) > 2:
return (msg, None)
elif len(section) == 2:
section2 = section[1].split(sep2)
if len(section2) > 2:
return (msg, None)
if len(section) != 1:
section = [section[0]] + section2
dataTypes = T.sectionFeatureTypes
sectionTypes = T.sectionTypes
sectionTyped = []
for (i, sectionPart) in enumerate(section):
if dataTypes[i] == 'int':
try:
part = int(sectionPart)
except ValueError:
return (msgi.format(sectionTypes[i], sectionPart))
else:
part = sectionPart
sectionTyped.append(part)
sectionNode = T.nodeFromSection(sectionTyped)
if sectionNode is None:
return (msg, None)
return ('', sectionNode)
def htmlEsc(val):
return '' if val is None else str(val).replace('&', '&').replace('<', '<')
def mdEsc(val):
return '' if val is None else str(val).replace('|', '|')
def dm(md):
display(Markdown(md))
def dh(html):
display(HTML(html))
def _tupleEnum(tuples, start, end, limit, item):
if start is None:
start = 1
i = -1
if not hasattr(tuples, '__len__'):
if end is None or end - start + 1 > limit:
end = start - 1 + limit
for tup in tuples:
i += 1
if i < start - 1:
continue
if i >= end:
break
yield (i + 1, tup)
else:
if end is None or end > len(tuples):
end = len(tuples)
rest = 0
if end - (start - 1) > limit:
rest = end - (start - 1) - limit
end = start - 1 + limit
for i in range(start - 1, end):
yield (i + 1, tuples[i])
if rest:
dm(
f'**{rest} more {item}s skipped** because we show a maximum of'
f' {limit} {item}s at a time'
)
def _condense(api, tuples, condenseType, multiple=False):
F = api.F
L = api.L
sortNodes = api.sortNodes
otypeRank = api.otypeRank
slotType = F.otype.slotType
condenseRank = otypeRank[condenseType]
containers = {}
if not multiple:
tuples = (tuples,)
for tup in tuples:
for n in tup:
nType = F.otype.v(n)
if nType == condenseType:
containers.setdefault(n, set())
elif nType == slotType:
up = L.u(n, otype=condenseType)
if up:
containers.setdefault(up[0], set()).add(n)
elif otypeRank[nType] < condenseRank:
slots = L.d(n, otype=slotType)
first = slots[0]
last = slots[-1]
firstUp = L.u(first, otype=condenseType)
lastUp = L.u(last, otype=condenseType)
allUps = set()
if firstUp:
allUps.add(firstUp[0])
if lastUp:
allUps.add(lastUp[0])
for up in allUps:
containers.setdefault(up, set()).add(n)
else:
pass
# containers.setdefault(n, set())
return tuple((c, ) + tuple(containers[c]) for c in sortNodes(containers))
def _condenseSet(api, tuples, condenseType, multiple=False):
F = api.F
L = api.L
sortNodes = api.sortNodes
otypeRank = api.otypeRank
slotType = F.otype.slotType
condenseRank = otypeRank[condenseType]
containers = set()
if not multiple:
tuples = (tuples,)
for tup in tuples:
for n in tup:
nType = F.otype.v(n)
if nType == condenseType:
containers.add(n)
elif nType == slotType:
up = L.u(n, otype=condenseType)
if up:
containers.add(up[0])
elif otypeRank[nType] < condenseRank:
slots = L.d(n, otype=slotType)
first = slots[0]
last = slots[-1]
firstUp = L.u(first, otype=condenseType)
lastUp = L.u(last, otype=condenseType)
if firstUp:
containers.add(firstUp[0])
if lastUp:
containers.add(lastUp[0])
else:
containers.add(n)
# we skip nodes with a higher rank than that of the container
return sortNodes(containers)
def _getHighlights(api, tuples, highlights, colorMap, condenseType, multiple=False):
F = api.F
otypeRank = api.otypeRank
condenseRank = otypeRank[condenseType]
if type(highlights) is set:
highlights = {m: '' for m in highlights}
newHighlights = {}
if highlights:
newHighlights.update(highlights)
if not multiple:
tuples = (tuples,)
for tup in tuples:
for (i, n) in enumerate(tup):
nType = F.otype.v(n)
if otypeRank[nType] < condenseRank:
thisHighlight = None
if highlights:
thisHighlight = highlights.get(n, None)
elif colorMap is not None:
thisHighlight = colorMap.get(i + 1, None)
else:
thisHighlight = ''
if thisHighlight is not None:
newHighlights[n] = thisHighlight
return newHighlights