swh:1:snp:7ce5f1105410d5ee1ad6abfdc873986c25b579e5
Tip revision: 2c7e94d622178a2f2479d3508a0c89d39817a9f7 authored by Dirk Roorda on 30 January 2019, 22:25:54 UTC
section checks in conversion
section checks in conversion
Tip revision: 2c7e94d
start.py
import sys
import os
from platform import system
import psutil
import webbrowser
from time import sleep
from subprocess import PIPE, Popen
from ..core.helpers import console
from ..parameters import NAME, VERSION
from ..applib.app import findApp, findAppConfig
from .command import (
argDebug,
argCheck,
argNoweb,
argDocker,
argLocalClones,
argModules,
argSets,
argParam,
)
from .kernel import TF_DONE, TF_ERROR
HELP = '''
USAGE
text-fabric --help
text-fabric --version
text-fabric datasource [-lgc] [-d] [-c] [-noweb] [-docker] [--mod=modules] [--set=file]
text-fabric -k [datasource]
EFFECT
If a datasource is given and the -k flag is not passed,
a TF kernel for that data source is started.
When the TF kernel is ready, a web server is started
serving a website that exposes the data through
a query interface.
The website can be served or on the internet.
The default browser will be opened, except when -noweb is passed.
--mod=modules
Optionally, you can pass a comma-separated list of modules.
Modules are extra sets of features on top op the chosen data source.
You specify a module by giving the github repository where it is created,
in the form
{org}/{repo}/{path}
where
{org} is the github organization,
{repo} the name of the repository in that organization
{path} the path to the data within that repo.
It is assumed that the data is stored in directories under {path},
where the directories are named as the versions that exists in the main data source.
--set=file
Optionally, you can pass a file name with the definition of custom sets in it.
This must be a dictionary were the keys are names of sets, and the values
are node sets.
This dictionary will be passed to the TF kernel, which will use it when it runs
queries.
DATA LOADING
Text-Fabric looks for data in ~/text-fabric-data.
If data is not found there, it first downloads the relevant data from
github.
-lgc Look for data first in local github clones under ~/github.
If data is not found there, get data in the normal way.
-c Check for data updates online. If a newer release of the data is found,
it will be downloaded.
MISCELLANEOUS
-d Debug mode. For developers of Text-Fabric itself.
The web server reloads when its code changes.
The web server is a Flask instance, started with reload=True.
-noweb Do not start the default browser
-docker Assume you are on docker. The local web server is passed 0.0.0.0 as host.
CLEAN UP
If you press Ctrl-C the web server is stopped, and after that the TF kernel
as well.
Normally, you do not have to do any clean up.
But if the termination is done in an irregular way, you may end up with
stray processes.
-k Kill mode. If a data source is given, the TF kernel and web server for that
data source are killed.
Without a data source, all local webinterface related processes are killed.
'''
FLAGS = set('''
-c
-d
-lgc
-noweb
-docker
'''.strip().split())
BANNER = f'This is {NAME} {VERSION}'
def filterProcess(proc):
procName = proc.info['name']
commandName = '' if procName is None else procName.lower()
# commandName = proc.info['name'].lower()
found = False
kind = None
modules = ''
sets = ''
dataSource = None
trigger = 'python'
if commandName.endswith(trigger) or commandName.endswith(f'{trigger}.exe'):
good = False
parts = proc.cmdline()
for part in parts:
part = part.lower()
if part in FLAGS:
continue
trigger = 'text-fabric'
if part.endswith(trigger) or part.endswith(f'{trigger}.exe'):
kind = 'tf'
continue
if part == 'tf.server.kernel':
kind = 'data'
good = True
continue
if part == 'tf.server.web':
kind = 'web'
good = True
continue
if part.endswith('web.py'):
kind = 'web'
good = True
continue
if kind in {'data', 'web', 'tf'}:
if kind == 'tf' and part == '-k':
break
if part.startswith('--mod='):
modules = part
elif part.startswith('--sets='):
sets = part
else:
dataSource = part
good = True
if good:
found = True
if found:
return (kind, dataSource, modules, sets)
return False
def killProcesses(dataSource, modules, sets, kill=False):
tfProcs = {}
for proc in psutil.process_iter(attrs=['pid', 'name']):
test = filterProcess(proc)
if test:
(kind, ds, mods, sts) = test
tfProcs.setdefault((ds, (mods, sts)), {}).setdefault(kind, []).append(proc.info['pid'])
item = 'killed' if kill else 'terminated'
myself = os.getpid()
for ((ds, (mods, sets)), kinds) in tfProcs.items():
if dataSource is None or (ds == dataSource and mods == modules and sts == sets):
checkKinds = ('data', 'web', 'tf')
for kind in checkKinds:
pids = kinds.get(kind, [])
for pid in pids:
if pid == myself:
continue
try:
proc = psutil.Process(pid=pid)
if kill:
proc.kill()
else:
proc.terminate()
console(f'Process {kind} server for {ds}: {item}')
except psutil.NoSuchProcess:
console(f'Process {kind} server for {ds}: already {item}', error=True)
def getKill(cargs=sys.argv):
for arg in cargs[1:]:
if arg == '-k':
return True
return False
def main(cargs=sys.argv):
console(BANNER)
if len(cargs) >= 2 and any(arg in {'--help', '-help', '-h', '?', '-?'} for arg in cargs[1:]):
console(HELP)
return
if len(cargs) >= 2 and any(arg in {'--version', '-version', '-v'} for arg in cargs[1:]):
return
isWin = system().lower().startswith('win')
kill = getKill(cargs=cargs)
dataSource = argParam(cargs=cargs, interactive=not kill)
modules = argModules(cargs=cargs)
sets = argSets(cargs=cargs)
if kill:
if dataSource is False:
return
killProcesses(dataSource, modules, sets)
return
noweb = argNoweb(cargs=cargs)
debug = argDebug(cargs=cargs)
docker = argDocker(cargs=cargs)
check = argCheck(cargs=cargs)
lgc = argLocalClones(cargs=cargs)
kdataSource = ('-lgc', dataSource) if lgc else (dataSource, )
kdataSource = ('-c', *kdataSource) if check else kdataSource
kdataSource = (modules, *kdataSource) if modules else kdataSource
kdataSource = (sets, *kdataSource) if sets else kdataSource
ddataSource = ('-d', dataSource) if debug else (dataSource, )
ddataSource = ('-docker', *ddataSource) if docker else ddataSource
ddataSource = ('-lgc', *ddataSource) if lgc else ddataSource
ddataSource = (modules, *ddataSource) if modules else ddataSource
if dataSource is not None:
(commit, appDir) = findApp(dataSource, lgc, check)
if appDir is None:
return
config = findAppConfig(dataSource, appDir)
pKernel = None
pWeb = None
if config is None:
return
console(f'Cleaning up remnant processes, if any ...')
killProcesses(dataSource, modules, sets, kill=True)
pythonExe = 'python' if isWin else 'python3'
pKernel = Popen(
[pythonExe, '-m', 'tf.server.kernel', *kdataSource],
stdout=PIPE,
bufsize=1,
encoding='utf-8',
)
console(f'Loading data for {dataSource}. Please wait ...')
for line in pKernel.stdout:
sys.stdout.write(line)
if line.rstrip() == TF_ERROR:
return
if line.rstrip() == TF_DONE:
break
sleep(1)
pWeb = Popen(
[pythonExe, '-m', f'tf.server.web', *ddataSource],
bufsize=0,
)
if not noweb:
sleep(1)
console(f'Opening {dataSource} in browser')
webbrowser.open(
f'{config.PROTOCOL}{config.HOST}:{config.PORT["web"]}',
new=2,
autoraise=True,
)
if pWeb and pKernel:
try:
for line in pKernel.stdout:
sys.stdout.write(line)
except KeyboardInterrupt:
console('')
if pWeb:
pWeb.terminate()
console('TF web server has stopped')
if pKernel:
pKernel.terminate()
for line in pKernel.stdout:
sys.stdout.write(line)
console('TF kernel has stopped')
if __name__ == "__main__":
main()