import sys
import os
import re
from glob import glob
from time import sleep
from shutil import rmtree
from subprocess import run, Popen
import psutil
from tf.core.helpers import console
# from tf.applib.find import findApp, findAppConfig
DIST = "dist"
VERSION_CONFIG = dict(
setup=dict(
file="setup.py",
re=re.compile(r"""version\s*=\s*['"]([^'"]*)['"]"""),
mask="version='{}'",
),
parameters=dict(
file="tf/parameters.py",
re=re.compile(r"""\bVERSION\s*=\s*['"]([^'"]*)['"]"""),
mask="VERSION = '{}'",
),
)
AN_BASE = os.path.expanduser("~/github/annotation")
TUT_BASE = f"{AN_BASE}/tutorials"
TF_BASE = f"{AN_BASE}/text-fabric"
TEST_BASE = f"{TF_BASE}/test"
APP_BASE = f"{TF_BASE}/apps"
PACKAGE = "text-fabric"
SCRIPT = "/Library/Frameworks/Python.framework/Versions/3.7/bin/text-fabric"
currentVersion = None
newVersion = None
appPat = r"^.*/app-([^/]*)$"
appRe = re.compile(appPat)
apps = set()
for appDir in glob(f"{AN_BASE}/app-*"):
match = appRe.fullmatch(appDir)
if match:
apps.add(match.group(1))
apps = sorted(apps)
appStr = ", ".join(apps)
HELP = f"""
python3 build.py command
command:
-h
--help
help : print help and exit
adocs : build apidocs
docs : serve docs locally
clean : clean local develop build
l : local develop build
i : local non-develop build
g : push to github, code and docs
v : show current version
r1 : version becomes r1+1.0.0
r2 : version becomes r1.r2+1.0
r3 : version becomes r1.r2.r3+1
ship : build for shipping
apps : commit and push all tf apps
tut : commit and push the tutorials repo
a : open text-fabric browser on specific dataset
({appStr})
t : run test suite (relations, qperf)
data : build data files for github release
For g and ship you need to pass a commit message.
For data you need to pass an app argument:
{appStr}
"""
def readArgs():
args = sys.argv[1:]
if not len(args) or args[0] in {"-h", "--help", "help"}:
console(HELP)
return (False, None, [])
arg = args[0]
if arg not in {
"a",
"t",
"adocs",
"docs",
"clean",
"l",
"lp",
"i",
"g",
"ship",
"data",
"apps",
"tut",
"v",
"r1",
"r2",
"r3",
}:
console(HELP)
return (False, None, [])
if arg in {"g", "apps", "tut", "ship"}:
if len(args) < 2:
console("Provide a commit message")
return (False, None, [])
return (arg, args[1], args[2:])
if arg in {"a", "t", "data"}:
if len(args) < 2:
if arg in {"a", "data"}:
console(f"Provide a data source [{appStr}]")
elif arg in {"t"}:
console("Provide a test suite [relations, qperf]")
return (False, None, [])
return (arg, args[1], args[2:])
return (arg, None, [])
def incVersion(version, task):
comps = [int(c) for c in version.split(".")]
(major, minor, update) = comps
if task == "r1":
major += 1
minor = 0
update = 0
elif task == "r2":
minor += 1
update = 0
elif task == "r3":
update += 1
return ".".join(str(c) for c in (major, minor, update))
def replaceVersion(task, mask):
def subVersion(match):
global currentVersion
global newVersion
currentVersion = match.group(1)
newVersion = incVersion(currentVersion, task)
return mask.format(newVersion)
return subVersion
def showVersion():
global currentVersion
versions = set()
for (key, c) in VERSION_CONFIG.items():
with open(c["file"]) as fh:
text = fh.read()
match = c["re"].search(text)
version = match.group(1)
console(f'{version} (according to {c["file"]})')
versions.add(version)
currentVersion = None
if len(versions) == 1:
currentVersion = list(versions)[0]
def adjustVersion(task):
for (key, c) in VERSION_CONFIG.items():
console(f'Adjusting version in {c["file"]}')
with open(c["file"]) as fh:
text = fh.read()
text = c["re"].sub(replaceVersion(task, c["mask"]), text)
with open(c["file"], "w") as fh:
fh.write(text)
if currentVersion == newVersion:
console(f"Rebuilding version {newVersion}")
else:
console(f"Replacing version {currentVersion} by {newVersion}")
def makeDist(pypi=True):
distFile = "{}-{}".format(PACKAGE, currentVersion)
distFileCompressed = f"{distFile}.tar.gz"
distPath = f"{DIST}/{distFileCompressed}"
print(distPath)
rmtree(DIST)
os.makedirs(DIST, exist_ok=True)
run(["python3", "setup.py", "sdist"])
if pypi:
run(["twine", "upload", "-u", "dirkroorda", distPath])
run("./purge.sh", shell=True)
def commit(task, msg):
run(["git", "add", "--all", "."])
run(["git", "commit", "-m", msg])
run(["git", "push", "origin", "master"])
if task in {"ship"}:
tagVersion = f"v{currentVersion}"
commitMessage = f"Release {currentVersion}: {msg}"
run(["git", "tag", "-a", tagVersion, "-m", commitMessage])
run(["git", "push", "origin", "--tags"])
def commitApps(msg):
for app in apps:
os.chdir(f"{AN_BASE}/app-{app}")
console(f"In {os.getcwd()}")
run(["git", "add", "--all", "."])
run(["git", "commit", "-m", msg])
run(["git", "push", "origin", "master"])
os.chdir(f"{TF_BASE}")
def commitTut(msg):
os.chdir(f"{TUT_BASE}")
console(f"In {os.getcwd()}")
run(["git", "add", "--all", "."])
run(["git", "commit", "-m", msg])
run(["git", "push", "origin", "master"])
os.chdir(f"{TF_BASE}")
def shipDocs():
codestats()
apidocs()
run(["mkdocs", "gh-deploy"])
def serveDocs():
codestats()
apidocs()
killProcesses()
proc = Popen(["mkdocs", "serve"])
sleep(3)
run("open http://127.0.0.1:8000", shell=True)
try:
proc.wait()
except KeyboardInterrupt:
pass
proc.terminate()
def killProcesses():
myself = os.getpid()
for proc in psutil.process_iter(attrs=["pid", "name"]):
pid = proc.info["pid"]
if pid == myself:
continue
if filterProcess(proc):
try:
proc.terminate()
console(f"mkdocs [{pid}] terminated")
except psutil.NoSuchProcess:
console(f"mkdocs [{pid}] already terminated")
def filterProcess(proc):
procName = proc.info["name"]
commandName = "" if procName is None else procName.lower()
found = False
if commandName.endswith("python"):
parts = proc.cmdline()
if len(parts) >= 3:
if parts[1].endswith("mkdocs") and parts[2] == "serve":
found = True
if parts[1] == "build.py" and parts[2] == "docs":
found = True
return found
def apidocs():
cmdLine = "pdoc3" " --force" " --html" " --output-dir docs/apidocs/html" " tf"
run(cmdLine, shell=True)
def codestats():
xd = (
".pytest_cache,__pycache__,node_modules,.tmp,.git,_temp,"
".ipynb_checkpoints,images,fonts,favicons,compiled"
)
xdtf = xd + ",applib,convert,compose,core,search,server,writing"
xdtest = xd + ",tf"
rfFmt = "docs/Code/Stats{}.md"
cmdLine = (
"cloc"
" --no-autogen"
" --exclude_dir={}"
" --exclude-list-file={}"
f" --report-file={rfFmt}"
" --md"
" {}"
)
nex = "cloc_exclude.lst"
run(cmdLine.format(xd, nex, "", ". ../app-*/code"), shell=True)
run(cmdLine.format(xdtf, nex, "Toplevel", "tf"), shell=True)
run(cmdLine.format(xd, nex, "Applib", "tf/applib"), shell=True)
run(cmdLine.format(xd, nex, "Apps", "../app-*/code"), shell=True)
run(cmdLine.format(xd, nex, "Compose", "tf/compose"), shell=True)
run(cmdLine.format(xd, nex, "Convert", "tf/convert"), shell=True)
run(cmdLine.format(xd, nex, "Core", "tf/core"), shell=True)
run(cmdLine.format(xd, nex, "Search", "tf/search"), shell=True)
run(cmdLine.format(xd, nex, "Server", "tf/server"), shell=True)
run(cmdLine.format(xd, nex, "Writing", "tf/writing"), shell=True)
run(cmdLine.format(xdtest, nex, "Test", "test/generic"), shell=True)
def tfbrowse(dataset, remaining):
rargs = " ".join(remaining)
cmdLine = f"text-fabric {dataset} {rargs}"
try:
run(cmdLine, shell=True)
except KeyboardInterrupt:
pass
def tftest(suite, remaining):
suiteDir = f"{TEST_BASE}/generic"
suiteFile = f"{suite}.py"
good = True
try:
os.chdir(suiteDir)
except Exception:
good = False
console(f'Cannot find TF test directory "{suiteDir}"')
if not good:
return
if not os.path.exists(suiteFile):
console(f'Cannot find TF test suite "{suite}"')
return
rargs = " ".join(remaining)
cmdLine = f"python3 {suiteFile} -v {rargs}"
try:
run(cmdLine, shell=True)
except KeyboardInterrupt:
pass
def clean():
run(["python3", "setup.py", "develop", "-u"])
if os.path.exists(SCRIPT):
os.unlink(SCRIPT)
run(["pip3", "uninstall", "-y", "text-fabric"])
def main():
(task, msg, remaining) = readArgs()
if not task:
return
elif task == "a":
tfbrowse(msg, remaining)
elif task == "t":
tftest(msg, remaining)
elif task == "adocs":
apidocs()
elif task == "docs":
serveDocs()
elif task == "clean":
clean()
elif task == "l":
clean()
run(["python3", "setup.py", "develop"])
elif task == "lp":
clean()
run(["python3", "setup.py", "sdist"])
distFiles = glob("dist/text-fabric-*.tar.gz")
run(["pip3", "install", distFiles[0]])
elif task == "i":
clean
makeDist(pypi=False)
run(
[
"pip3",
"install",
"--upgrade",
"--no-index",
"--find-links",
f'file://{TF_BASE}/dist"',
"text-fabric",
]
)
elif task == "g":
shipDocs()
commit(task, msg)
elif task == "apps":
commitApps(msg)
elif task == "tut":
commitTut(msg)
elif task == "v":
showVersion()
elif task in {"r", "r1", "r2", "r3"}:
adjustVersion(task)
elif task == "ship":
showVersion()
if not currentVersion:
console("No current version")
return
answer = input("right version ? [yn]")
if answer != "y":
return
shipDocs()
makeDist()
commit(task, msg)
main()