Preprocessor.py
"""
This is a very primitive line based preprocessor, for times when using
a C preprocessor isn't an option.
"""
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import sys
import os
import os.path
import re
from optparse import OptionParser
import errno
# hack around win32 mangling our line endings
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
if sys.platform == "win32":
import msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
os.linesep = '\n'
import Expression
__all__ = ['Preprocessor', 'preprocess']
class Preprocessor:
"""
Class for preprocessing text files.
"""
class Error(RuntimeError):
def __init__(self, cpp, MSG, context):
self.file = cpp.context['FILE']
self.line = cpp.context['LINE']
self.key = MSG
RuntimeError.__init__(self, (self.file, self.line, self.key, context))
def __init__(self):
self.context = Expression.Context()
for k,v in {'FILE': '',
'LINE': 0,
'DIRECTORY': os.path.abspath('.')}.iteritems():
self.context[k] = v
self.actionLevel = 0
self.disableLevel = 0
# ifStates can be
# 0: hadTrue
# 1: wantsTrue
# 2: #else found
self.ifStates = []
self.checkLineNumbers = False
self.writtenLines = 0
self.filters = []
self.cmds = {}
for cmd, level in {'define': 0,
'undef': 0,
'if': sys.maxint,
'ifdef': sys.maxint,
'ifndef': sys.maxint,
'else': 1,
'elif': 1,
'elifdef': 1,
'elifndef': 1,
'endif': sys.maxint,
'expand': 0,
'literal': 0,
'filter': 0,
'unfilter': 0,
'include': 0,
'includesubst': 0,
'error': 0}.iteritems():
self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
self.out = sys.stdout
self.setMarker('#')
self.LE = '\n'
self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)
def warnUnused(self, file):
if self.actionLevel == 0:
sys.stderr.write('{0}: WARNING: no preprocessor directives found\n'.format(file))
elif self.actionLevel == 1:
sys.stderr.write('{0}: WARNING: no useful preprocessor directives found\n'.format(file))
pass
def setLineEndings(self, aLE):
"""
Set the line endings to be used for output.
"""
self.LE = {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE]
def setMarker(self, aMarker):
"""
Set the marker to be used for processing directives.
Used for handling CSS files, with pp.setMarker('%'), for example.
The given marker may be None, in which case no markers are processed.
"""
self.marker = aMarker
if aMarker:
self.instruction = re.compile('{0}(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'
.format(aMarker),
re.U)
self.comment = re.compile(aMarker, re.U)
else:
class NoMatch(object):
def match(self, *args):
return False
self.instruction = self.comment = NoMatch()
def clone(self):
"""
Create a clone of the current processor, including line ending
settings, marker, variable definitions, output stream.
"""
rv = Preprocessor()
rv.context.update(self.context)
rv.setMarker(self.marker)
rv.LE = self.LE
rv.out = self.out
return rv
def applyFilters(self, aLine):
for f in self.filters:
aLine = f[1](aLine)
return aLine
def write(self, aLine):
"""
Internal method for handling output.
"""
if self.checkLineNumbers:
self.writtenLines += 1
ln = self.context['LINE']
if self.writtenLines != ln:
self.out.write('//@line {line} "{file}"{le}'.format(line=ln,
file=self.context['FILE'],
le=self.LE))
self.writtenLines = ln
filteredLine = self.applyFilters(aLine)
if filteredLine != aLine:
self.actionLevel = 2
# ensure our line ending. Only need to handle \n, as we're reading
# with universal line ending support, at least for files.
filteredLine = re.sub('\n', self.LE, filteredLine)
self.out.write(filteredLine)
def handleCommandLine(self, args, defaultToStdin = False):
"""
Parse a commandline into this parser.
Uses OptionParser internally, no args mean sys.argv[1:].
"""
p = self.getCommandLineParser()
(options, args) = p.parse_args(args=args)
includes = options.I
if options.output:
dir = os.path.dirname(options.output)
if dir and not os.path.exists(dir):
try:
os.makedirs(dir)
except OSError as error:
if error.errno != errno.EEXIST:
raise
self.out = open(options.output, 'wb')
if defaultToStdin and len(args) == 0:
args = [sys.stdin]
includes.extend(args)
if includes:
for f in includes:
self.do_include(f, False)
self.warnUnused(f)
pass
def getCommandLineParser(self, unescapeDefines = False):
escapedValue = re.compile('".*"$')
numberValue = re.compile('\d+$')
def handleE(option, opt, value, parser):
for k,v in os.environ.iteritems():
self.context[k] = v
def handleD(option, opt, value, parser):
vals = value.split('=', 1)
if len(vals) == 1:
vals.append(1)
elif unescapeDefines and escapedValue.match(vals[1]):
# strip escaped string values
vals[1] = vals[1][1:-1]
elif numberValue.match(vals[1]):
vals[1] = int(vals[1])
self.context[vals[0]] = vals[1]
def handleU(option, opt, value, parser):
del self.context[value]
def handleF(option, opt, value, parser):
self.do_filter(value)
def handleLE(option, opt, value, parser):
self.setLineEndings(value)
def handleMarker(option, opt, value, parser):
self.setMarker(value)
p = OptionParser()
p.add_option('-I', action='append', type="string", default = [],
metavar="FILENAME", help='Include file')
p.add_option('-E', action='callback', callback=handleE,
help='Import the environment into the defined variables')
p.add_option('-D', action='callback', callback=handleD, type="string",
metavar="VAR[=VAL]", help='Define a variable')
p.add_option('-U', action='callback', callback=handleU, type="string",
metavar="VAR", help='Undefine a variable')
p.add_option('-F', action='callback', callback=handleF, type="string",
metavar="FILTER", help='Enable the specified filter')
p.add_option('-o', '--output', type="string", default=None,
metavar="FILENAME", help='Output to the specified file '+
'instead of stdout')
p.add_option('--line-endings', action='callback', callback=handleLE,
type="string", metavar="[cr|lr|crlf]",
help='Use the specified line endings [Default: OS dependent]')
p.add_option('--marker', action='callback', callback=handleMarker,
type="string",
help='Use the specified marker instead of #')
return p
def handleLine(self, aLine):
"""
Handle a single line of input (internal).
"""
if self.actionLevel == 0 and self.comment.match(aLine):
self.actionLevel = 1
m = self.instruction.match(aLine)
if m:
args = None
cmd = m.group('cmd')
try:
args = m.group('args')
except IndexError:
pass
if cmd not in self.cmds:
raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
level, cmd = self.cmds[cmd]
if (level >= self.disableLevel):
cmd(args)
if cmd != 'literal':
self.actionLevel = 2
elif self.disableLevel == 0 and not self.comment.match(aLine):
self.write(aLine)
pass
# Instruction handlers
# These are named do_'instruction name' and take one argument
# Variables
def do_define(self, args):
m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
if not m:
raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
val = 1
if m.group('value'):
val = self.applyFilters(m.group('value'))
try:
val = int(val)
except:
pass
self.context[m.group('name')] = val
def do_undef(self, args):
m = re.match('(?P<name>\w+)$', args, re.U)
if not m:
raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
if args in self.context:
del self.context[args]
# Logic
def ensure_not_else(self):
if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
sys.stderr.write('WARNING: bad nesting of #else\n')
def do_if(self, args, replace=False):
if self.disableLevel and not replace:
self.disableLevel += 1
return
val = None
try:
e = Expression.Expression(args)
val = e.evaluate(self.context)
except Exception:
# XXX do real error reporting
raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
if type(val) == str:
# we're looking for a number value, strings are false
val = False
if not val:
self.disableLevel = 1
if replace:
if val:
self.disableLevel = 0
self.ifStates[-1] = self.disableLevel
else:
self.ifStates.append(self.disableLevel)
pass
def do_ifdef(self, args, replace=False):
if self.disableLevel and not replace:
self.disableLevel += 1
return
if re.match('\W', args, re.U):
raise Preprocessor.Error(self, 'INVALID_VAR', args)
if args not in self.context:
self.disableLevel = 1
if replace:
if args in self.context:
self.disableLevel = 0
self.ifStates[-1] = self.disableLevel
else:
self.ifStates.append(self.disableLevel)
pass
def do_ifndef(self, args, replace=False):
if self.disableLevel and not replace:
self.disableLevel += 1
return
if re.match('\W', args, re.U):
raise Preprocessor.Error(self, 'INVALID_VAR', args)
if args in self.context:
self.disableLevel = 1
if replace:
if args not in self.context:
self.disableLevel = 0
self.ifStates[-1] = self.disableLevel
else:
self.ifStates.append(self.disableLevel)
pass
def do_else(self, args, ifState = 2):
self.ensure_not_else()
hadTrue = self.ifStates[-1] == 0
self.ifStates[-1] = ifState # in-else
if hadTrue:
self.disableLevel = 1
return
self.disableLevel = 0
def do_elif(self, args):
if self.disableLevel == 1:
if self.ifStates[-1] == 1:
self.do_if(args, replace=True)
else:
self.do_else(None, self.ifStates[-1])
def do_elifdef(self, args):
if self.disableLevel == 1:
if self.ifStates[-1] == 1:
self.do_ifdef(args, replace=True)
else:
self.do_else(None, self.ifStates[-1])
def do_elifndef(self, args):
if self.disableLevel == 1:
if self.ifStates[-1] == 1:
self.do_ifndef(args, replace=True)
else:
self.do_else(None, self.ifStates[-1])
def do_endif(self, args):
if self.disableLevel > 0:
self.disableLevel -= 1
if self.disableLevel == 0:
self.ifStates.pop()
# output processing
def do_expand(self, args):
lst = re.split('__(\w+)__', args, re.U)
do_replace = False
def vsubst(v):
if v in self.context:
return str(self.context[v])
return ''
for i in range(1, len(lst), 2):
lst[i] = vsubst(lst[i])
lst.append('\n') # add back the newline
self.write(reduce(lambda x, y: x+y, lst, ''))
def do_literal(self, args):
self.write(args + self.LE)
def do_filter(self, args):
filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
if len(filters) == 0:
return
current = dict(self.filters)
for f in filters:
current[f] = getattr(self, 'filter_' + f)
filterNames = current.keys()
filterNames.sort()
self.filters = [(fn, current[fn]) for fn in filterNames]
return
def do_unfilter(self, args):
filters = args.split(' ')
current = dict(self.filters)
for f in filters:
if f in current:
del current[f]
filterNames = current.keys()
filterNames.sort()
self.filters = [(fn, current[fn]) for fn in filterNames]
return
# Filters
#
# emptyLines
# Strips blank lines from the output.
def filter_emptyLines(self, aLine):
if aLine == '\n':
return ''
return aLine
# slashslash
# Strips everything after //
def filter_slashslash(self, aLine):
if (aLine.find('//') == -1):
return aLine
[aLine, rest] = aLine.split('//', 1)
if rest:
aLine += '\n'
return aLine
# spaces
# Collapses sequences of spaces into a single space
def filter_spaces(self, aLine):
return re.sub(' +', ' ', aLine).strip(' ')
# substition
# helper to be used by both substition and attemptSubstitution
def filter_substitution(self, aLine, fatal=True):
def repl(matchobj):
varname = matchobj.group('VAR')
if varname in self.context:
return str(self.context[varname])
if fatal:
raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
return matchobj.group(0)
return self.varsubst.sub(repl, aLine)
def filter_attemptSubstitution(self, aLine):
return self.filter_substitution(aLine, fatal=False)
# File ops
def do_include(self, args, filters=True):
"""
Preprocess a given file.
args can either be a file name, or a file-like object.
Files should be opened, and will be closed after processing.
"""
isName = type(args) == str or type(args) == unicode
oldWrittenLines = self.writtenLines
oldCheckLineNumbers = self.checkLineNumbers
self.checkLineNumbers = False
if isName:
try:
args = str(args)
if filters:
args = self.applyFilters(args)
if not os.path.isabs(args):
args = os.path.join(self.context['DIRECTORY'], args)
args = open(args, 'rU')
except Preprocessor.Error:
raise
except:
raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
self.checkLineNumbers = bool(re.search('\.(js|jsm|java)(?:\.in)?$', args.name))
oldFile = self.context['FILE']
oldLine = self.context['LINE']
oldDir = self.context['DIRECTORY']
if args.isatty():
# we're stdin, use '-' and '' for file and dir
self.context['FILE'] = '-'
self.context['DIRECTORY'] = ''
else:
abspath = os.path.abspath(args.name)
self.context['FILE'] = abspath
self.context['DIRECTORY'] = os.path.dirname(abspath)
self.context['LINE'] = 0
self.writtenLines = 0
for l in args:
self.context['LINE'] += 1
self.handleLine(l)
args.close()
self.context['FILE'] = oldFile
self.checkLineNumbers = oldCheckLineNumbers
self.writtenLines = oldWrittenLines
self.context['LINE'] = oldLine
self.context['DIRECTORY'] = oldDir
def do_includesubst(self, args):
args = self.filter_substitution(args)
self.do_include(args)
def do_error(self, args):
raise Preprocessor.Error(self, 'Error: ', str(args))
def main():
pp = Preprocessor()
pp.handleCommandLine(None, True)
return
def preprocess(includes=[sys.stdin], defines={},
output = sys.stdout,
line_endings='\n', marker='#'):
pp = Preprocessor()
pp.context.update(defines)
pp.setLineEndings(line_endings)
pp.setMarker(marker)
pp.out = output
for f in includes:
pp.do_include(f, False)
if __name__ == "__main__":
main()