Index: genshi/tests/builder.py =================================================================== --- genshi/tests/builder.py (revision 466) +++ genshi/tests/builder.py (working copy) @@ -45,11 +45,11 @@ def test_stream_as_child(self): xml = list(tag.span(XML('Foo')).generate()) self.assertEqual(5, len(xml)) - self.assertEqual((Stream.START, ('span', ()), (None, -1, -1)), xml[0]) - self.assertEqual((Stream.START, ('b', ()), (None, 1, 0)), xml[1]) - self.assertEqual((Stream.TEXT, 'Foo', (None, 1, 3)), xml[2]) - self.assertEqual((Stream.END, 'b', (None, 1, 6)), xml[3]) - self.assertEqual((Stream.END, 'span', (None, -1, -1)), xml[4]) + self.assertEqual((Stream.START, ('span', ())), xml[0][:2]) + self.assertEqual((Stream.START, ('b', ())), xml[1][:2]) + self.assertEqual((Stream.TEXT, 'Foo'), xml[2][:2]) + self.assertEqual((Stream.END, 'b'), xml[3][:2]) + self.assertEqual((Stream.END, 'span'), xml[4][:2]) def suite(): Index: genshi/path.py =================================================================== --- genshi/path.py (revision 466) +++ genshi/path.py (working copy) @@ -156,8 +156,8 @@ >>> test = Path('child').test() >>> for event in xml: ... if test(event, {}, {}): - ... print event - ('START', (QName(u'child'), Attrs([(QName(u'id'), u'2')])), (None, 1, 34)) + ... print event[0], repr(event[1]) + START (QName(u'child'), Attrs([(QName(u'id'), u'2')])) """ paths = [(p, len(p), [0], [], [0] * len(p)) for p in [ (ignore_context and [_DOTSLASHSLASH] or []) + p for p in self.paths Index: genshi/template/core.py =================================================================== --- genshi/template/core.py (revision 466) +++ genshi/template/core.py (working copy) @@ -107,6 +107,21 @@ def __repr__(self): return repr(list(self.frames)) + def __contains__(self, key): + """Return whether a variable exists in any of the scopes.""" + return self._find(key)[1] is not None + + def __getitem__(self, key): + """Get a variables's value, starting at the current scope and going + upward. + + Raises `KeyError` if the requested variable wasn't found in any scope. + """ + value, frame = self._find(key) + if frame is None: + raise KeyError(key) + return value + def __setitem__(self, key, value): """Set a variable in the current scope.""" self.frames[0][key] = value @@ -129,8 +144,16 @@ if key in frame: return frame[key] return default - __getitem__ = get + def keys(self): + keys = [] + for frame in self.frames: + keys += [key for key in frame if key not in keys] + return keys + + def items(self): + return [(key, self.get(key)) for key in self.keys()] + def push(self, data): """Push a new scope on the stack.""" Index: genshi/template/tests/markup.py =================================================================== --- genshi/template/tests/markup.py (revision 466) +++ genshi/template/tests/markup.py (working copy) @@ -195,6 +195,28 @@ \xf6 """, unicode(tmpl.generate())) + def test_exec_import(self): + tmpl = MarkupTemplate(u""" +
+ ${timedelta(days=2)} +
""") + self.assertEqual(u"""
+ 2 days, 0:00:00 +
""", str(tmpl.generate())) + + def test_exec_def(self): + tmpl = MarkupTemplate(u""" + +
+ ${foo()} +
""") + self.assertEqual(u"""
+ 42 +
""", str(tmpl.generate())) + def test_include_in_loop(self): dirname = tempfile.mkdtemp(suffix='genshi_test') try: Index: genshi/template/tests/eval.py =================================================================== --- genshi/template/tests/eval.py (revision 466) +++ genshi/template/tests/eval.py (working copy) @@ -15,7 +15,7 @@ import sys import unittest -from genshi.template.eval import Expression, Undefined +from genshi.template.eval import Expression, Suite, Undefined class ExpressionTestCase(unittest.TestCase): @@ -382,10 +382,100 @@ self.assertRaises(TypeError, expr.evaluate, {'nothing': object()}) +class SuiteTestCase(unittest.TestCase): + + def test_assign(self): + suite = Suite("foo = 42") + data = {} + suite.execute(data) + self.assertEqual(42, data['foo']) + + def test_def(self): + suite = Suite("def donothing(): pass") + data = {} + suite.execute(data) + assert 'donothing' in data + self.assertEqual(None, data['donothing']()) + + def test_delete(self): + suite = Suite("""foo = 42 +del foo +""") + data = {} + suite.execute(data) + assert 'foo' not in data + + def test_class(self): + suite = Suite("class plain(object): pass") + data = {} + suite.execute(data) + assert 'plain' in data + + def test_import(self): + suite = Suite("from itertools import ifilter") + data = {} + suite.execute(data) + assert 'ifilter' in data + + def test_for(self): + suite = Suite("""x = [] +for i in range(3): + x.append(i**2) +""") + data = {} + suite.execute(data) + self.assertEqual([0, 1, 4], data['x']) + + def test_if(self): + suite = Suite("""if foo == 42: + x = True +""") + data = {'foo': 42} + suite.execute(data) + self.assertEqual(True, data['x']) + + def test_raise(self): + suite = Suite("""raise NotImplementedError""") + self.assertRaises(NotImplementedError, suite.execute, {}) + + def test_try_except(self): + suite = Suite("""try: + import somemod +except ImportError: + somemod = None +else: + somemod.dosth()""") + data = {} + suite.execute(data) + self.assertEqual(None, data['somemod']) + + def test_finally(self): + suite = Suite("""try: + x = 2 +finally: + x = None +""") + data = {} + suite.execute(data) + self.assertEqual(None, data['x']) + + def test_while_break(self): + suite = Suite("""x = 0 +while x < 5: + x += step + if x == 4: + break +""") + data = {'step': 2} + suite.execute(data) + self.assertEqual(4, data['x']) + + def suite(): suite = unittest.TestSuite() suite.addTest(doctest.DocTestSuite(Expression.__module__)) suite.addTest(unittest.makeSuite(ExpressionTestCase, 'test')) + suite.addTest(unittest.makeSuite(SuiteTestCase, 'test')) return suite if __name__ == '__main__': Index: genshi/template/markup.py =================================================================== --- genshi/template/markup.py (revision 466) +++ genshi/template/markup.py (working copy) @@ -14,16 +14,24 @@ """Markup templating engine.""" from itertools import chain +import sys +from textwrap import dedent from genshi.core import Attrs, Namespace, Stream, StreamEventKind -from genshi.core import START, END, START_NS, END_NS, TEXT, COMMENT +from genshi.core import START, END, START_NS, END_NS, TEXT, PI, COMMENT from genshi.input import XMLParser from genshi.template.core import BadDirectiveError, Template, \ - _apply_directives, SUB + TemplateSyntaxError, _apply_directives, SUB +from genshi.template.eval import Suite from genshi.template.loader import TemplateNotFound from genshi.template.directives import * +if sys.version_info < (2, 4): + _ctxt2dict = lambda ctxt: ctxt.frames[0] +else: + _ctxt2dict = lambda ctxt: ctxt + class MarkupTemplate(Template): """Implementation of the template language for XML-based templates. @@ -35,6 +43,7 @@
  • 1
  • 2
  • 3
    • """ + EXEC = StreamEventKind('EXEC') INCLUDE = StreamEventKind('INCLUDE') DIRECTIVE_NAMESPACE = Namespace('http://genshi.edgewall.org/') @@ -59,7 +68,7 @@ Template.__init__(self, source, basedir=basedir, filename=filename, loader=loader, encoding=encoding) - self.filters.append(self._match) + self.filters += [self._exec, self._match] if loader: self.filters.append(self._include) @@ -169,6 +178,27 @@ stream[start_offset:] = [(SUB, (directives, substream), pos)] + elif kind is PI and data[0] == 'python': + try: + # As Expat doesn't report whitespace between the PI target + # and the data, we have to jump through some hoops here to + # get correctly indented Python code + # Unfortunately, we'll still probably not get the line + # number quite right + lines = [line.expandtabs() for line in data[1].splitlines()] + first = lines[0] + rest = dedent('\n'.join(lines[1:])) + if first.rstrip().endswith(':') and not rest[0].isspace(): + rest = '\n'.join([' ' + line for line + in rest.splitlines()]) + source = '\n'.join([first, rest]) + suite = Suite(source, self.filepath, pos[1]) + except SyntaxError, err: + raise TemplateSyntaxError(err, self.filepath, + pos[1] + (err.lineno or 1) - 1, + pos[2] + (err.offset or 0)) + stream.append((EXEC, suite, pos)) + elif kind is TEXT: for kind, data, pos in self._interpolate(data, self.basedir, *pos): @@ -190,6 +220,16 @@ data = data[0], list(self._prepare(data[1])) yield kind, data, pos + def _exec(self, stream, ctxt): + """Internal stream filter that executes code in processing + instructions. + """ + for event in stream: + if event[0] is EXEC: + event[1].execute(_ctxt2dict(ctxt)) + else: + yield event + def _include(self, stream, ctxt): """Internal stream filter that performs inclusion of external template files. @@ -291,4 +331,5 @@ yield event +EXEC = MarkupTemplate.EXEC INCLUDE = MarkupTemplate.INCLUDE Index: genshi/template/eval.py =================================================================== --- genshi/template/eval.py (revision 466) +++ genshi/template/eval.py (working copy) @@ -15,19 +15,65 @@ import __builtin__ from compiler import ast, parse -from compiler.pycodegen import ExpressionCodeGenerator +from compiler.pycodegen import ExpressionCodeGenerator, ModuleCodeGenerator import new try: set except NameError: from sets import Set as set +import sys from genshi.util import flatten -__all__ = ['Expression', 'Undefined'] +if sys.version_info < (2, 4): + _locals = dict +else: + _locals = lambda x: x +__all__ = ['Expression', 'Suite', 'Undefined'] -class Expression(object): + +class Code(object): + """Abstract base class for the `Expression` and `Suite` classes.""" + __slots__ = ['source', 'code'] + + def __init__(self, source, filename=None, lineno=-1): + """Create the code object, either from a string, or from an AST node. + + @param source: either a string containing the source code, or an AST + node + @param filename: the (preferably absolute) name of the file containing + the code + @param lineno: the number of the line on which the code was found + """ + if isinstance(source, basestring): + self.source = source + node = _parse(source, mode=self.mode) + else: + assert isinstance(source, ast.Node) + self.source = '?' + if self.mode == 'eval': + node = ast.Expression(source) + else: + node = ast.Module(None, source) + + self.code = _compile(node, self.source, mode=self.mode, + filename=filename, lineno=lineno) + + def __eq__(self, other): + return (type(other) == type(self)) and (self.code == other.code) + + def __hash__(self): + return hash(self.code) + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.source) + + +class Expression(Code): """Evaluates Python expressions used in templates. >>> data = dict(test='Foo', items=[1, 2, 3], dict={'some': 'thing'}) @@ -68,39 +114,9 @@ >>> Expression('len(items)').evaluate(data) 3 """ - __slots__ = ['source', 'code'] + __slots__ = [] + mode = 'eval' - def __init__(self, source, filename=None, lineno=-1): - """Create the expression, either from a string, or from an AST node. - - @param source: either a string containing the source code of the - expression, or an AST node - @param filename: the (preferably absolute) name of the file containing - the expression - @param lineno: the number of the line on which the expression was found - """ - if isinstance(source, basestring): - self.source = source - self.code = _compile(_parse(source), self.source, filename=filename, - lineno=lineno) - else: - assert isinstance(source, ast.Node) - self.source = '?' - self.code = _compile(ast.Expression(source), filename=filename, - lineno=lineno) - - def __eq__(self, other): - return (type(other) == Expression) and (self.code == other.code) - - def __hash__(self): - return hash(self.code) - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return 'Expression(%r)' % self.source - def evaluate(self, data): """Evaluate the expression against the given data dictionary. @@ -110,10 +126,31 @@ return eval(self.code, {'data': data, '_lookup_name': _lookup_name, '_lookup_attr': _lookup_attr, - '_lookup_item': _lookup_item}, - {'data': data}) + '_lookup_item': _lookup_item}, _locals(data)) +class Suite(Code): + """Executes Python statements used in templates. + + >>> data = dict(test='Foo', items=[1, 2, 3], dict={'some': 'thing'}) + >>> Suite('foo = dict.some').execute(data) + >>> data['foo'] + 'thing' + """ + __slots__ = [] + mode = 'exec' + + def execute(self, data): + """Execute the suite in the given data dictionary. + + @param data: a mapping containing the data to execute in + """ + exec self.code in {'data': data, + '_lookup_name': _lookup_name, + '_lookup_attr': _lookup_attr, + '_lookup_item': _lookup_item}, data + + class Undefined(object): """Represents a reference to an undefined variable. @@ -176,8 +213,8 @@ source = '\xef\xbb\xbf' + source.encode('utf-8') return parse(source, mode) -def _compile(node, source=None, filename=None, lineno=-1): - tree = ExpressionASTTransformer().visit(node) +def _compile(node, source=None, mode='eval', filename=None, lineno=-1): + tree = TemplateASTTransformer().visit(node) if isinstance(filename, unicode): # unicode file names not allowed for code objects filename = filename.encode('utf-8', 'replace') @@ -187,7 +224,12 @@ if lineno <= 0: lineno = 1 - gen = ExpressionCodeGenerator(tree) + if mode == 'eval': + gen = ExpressionCodeGenerator(tree) + name = '' % (repr(source or '?').replace("'", '"')) + else: + gen = ModuleCodeGenerator(tree) + name = '' gen.optimized = True code = gen.getCode() @@ -195,9 +237,8 @@ # clone the code object while adjusting the line number return new.code(0, code.co_nlocals, code.co_stacksize, code.co_flags | 0x0040, code.co_code, code.co_consts, - code.co_names, code.co_varnames, filename, - '' % (repr(source or '?').replace("'", '"')), - lineno, code.co_lnotab, (), ()) + code.co_names, code.co_varnames, filename, name, lineno, + code.co_lnotab, (), ()) BUILTINS = __builtin__.__dict__.copy() BUILTINS['Undefined'] = Undefined @@ -231,7 +272,7 @@ key = key[0] try: return obj[key] - except (KeyError, IndexError, TypeError), e: + except (AttributeError, KeyError, IndexError, TypeError), e: if isinstance(key, basestring): val = getattr(obj, key, _UNDEF) if val is _UNDEF: @@ -249,18 +290,32 @@ _visitors = {} def visit(self, node): + if node is None: + return None v = self._visitors.get(node.__class__) if not v: - v = getattr(self, 'visit%s' % node.__class__.__name__) + v = getattr(self.__class__, 'visit%s' % node.__class__.__name__, + self.__class__._visitDefault) self._visitors[node.__class__] = v - return v(node) + return v(self, node) + def _visitDefault(self, node): + return node + def visitExpression(self, node): node.node = self.visit(node.node) return node - # Functions & Accessors + def visitModule(self, node): + node.node = self.visit(node.node) + return node + def visitStmt(self, node): + node.nodes = [self.visit(x) for x in node.nodes] + return node + + # Classes, Functions & Accessors + def visitCallFunc(self, node): node.node = self.visit(node.node) node.args = [self.visit(x) for x in node.args] @@ -270,20 +325,101 @@ node.dstar_args = self.visit(node.dstar_args) return node - def visitLambda(self, node): + def visitClass(self, node): + node.bases = [self.visit(x) for x in node.bases] node.code = self.visit(node.code) node.filename = '' # workaround for bug in pycodegen return node + def visitFunction(self, node): + if hasattr(node, 'decorators'): + node.decorators = self.visit(node.decorators) + node.defaults = [self.visit(x) for x in node.defaults] + node.code = self.visit(node.code) + node.filename = '' # workaround for bug in pycodegen + return node + def visitGetattr(self, node): node.expr = self.visit(node.expr) return node + def visitLambda(self, node): + node.code = self.visit(node.code) + node.filename = '' # workaround for bug in pycodegen + return node + def visitSubscript(self, node): node.expr = self.visit(node.expr) node.subs = [self.visit(x) for x in node.subs] return node + # Statements + + def visitAssert(self, node): + node.test = self.visit(node.test) + node.fail = self.visit(node.fail) + return node + + def visitAssign(self, node): + node.nodes = [self.visit(x) for x in node.nodes] + node.expr = self.visit(node.expr) + return node + + def visitDecorators(self, node): + node.nodes = [self.visit(x) for x in node.nodes] + return node + + def visitFor(self, node): + node.assign = self.visit(node.assign) + node.list = self.visit(node.list) + node.body = self.visit(node.body) + node.else_ = self.visit(node.else_) + return node + + def visitIf(self, node): + node.tests = [self.visit(x) for x in node.tests] + node.else_ = self.visit(node.else_) + return node + + def _visitPrint(self, node): + node.nodes = [self.visit(x) for x in node.nodes] + node.dest = self.visit(node.dest) + return node + visitPrint = visitPrintnl = _visitPrint + + def visitRaise(self, node): + node.expr1 = self.visit(node.expr1) + node.expr2 = self.visit(node.expr2) + node.expr3 = self.visit(node.expr3) + return node + + def visitTryExcept(self, node): + node.body = self.visit(node.body) + node.handlers = self.visit(node.handlers) + node.else_ = self.visit(node.else_) + return node + + def visitTryFinally(self, node): + node.body = self.visit(node.body) + node.final = self.visit(node.final) + return node + + def visitWhile(self, node): + node.test = self.visit(node.test) + node.body = self.visit(node.body) + node.else_ = self.visit(node.else_) + return node + + def visitWith(self, node): + node.expr = self.visit(node.expr) + node.vars = [self.visit(x) for x in node.vars] + node.body = self.visit(node.body) + return node + + def visitYield(self, node): + node.value = self.visit(node.value) + return node + # Operators def _visitBoolOp(self, node): @@ -308,14 +444,10 @@ node.expr = self.visit(node.expr) return node visitUnaryAdd = visitUnarySub = visitNot = visitInvert = _visitUnaryOp - visitBackquote = _visitUnaryOp + visitBackquote = visitDiscard = _visitUnaryOp # Identifiers, Literals and Comprehensions - def _visitDefault(self, node): - return node - visitAssName = visitConst = visitName = _visitDefault - def visitDict(self, node): node.items = [(self.visit(k), self.visit(v)) for k, v in node.items] @@ -381,9 +513,9 @@ return node -class ExpressionASTTransformer(ASTTransformer): +class TemplateASTTransformer(ASTTransformer): """Concrete AST transformer that implements the AST transformations needed - for template expressions. + for code embedded in templates. """ def __init__(self): @@ -398,9 +530,28 @@ return node def visitAssName(self, node): - self.locals[-1].add(node.name) + if self.locals: + self.locals[-1].add(node.name) return node + def visitClass(self, node): + self.locals.append(set()) + node = ASTTransformer.visitClass(self, node) + self.locals.pop() + return node + + def visitFor(self, node): + self.locals.append(set()) + node = ASTTransformer.visitFor(self, node) + self.locals.pop() + return node + + def visitFunction(self, node): + self.locals.append(set(node.argnames)) + node = ASTTransformer.visitFunction(self, node) + self.locals.pop() + return node + def visitGenExpr(self, node): self.locals.append(set()) node = ASTTransformer.visitGenExpr(self, node)