Edgewall Software

Changeset 1142


Ignore:
Timestamp:
Oct 25, 2010, 12:39:08 AM (13 years ago)
Author:
hodgestar
Message:

add support for python 3 to genshi.template expression evaluator:

  • add support for python 3 AST:
    • AST for raise has changed in Python 3.
    • Python 3 adds AST nodes for individual arguments and Bytes.
  • use genshi.compat functions for dealing with code objects.
  • do not coerce byte strings to unicode in Python 3 ASTTransformer.
  • replace doctests that reply on exception names with uglier but more compatible try:.. except:.. doctest
  • handle filename preferences of Python 2 and 3 (2 prefers bytes, 3 prefers unicode).
  • ifilter is gone from itertools in Python 3 so use repeat for tests instead.
Location:
branches/experimental/py3k/genshi/template
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • branches/experimental/py3k/genshi/template/astutil.py

    r1120 r1142  
    2222        return compile(source, '', mode, _ast.PyCF_ONLY_AST)
    2323
     24from genshi.compat import IS_PYTHON2
    2425
    2526__docformat__ = 'restructuredtext en'
    … …  
    130131            self._write('**' + node.kwarg)
    131132
     133    if not IS_PYTHON2:
     134        # In Python 3 arguments get a special node
     135        def visit_arg(self, node):
     136            self._write(node.arg)
     137
    132138    # FunctionDef(identifier name, arguments args,
    133139    #                           stmt* body, expr* decorator_list)
    … …  
    290296
    291297
    292     # Raise(expr? type, expr? inst, expr? tback)
    293     def visit_Raise(self, node):
    294         self._new_line()
    295         self._write('raise')
    296         if not node.type:
    297             return
    298         self._write(' ')
    299         self.visit(node.type)
    300         if not node.inst:
    301             return
    302         self._write(', ')
    303         self.visit(node.inst)
    304         if not node.tback:
    305             return
    306         self._write(', ')
    307         self.visit(node.tback)
     298    if IS_PYTHON2:
     299        # Raise(expr? type, expr? inst, expr? tback)
     300        def visit_Raise(self, node):
     301            self._new_line()
     302            self._write('raise')
     303            if not node.type:
     304                return
     305            self._write(' ')
     306            self.visit(node.type)
     307            if not node.inst:
     308                return
     309            self._write(', ')
     310            self.visit(node.inst)
     311            if not node.tback:
     312                return
     313            self._write(', ')
     314            self.visit(node.tback)
     315    else:
     316        # Raise(expr? exc from expr? cause)
     317        def visit_Raise(self, node):
     318            self._new_line()
     319            self._write('raise')
     320            if not node.exc:
     321                return
     322            self._write(' ')
     323            self.visit(node.exc)
     324            if not node.cause:
     325                return
     326            self._write(' from ')
     327            self.visit(node.cause)
    308328
    309329    # TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)
    … …  
    626646    def visit_Str(self, node):
    627647        self._write(repr(node.s))
     648
     649    if not IS_PYTHON2:
     650        # Bytes(bytes s)
     651        def visit_Bytes(self, node):
     652            self._write(repr(node.s))
    628653
    629654    # Attribute(expr value, identifier attr, expr_context ctx)

  • branches/experimental/py3k/genshi/template/eval.py

    r1120 r1142  
    2424from genshi.template.base import TemplateRuntimeError
    2525from genshi.util import flatten
     26
     27from genshi.compat import get_code_params, build_code_chunk, IS_PYTHON2
    2628
    2729__all__ = ['Code', 'Expression', 'Suite', 'LenientLookup', 'StrictLookup',
    … …  
    99101        state = {'source': self.source, 'ast': self.ast,
    100102                 'lookup': self._globals.im_self}
    101         c = self.code
    102         state['code'] = (c.co_nlocals, c.co_stacksize, c.co_flags, c.co_code,
    103                          c.co_consts, c.co_names, c.co_varnames, c.co_filename,
    104                          c.co_name, c.co_firstlineno, c.co_lnotab, (), ())
     103        state['code'] = get_code_params(self.code)
    105104        return state
    106105
    … …  
    237236    reference that undefined variable.
    238237   
    239     >>> foo('bar')
    240     Traceback (most recent call last):
    241         ...
    242     UndefinedError: "foo" not defined
    243 
    244     >>> foo.bar
    245     Traceback (most recent call last):
    246         ...
    247     UndefinedError: "foo" not defined
     238    >>> try:
     239    ...     foo('bar')
     240    ... except UndefinedError, e:
     241    ...     print e.msg
     242    "foo" not defined
     243
     244    >>> try:
     245    ...     foo.bar
     246    ... except UndefinedError, e:
     247    ...     print e.msg
     248    "foo" not defined
    248249   
    249250    :see: `LenientLookup`
    … …  
    389390   
    390391    >>> expr = Expression('nothing', lookup='strict')
    391     >>> expr.evaluate({})
    392     Traceback (most recent call last):
    393         ...
    394     UndefinedError: "nothing" not defined
     392    >>> try:
     393    ...     expr.evaluate({})
     394    ... except UndefinedError, e:
     395    ...     print e.msg
     396    "nothing" not defined
    395397   
    396398    The same happens when a non-existing attribute or item is accessed on an
    … …  
    398400   
    399401    >>> expr = Expression('something.nil', lookup='strict')
    400     >>> expr.evaluate({'something': dict()})
    401     Traceback (most recent call last):
    402         ...
    403     UndefinedError: {} has no member named "nil"
     402    >>> try:
     403    ...     expr.evaluate({'something': dict()})
     404    ... except UndefinedError, e:
     405    ...     print e.msg
     406    {} has no member named "nil"
    404407    """
    405408
    … …  
    422425            source = '\n'.join([first, rest])
    423426    if isinstance(source, unicode):
    424         source = '\xef\xbb\xbf' + source.encode('utf-8')
     427        source = (u'\ufeff' + source).encode('utf-8')
    425428    return parse(source, mode)
    426429
    … …  
    428431def _compile(node, source=None, mode='eval', filename=None, lineno=-1,
    429432             xform=None):
    430     if isinstance(filename, unicode):
    431         # unicode file names not allowed for code objects
    432         filename = filename.encode('utf-8', 'replace')
    433     elif not filename:
     433    if not filename:
    434434        filename = '<string>'
     435    if IS_PYTHON2:
     436        # Python 2 requires non-unicode filenames
     437        if isinstance(filename, unicode):
     438            filename = filename.encode('utf-8', 'replace')
     439    else:
     440        # Python 3 requires unicode filenames
     441        if not isinstance(filename, unicode):
     442            filename = filename.decode('utf-8', 'replace')
    435443    if lineno <= 0:
    436444        lineno = 1
    … …  
    459467        # We'd like to just set co_firstlineno, but it's readonly. So we need
    460468        # to clone the code object while adjusting the line number
    461         return CodeType(0, code.co_nlocals, code.co_stacksize,
    462                         code.co_flags | 0x0040, code.co_code, code.co_consts,
    463                         code.co_names, code.co_varnames, filename, name,
    464                         lineno, code.co_lnotab, (), ())
     469        return build_code_chunk(code, filename, name, lineno)
    465470    except RuntimeError:
    466471        return code
    … …  
    494499        names = set()
    495500        def _process(node):
     501            if not IS_PYTHON2 and isinstance(node, _ast.arg):
     502                names.add(node.arg)
    496503            if isinstance(node, _ast.Name):
    497504                names.add(node.id)
    … …  
    514521
    515522    def visit_Str(self, node):
    516         if isinstance(node.s, str):
     523        if not isinstance(node.s, unicode):
    517524            try: # If the string is ASCII, return a `str` object
    518525                node.s.decode('ascii')

  • branches/experimental/py3k/genshi/template/tests/eval.py

    r1120 r1142  
    1515import os
    1616import pickle
    17 from StringIO import StringIO
    1817import sys
    1918from tempfile import mkstemp
    … …  
    2423from genshi.template.eval import Expression, Suite, Undefined, UndefinedError, \
    2524                                 UNDEFINED
     25from genshi.compat import BytesIO, IS_PYTHON2, wrapped_bytes
    2626
    2727
    … …  
    4040    def test_pickle(self):
    4141        expr = Expression('1 < 2')
    42         buf = StringIO()
     42        buf = BytesIO()
    4343        pickle.dump(expr, buf, 2)
    4444        buf.seek(0)
    … …  
    5959        self.assertEqual('foo', Expression('"foo"').evaluate({}))
    6060        self.assertEqual('foo', Expression('"""foo"""').evaluate({}))
    61         self.assertEqual('foo', Expression("'foo'").evaluate({}))
     61        self.assertEqual(u'foo'.encode('utf-8'),
     62                         Expression(wrapped_bytes("b'foo'")).evaluate({}))
    6263        self.assertEqual('foo', Expression("'''foo'''").evaluate({}))
    6364        self.assertEqual('foo', Expression("u'foo'").evaluate({}))
    … …  
    6970        expr = Expression("u'\xfe'")
    7071        self.assertEqual(u'þ', expr.evaluate({}))
    71         expr = Expression("'\xc3\xbe'")
    72         self.assertEqual(u'þ', expr.evaluate({}))
     72        # On Python2 strings are converted to unicode if they contained
     73        # non-ASCII characters.
     74        # On Py3k, we have no need to do this as non-prefixed strings aren't
     75        # raw.
     76        expr = Expression(wrapped_bytes(r"b'\xc3\xbe'"))
     77        if IS_PYTHON2:
     78            self.assertEqual(u'þ', expr.evaluate({}))
     79        else:
     80            self.assertEqual(u'þ'.encode('utf-8'), expr.evaluate({}))
    7381
    7482    def test_num_literal(self):
    7583        self.assertEqual(42, Expression("42").evaluate({}))
    76         self.assertEqual(42L, Expression("42L").evaluate({}))
     84        if IS_PYTHON2:
     85            self.assertEqual(42L, Expression("42L").evaluate({}))
    7786        self.assertEqual(.42, Expression(".42").evaluate({}))
    78         self.assertEqual(07, Expression("07").evaluate({}))
     87        if IS_PYTHON2:
     88            self.assertEqual(07, Expression("07").evaluate({}))
    7989        self.assertEqual(0xF2, Expression("0xF2").evaluate({}))
    8090        self.assertEqual(0XF2, Expression("0XF2").evaluate({}))
    … …  
    247257        data = {'items': range(5)}
    248258        expr = Expression("filter(lambda x: x > 2, items)")
    249         self.assertEqual([3, 4], expr.evaluate(data))
     259        self.assertEqual([3, 4], list(expr.evaluate(data)))
    250260
    251261    def test_lambda_tuple_arg(self):
     262        # This syntax goes away in Python 3
     263        if not IS_PYTHON2:
     264            return
    252265        data = {'items': [(1, 2), (2, 1)]}
    253266        expr = Expression("filter(lambda (x, y): x > y, items)")
    254         self.assertEqual([(2, 1)], expr.evaluate(data))
     267        self.assertEqual([(2, 1)], list(expr.evaluate(data)))
    255268
    256269    def test_list_comprehension(self):
    … …  
    471484    def test_pickle(self):
    472485        suite = Suite('foo = 42')
    473         buf = StringIO()
     486        buf = BytesIO()
    474487        pickle.dump(suite, buf, 2)
    475488        buf.seek(0)
    … …  
    646659
    647660    def test_import(self):
    648         suite = Suite("from itertools import ifilter")
    649         data = {}
    650         suite.execute(data)
    651         assert 'ifilter' in data
     661        suite = Suite("from itertools import repeat")
     662        data = {}
     663        suite.execute(data)
     664        assert 'repeat' in data
    652665
    653666    def test_import_star(self):
    … …  
    655668        data = Context()
    656669        suite.execute(data)
    657         assert 'ifilter' in data
     670        assert 'repeat' in data
    658671
    659672    def test_import_in_def(self):
    660673        suite = Suite("""def fun():
    661     from itertools import ifilter
    662     return ifilter(None, range(3))
     674    from itertools import repeat
     675    return repeat(1, 3)
    663676""")
    664677        data = Context()
    665678        suite.execute(data)
    666         assert 'ifilter' not in data
    667         self.assertEqual([1, 2], list(data['fun']()))
     679        assert 'repeat' not in data
     680        self.assertEqual([1, 1, 1], list(data['fun']()))
    668681
    669682    def test_for(self):
    … …  
    767780
    768781    def test_exec(self):
    769         suite = Suite("x = 1; exec d['k']; assert x == 42, x")
     782        suite = Suite("x = 1; exec(d['k']); assert x == 42, x")
    770783        suite.execute({"d": {"k": "x = 42"}})
    771784
    … …  
    829842        def test_yield_expression(self):
    830843            d = {}
    831             suite = Suite("""results = []
     844            suite = Suite("""from genshi.compat import next
     845results = []
    832846def counter(maximum):
    833847    i = 0
    … …  
    839853            i += 1
    840854it = counter(5)
    841 results.append(it.next())
     855results.append(next(it))
    842856results.append(it.send(3))
    843 results.append(it.next())
     857results.append(next(it))
    844858""")
    845859            suite.execute(d)
Note: See TracChangeset for help on using the changeset viewer.