diff --git a/genshi/filters/i18n.py b/genshi/filters/i18n.py --- a/genshi/filters/i18n.py +++ b/genshi/filters/i18n.py @@ -19,10 +19,14 @@ """ from gettext import NullTranslations -import os import re from types import FunctionType +try: + from babel.messages.catalog import PYTHON_FORMAT +except ImportError: + PYTHON_FORMAT = None + from genshi.core import Attrs, Namespace, QName, START, END, TEXT, START_NS, \ END_NS, XML_NAMESPACE, _ensure, StreamEventKind from genshi.template.eval import _ast @@ -68,7 +72,7 @@ >>> translator = Translator() >>> translator.setup(tmpl) >>> list(translator.extract(tmpl.stream)) - [(2, None, u'Foo', [u'As in Foo Bar'])] + [(2, None, u'Foo', {'comments': [u'As in Foo Bar']})] >>> """ @@ -95,8 +99,11 @@ >>> translator = Translator() >>> translator.setup(tmpl) - >>> list(translator.extract(tmpl.stream)) - [(2, None, u'[1:Foo]\n [2:Bar]', []), (6, None, u'Foo [1:bar]!', [])] + >>> list(translator.extract(tmpl.stream)) #doctest: +NORMALIZE_WHITESPACE + [(2, None, u'[1:Foo]\n [2:Bar]', {'flags': ['genshi-format'], + 'comments': []}), + (6, None, u'Foo [1:bar]!', {'flags': ['genshi-format'], + 'comments': []})] >>> print tmpl.generate().render()

Foo

@@ -113,8 +120,9 @@ ... ''') >>> translator.setup(tmpl) >>> list(translator.extract(tmpl.stream)) #doctest: +NORMALIZE_WHITESPACE - [(2, None, u'[1:First Name: %(fname)s]\n [2:Last Name: %(lname)s]', []), - (6, None, u'Foo [1:bar]!', [])] + [(2, None, u'[1:First Name: %(fname)s]\n [2:Last Name: %(lname)s]', + {'flags': ['genshi-format', 'python-format'], 'comments': []}), + (6, None, u'Foo [1:bar]!', {'flags': ['genshi-format'], 'comments': []})] >>> >>> tmpl = MarkupTemplate(''' ...
@@ -193,8 +201,17 @@ msgbuf.append(*previous) if msgbuf.valid: - yield (None, msgbuf.format(), - filter(None, [ctxt.get('_i18n.comment')])) + message = msgbuf.format() + info = { + 'comments': filter(None, [ctxt.get('_i18n.comment')]) + } + if len(parse_msg(message)) > 1: + info['flags'] = ['genshi-format'] + if PYTHON_FORMAT and bool(filter(None, + [PYTHON_FORMAT.search(id) for id in + [message] if id])): + info.setdefault('flags', []).append('python-format') + yield None, message, info class InnerChooseDirective(I18NDirective): @@ -264,7 +281,8 @@ >>> translator.setup(tmpl) >>> list(translator.extract(tmpl.stream)) #doctest: +NORMALIZE_WHITESPACE [(2, 'ngettext', (u'There is %(num)s coin', - u'There are %(num)s coins'), [])] + u'There are %(num)s coins'), + {'flags': ['python-format'], 'comments': []})] >>> >>> tmpl = MarkupTemplate('''\ @@ -299,7 +317,8 @@ >>> translator.setup(tmpl) >>> list(translator.extract(tmpl.stream)) #doctest: +NORMALIZE_WHITESPACE [(2, 'ngettext', (u'There is %(num)s coin', - u'There are %(num)s coins'), [])] + u'There are %(num)s coins'), + {'flags': ['python-format'], 'comments': []})] >>> """ @@ -410,9 +429,16 @@ singular_msgbuf.append(kind, event, pos) plural_msgbuf.append(kind, event, pos) if singular_msgbuf.valid and plural_msgbuf.valid: - yield 'ngettext', \ - (singular_msgbuf.format(), plural_msgbuf.format()), \ - filter(None, [ctxt.get('_i18n.comment')]) + singular = singular_msgbuf.format() + plural = plural_msgbuf.format() + info = {'comments': filter(None, [ctxt.get('_i18n.comment')])} + if (len(parse_msg(singular)) or len(parse_msg(plural))) > 1: + info.setdefault('flags', []).append('genshi-format') + if PYTHON_FORMAT and bool(filter(None, + [PYTHON_FORMAT.search(id) for id in + [singular, plural] if id])): + info.setdefault('flags', []).append('python-format') + yield 'ngettext', (singular, plural), info class DomainDirective(I18NDirective): @@ -728,16 +754,25 @@ """Extract localizable strings from the given template stream. For every string found, this function yields a ``(lineno, function, - message, comments)`` tuple, where: + message, info)`` tuple, where: * ``lineno`` is the number of the line on which the string was found, * ``function`` is the name of the ``gettext`` function used (if the string was extracted from embedded Python code), and - * ``message`` is the string itself (a ``unicode`` object, or a tuple - of ``unicode`` objects for functions with multiple string - arguments). - * ``comments`` is a list of comments related to the message, extracted - from ``i18n:comment`` attributes found in the markup + * ``message`` is the string itself (a ``unicode`` object, or a tuple + of ``unicode`` objects for functions with multiple string + arguments). + * `info`` is a dictionary that contains additional information about the + message being extracted like: + `comments`: A list of comments embedded in the source code; + With genshi you define the comments to be extracted + from the ``i18n:comment`` attributes found in the + markup; + `context`: A string containing the message context; Not yet used + in genshi. + `flags`: A list of flags used on the message, ie, + ``python-format``, etc, or even a custom flag, + ``genshi-format`` for genshi nested markup; >>> from genshi.template import MarkupTemplate >>> @@ -752,12 +787,14 @@ ... ... ''', filename='example.html') >>> - >>> for line, func, msg, comments in Translator().extract(tmpl.stream): - ... print "%d, %r, %r" % (line, func, msg) - 3, None, u'Example' - 6, None, u'Example' - 7, '_', u'Hello, %(name)s' - 8, 'ngettext', (u'You have %d item', u'You have %d items', None) + >>> for line, func, msg, info in Translator().extract(tmpl.stream): + ... print "%d, %r, %r, %r" % ( + ... line, func,msg, info) #doctest: +NORMALIZE_WHITESPACE + 3, None, u'Example', {'comments': []} + 6, None, u'Example', {'comments': []} + 7, '_', u'Hello, %(name)s', {'flags': ['python-format'], 'comments': []} + 8, 'ngettext', (u'You have %d item', u'You have %d items', None), + {'flags': ['python-format'], 'comments': []} :param stream: the event stream to extract strings from; can be a regular stream or a template stream @@ -807,13 +844,13 @@ text = value.strip() if text: # XXX: Do we need to grab i18n:comment from ctxt ??? - yield pos[1], None, text, [] + yield pos[1], None, text, {'comments': []} else: - for lineno, funcname, text, comments in self.extract( - _ensure(value), gettext_functions, - search_text=False, - error_callback=error_callback): - yield lineno, funcname, text, comments + for lineno, funcname, text, info in \ + self.extract(_ensure(value), gettext_functions, + search_text=False, + error_callback=error_callback): + yield lineno, funcname, text, info if msgbuf: msgbuf.append(kind, data, pos) @@ -822,25 +859,41 @@ if not msgbuf: text = data.strip() if text and filter(None, [ch.isalpha() for ch in text]): - yield pos[1], None, text, \ - filter(None, [ctxt.get('_i18n.comment')]) + yield pos[1], None, text, { + 'comments': filter(None, + [ctxt.get('_i18n.comment')])} else: msgbuf.append(kind, data, pos) elif not skip and msgbuf and kind is END: msgbuf.append(kind, data, pos) if not msgbuf.depth and msgbuf.valid: - yield msgbuf.lineno, None, msgbuf.format(), \ - filter(None, [msgbuf.comment]) + message = msgbuf.format() + info = {'comments': filter(None, [msgbuf.comment])} + if len(message) > 1: + singular, plural = message + if (len(parse_msg(singular)) or + len(parse_msg(plural))) > 1: + info.setdefault('flags', []).append('genshi-format') + if PYTHON_FORMAT and bool(filter( + None, [PYTHON_FORMAT.search(id) + for id in [message] if id])): + info.setdefault('flags', []).append('python-format') + elif len(parse_msg(message)) > 1: + info.setdefault('flags', []).append('genshi-format') + if PYTHON_FORMAT and bool(filter( + None, [PYTHON_FORMAT.search(id) + for id in [message] if id])): + info.setdefault('flags', []).append('python-format') + yield msgbuf.lineno, None, message, info msgbuf = None elif kind is EXPR or kind is EXEC: if msgbuf: msgbuf.append(kind, data, pos) - for funcname, strings in extract_from_code(data, - gettext_functions): - # XXX: Do we need to grab i18n:comment from ctxt ??? - yield pos[1], funcname, strings, [] + for funcname, strings, info in extract_from_code( + data, gettext_functions): + yield pos[1], funcname, strings, info elif kind is SUB: directives, substream = data @@ -860,8 +913,8 @@ search_text=search_text and not skip, msgbuf=msgbuf, ctxt=ctxt, error_callback=error_callback) - for lineno, funcname, text, comments in messages: - yield lineno, funcname, text, comments + for lineno, funcname, text, info in messages: + yield lineno, funcname, text, info directives.pop(idx) elif not isinstance(directive, I18NDirective): # Remove all other non i18n directives from the process @@ -874,23 +927,23 @@ messages = self.extract( substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf) - for lineno, funcname, text, comments in messages: - yield lineno, funcname, text, comments + for lineno, funcname, text, info in messages: + yield lineno, funcname, text, info for directive in directives: if isinstance(directive, I18NDirectiveExtract): messages = directive.extract( substream, ctxt, error_callback=error_callback ) - for funcname, text, comments in messages: - yield pos[1], funcname, text, comments + for funcname, text, info in messages: + yield pos[1], funcname, text, info else: messages = self.extract( substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf, error_callback=error_callback) - for lineno, funcname, text, comments in messages: - yield lineno, funcname, text, comments + for lineno, funcname, text, info in messages: + yield lineno, funcname, text, info if comment: ctxt.pop() @@ -1150,12 +1203,14 @@ >>> expr = Expression('_("Hello")') >>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS)) - [('_', u'Hello')] + [('_', u'Hello', {'comments': []})] >>> expr = Expression('ngettext("You have %(num)s item", ' ... '"You have %(num)s items", num)') - >>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS)) - [('ngettext', (u'You have %(num)s item', u'You have %(num)s items', None))] + >>> list(extract_from_code( + ... expr, Translator.GETTEXT_FUNCTIONS)) #doctest: +NORMALIZE_WHITESPACE + [('ngettext', (u'You have %(num)s item', u'You have %(num)s items', None), + {'flags': ['python-format'], 'comments': []})] :param code: the `Code` object :type code: `genshi.template.eval.Code` @@ -1174,11 +1229,20 @@ [_add(arg) for arg in node.args] _add(node.starargs) _add(node.kwargs) + info = {'comments': []} if len(strings) == 1: strings = strings[0] + if PYTHON_FORMAT and bool(filter(None, [PYTHON_FORMAT.search(id) + for id in [strings] + if id])): + info.setdefault('flags', []).append('python-format') else: strings = tuple(strings) - yield node.func.id, strings + if PYTHON_FORMAT and bool(filter(None, [PYTHON_FORMAT.search(id) + for id in strings + if id])): + info.setdefault('flags', []).append('python-format') + yield node.func.id, strings, info elif node._fields: children = [] for field in node._fields: @@ -1189,21 +1253,41 @@ elif isinstance(child, _ast.AST): children.append(child) for child in children: - for funcname, strings in _walk(child): - yield funcname, strings + for funcname, strings, info in _walk(child): + yield funcname, strings, info return _walk(code.ast) -def extract(fileobj, keywords, comment_tags, options): +def extract(fileobj, keywords, comment_tags, options, error_callback): """Babel extraction method for Genshi templates. + This function generates tuples of the form: + + ``(lineno, funcname, message, info)`` + + * `info`` is a dictionary that contains additional information about the + message being extracted like: + `comments`: A list of comments embedded in the source code; + With genshi you define the comments to be extracted + from the ``i18n:comment`` attributes found in the + markup; + `context`: A string containing the message context; Not yet used + in genshi. + `flags`: A list of flags used on the message, ie, + ``python-format``, etc, or even a custom flag, + ``genshi-format`` for genshi nested markup; + :param fileobj: the file-like object the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognized as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) - :return: an iterator over ``(lineno, funcname, message, comments)`` tuples + :param error_callback: an error function called if defined, and, in case + of errors, to send them to the user. + Accepts three arguments; + ``filename``, ``line_number``, ``error_message`` + :return: an iterator over ``(lineno, funcname, message, info)`` tuples :rtype: ``iterator`` """ template_class = options.get('template_class', MarkupTemplate) @@ -1229,14 +1313,12 @@ tmpl = template_class(fileobj, filename=getattr(fileobj, 'name', None), encoding=encoding) - error_callback = options.get('error_callback') - translator = Translator(None, ignore_tags, include_attrs, extract_text) if hasattr(tmpl, 'add_directives'): tmpl.add_directives(Translator.NAMESPACE, translator) - for message in translator.extract(tmpl.stream, gettext_functions=keywords, - error_callback=error_callback): - yield message + for lineno, funcname, messages, info in translator.extract( + tmpl.stream, gettext_functions=keywords, error_callback=error_callback): + yield lineno, funcname, messages, info def setup_i18n(tmpl, translator): diff --git a/genshi/filters/tests/i18n.py b/genshi/filters/tests/i18n.py --- a/genshi/filters/tests/i18n.py +++ b/genshi/filters/tests/i18n.py @@ -88,7 +88,8 @@ translator = Translator(extract_text=False) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((3, 'ngettext', (u'Singular', u'Plural', None), []), + self.assertEqual((3, 'ngettext', (u'Singular', u'Plural', None), + {'comments': []}), messages[0]) def test_extract_plural_form(self): @@ -98,7 +99,8 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, 'ngettext', (u'Singular', u'Plural', None), []), + self.assertEqual((2, 'ngettext', (u'Singular', u'Plural', None), + {'comments': []}), messages[0]) def test_extract_funky_plural_form(self): @@ -108,7 +110,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, 'ngettext', (None, None), []), messages[0]) + self.assertEqual((2, 'ngettext', (None, None), {'comments': []}), messages[0]) def test_extract_gettext_with_unicode_string(self): tmpl = MarkupTemplate(""" @@ -117,7 +119,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, 'gettext', u'Gr\xfc\xdfe', []), messages[0]) + self.assertEqual((2, 'gettext', u'Gr\xfc\xdfe', {'comments': []}), messages[0]) def test_extract_included_attribute_text(self): tmpl = MarkupTemplate(""" @@ -126,7 +128,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, None, u'Foo', []), messages[0]) + self.assertEqual((2, None, u'Foo', {'comments': []}), messages[0]) def test_extract_attribute_expr(self): tmpl = MarkupTemplate(""" @@ -135,7 +137,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, '_', u'Save', []), messages[0]) + self.assertEqual((2, '_', u'Save', {'comments': []}), messages[0]) def test_extract_non_included_attribute_interpolated(self): tmpl = MarkupTemplate(""" @@ -144,7 +146,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, None, u'Foo', []), messages[0]) + self.assertEqual((2, None, u'Foo', {'comments': []}), messages[0]) def test_extract_text_from_sub(self): tmpl = MarkupTemplate(""" @@ -153,7 +155,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, None, u'Foo', []), messages[0]) + self.assertEqual((2, None, u'Foo', {'comments': []}), messages[0]) def test_ignore_tag_with_fixed_xml_lang(self): tmpl = MarkupTemplate(""" @@ -170,7 +172,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, None, u'(c) 2007 Edgewall Software', []), + self.assertEqual((2, None, u'(c) 2007 Edgewall Software', {'comments': []}), messages[0]) def test_ignore_attribute_with_expression(self): @@ -468,7 +470,8 @@ tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((3, None, u'Foo', ['As in foo bar']), messages[0]) + self.assertEqual((3, None, u'Foo', {'comments': [u'As in foo bar']}), + messages[0]) tmpl = MarkupTemplate("""

Foo

@@ -477,7 +480,8 @@ tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((3, None, u'Foo', ['As in foo bar']), messages[0]) + self.assertEqual((3, None, u'Foo', {'comments': [u'As in foo bar']}), + messages[0]) def test_translate_i18n_msg_with_comment(self): tmpl = MarkupTemplate("""${ngettext("You have %d item", "You have %d items", num)}

""") - results = list(extract(buf, ['_', 'ngettext'], [], {})) + results = list(extract(buf, ['_', 'ngettext'], [], {}, None)) self.assertEqual([ - (3, None, u'Example', []), - (6, None, u'Example', []), - (7, '_', u'Hello, %(name)s', []), + (3, None, u'Example', {'comments': []}), + (6, None, u'Example', {'comments': []}), + (7, '_', u'Hello, %(name)s', {'flags': ['python-format'], 'comments': []}), (8, 'ngettext', (u'You have %d item', u'You have %d items', None), - []), + {'flags': ['python-format'], 'comments': []}), ], results) def test_extraction_without_text(self): @@ -1488,9 +1504,9 @@ """) results = list(extract(buf, ['_', 'ngettext'], [], { 'extract_text': 'no' - })) + }, None)) self.assertEqual([ - (3, 'ngettext', (u'Singular', u'Plural', None), []), + (3, 'ngettext', (u'Singular', u'Plural', None), {'comments': []}), ], results) def test_text_template_extraction(self): @@ -1505,29 +1521,30 @@ Foobar""") results = list(extract(buf, ['_', 'ngettext'], [], { 'template_class': 'genshi.template:TextTemplate' - })) + }, None)) self.assertEqual([ - (1, '_', u'Dear %(name)s', []), - (3, 'ngettext', (u'Your item:', u'Your items', None), []), - (7, None, u'All the best,\n Foobar', []) + (1, '_', u'Dear %(name)s', {'flags': ['python-format'], 'comments': []}), + (3, 'ngettext', (u'Your item:', u'Your items', None), + {'comments': []}), + (7, None, u'All the best,\n Foobar', {'comments': []}) ], results) def test_extraction_with_keyword_arg(self): buf = StringIO(""" ${gettext('Foobar', foo='bar')} """) - results = list(extract(buf, ['gettext'], [], {})) + results = list(extract(buf, ['gettext'], [], {}, None)) self.assertEqual([ - (2, 'gettext', (u'Foobar'), []), + (2, 'gettext', (u'Foobar'), {'comments': []}), ], results) def test_extraction_with_nonstring_arg(self): buf = StringIO(""" ${dgettext(curdomain, 'Foobar')} """) - results = list(extract(buf, ['dgettext'], [], {})) + results = list(extract(buf, ['dgettext'], [], {}, None)) self.assertEqual([ - (2, 'dgettext', (None, u'Foobar'), []), + (2, 'dgettext', (None, u'Foobar'), {'comments': []}), ], results) def test_extraction_inside_ignored_tags(self): @@ -1539,10 +1556,9 @@ }); """) - results = list(extract(buf, ['_'], [], {})) - self.assertEqual([ - (5, '_', u'Please wait...', []), - ], results) + results = list(extract(buf, ['_'], [], {}, None)) + self.assertEqual([(5, '_', u'Please wait...', {'comments': []})], + results) def test_extraction_inside_ignored_tags_with_directives(self): buf = StringIO(""" @@ -1552,7 +1568,7 @@ """) - self.assertEqual([], list(extract(buf, ['_'], [], {}))) + self.assertEqual([], list(extract(buf, ['_'], [], {}, None))) def test_extract_py_def_directive_with_py_strip(self): # Failed extraction from Trac @@ -1597,16 +1613,17 @@ messages = list(translator.extract(tmpl.stream)) self.assertEqual(10, len(messages)) self.assertEqual([ - (3, None, u'View differences', []), - (6, None, u'inline', []), - (8, None, u'side by side', []), - (10, None, u'Show', []), - (13, None, u'lines around each change', []), - (16, None, u'Ignore:', []), - (20, None, u'Blank lines', []), - (25, None, u'Case changes',[]), - (30, None, u'White space changes', []), - (34, '_', u'Update', [])], messages) + (3, None, u'View differences', {'comments': []}), + (6, None, u'inline', {'comments': []}), + (8, None, u'side by side', {'comments': []}), + (10, None, u'Show', {'comments': []}), + (13, None, u'lines around each change', {'comments': []}), + (16, None, u'Ignore:', {'comments': []}), + (20, None, u'Blank lines', {'comments': []}), + (25, None, u'Case changes', {'comments': []}), + (30, None, u'White space changes', {'comments': []}), + (34, '_', u'Update', {'comments': []}) + ], messages) def suite():