Index: genshi/filters/i18n.py =================================================================== --- genshi/filters/i18n.py (revision 928) +++ genshi/filters/i18n.py (working copy) @@ -17,12 +17,12 @@ """ from compiler import ast -from gettext import gettext +from gettext import NullTranslations import re from genshi.core import Attrs, Namespace, QName, START, END, TEXT, START_NS, \ END_NS, XML_NAMESPACE, _ensure -from genshi.template.base import Template, EXPR, SUB +from genshi.template.base import Template, EXPR, SUB, INCLUDE from genshi.template.markup import MarkupTemplate, EXEC __all__ = ['Translator', 'extract'] @@ -35,7 +35,7 @@ """Can extract and translate localizable strings from markup streams and templates. - For example, assume the followng template: + For example, assume the following template: >>> from genshi.template import MarkupTemplate >>> @@ -91,7 +91,7 @@ INCLUDE_ATTRS = frozenset(['abbr', 'alt', 'label', 'prompt', 'standby', 'summary', 'title']) - def __init__(self, translate=gettext, ignore_tags=IGNORE_TAGS, + def __init__(self, translator=NullTranslations(), ignore_tags=IGNORE_TAGS, include_attrs=INCLUDE_ATTRS, extract_text=True): """Initialize the translator. @@ -103,10 +103,11 @@ extracted, or only text in explicit ``gettext`` function calls """ - self.translate = translate + self.translator = translator self.ignore_tags = ignore_tags self.include_attrs = include_attrs self.extract_text = extract_text + self.i18n_domains = [] def __call__(self, stream, ctxt=None, search_text=True, msgbuf=None): """Translate any localizable strings in the given stream. @@ -126,11 +127,22 @@ """ ignore_tags = self.ignore_tags include_attrs = self.include_attrs - translate = self.translate + ugettext = self.translator.ugettext + ungettext = self.translator.ungettext + try: + dugettext = self.translator.dugettext + dungettext = self.translator.dungettext + except AttributeError: + # No domain support, show a warning??? + dugettext = lambda d, s: ugettext(s) + dungettext = lambda d, s, p, n: ungettext(s, p, n) + if not self.extract_text: search_text = False skip = 0 i18n_msg = I18N_NAMESPACE['msg'] + i18n_choose = I18N_NAMESPACE['choose'] + i18n_domain = I18N_NAMESPACE['domain'] ns_prefixes = [] xml_lang = XML_NAMESPACE['lang'] @@ -148,23 +160,33 @@ # handle different events that can be localized if kind is START: tag, attrs = data + + if i18n_domain in attrs: + self.i18n_domains.append( + (tag, attrs.get(i18n_domain).strip()) + ) + attrs -= i18n_domain + if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 yield kind, data, pos continue - + new_attrs = [] changed = False for name, value in attrs: newval = value if search_text and isinstance(value, basestring): if name in include_attrs: - newval = self.translate(value) + if self.i18n_domains: + newval = dugettext(self.i18n_domains[-1][1], + value) + else: + newval = ugettext(value) else: newval = list(self(_ensure(value), ctxt, - search_text=False) - ) + search_text=False)) if newval != value: value = newval changed = True @@ -177,10 +199,12 @@ continue elif i18n_msg in attrs: params = attrs.get(i18n_msg) - if params and type(params) is list: # event tuple - params = params[0][1] msgbuf = MessageBuffer(params) attrs -= i18n_msg + elif i18n_choose in attrs: + params = attrs.get(i18n_choose) + msgbuf = MessageBuffer(params) + attrs -= i18n_choose yield kind, (tag, attrs), pos @@ -188,7 +212,12 @@ if not msgbuf: text = data.strip() if text: - data = data.replace(text, unicode(translate(text))) + if self.i18n_domains: + data = data.replace(text, + unicode(dugettext(self.i18n_domains[-1][1], + text))) + else: + data = data.replace(text, unicode(ugettext(text))) yield kind, data, pos else: msgbuf.append(kind, data, pos) @@ -199,11 +228,26 @@ elif not skip and msgbuf and kind is END: msgbuf.append(kind, data, pos) if not msgbuf.depth: - for event in msgbuf.translate(translate(msgbuf.format())): + if msgbuf.singular or msgbuf.plural: + singular, plural, expr = msgbuf.format() + if self.i18n_domains: + events = dungettext(self.i18n_domains[-1][1], + singular, plural, + expr.evaluate(ctxt)) + else: + events = ungettext(singular, plural, + expr.evaluate(ctxt)) + else: + if self.i18n_domains: + events = dugettext(self.i18n_domains[-1][1], + msgbuf.format()) + else: + events = ugettext(msgbuf.format()) + for event in msgbuf.translate(events): yield event msgbuf = None yield kind, data, pos - + elif kind is SUB: subkind, substream = data new_substream = list(self(substream, ctxt, msgbuf=msgbuf)) @@ -214,9 +258,12 @@ elif kind is END_NS and data in ns_prefixes: ns_prefixes.remove(data) - else: yield kind, data, pos + + if kind is END and self.i18n_domains and \ + self.i18n_domains[-1][1] == data: + self.i18n_domains.pop() GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext', 'dgettext', 'dngettext', 'ugettext', 'ungettext') @@ -270,6 +317,8 @@ search_text = False skip = 0 i18n_msg = I18N_NAMESPACE['msg'] + i18n_comment = I18N_NAMESPACE['comment'] + i18n_choose = I18N_NAMESPACE['choose'] xml_lang = XML_NAMESPACE['lang'] for kind, data, pos in stream: @@ -293,33 +342,42 @@ if name in self.include_attrs: text = value.strip() if text: - yield pos[1], None, text + yield pos[1], None, text, [] else: - for lineno, funcname, text in self.extract( + for lineno, funcname, text, comments in self.extract( _ensure(value), gettext_functions, search_text=False): - yield lineno, funcname, text + yield lineno, funcname, text, comments if msgbuf: msgbuf.append(kind, data, pos) elif i18n_msg in attrs: params = attrs.get(i18n_msg) - if params and type(params) is list: # event tuple - params = params[0][1] msgbuf = MessageBuffer(params, pos[1]) + elif i18n_choose in attrs: + params = attrs.get(i18n_choose) + msgbuf = MessageBuffer(params, pos[1]) + if i18n_comment in attrs and msgbuf: + msgbuf.comments.append(attrs.get(i18n_comment)) elif not skip and search_text and kind is TEXT: if not msgbuf: text = data.strip() if text and filter(None, [ch.isalpha() for ch in text]): - yield pos[1], None, text + yield pos[1], None, text, [] else: msgbuf.append(kind, data, pos) elif not skip and msgbuf and kind is END: msgbuf.append(kind, data, pos) if not msgbuf.depth: - yield msgbuf.lineno, None, msgbuf.format() + if msgbuf.singular or msgbuf.plural: + singular, plural, num = msgbuf.format() + yield msgbuf.lineno, 'ngettext', (singular, plural), \ + msgbuf.comments + else: + yield msgbuf.lineno, None, msgbuf.format(), \ + msgbuf.comments msgbuf = None elif kind is EXPR or kind is EXEC: @@ -327,17 +385,17 @@ msgbuf.append(kind, data, pos) for funcname, strings in extract_from_code(data, gettext_functions): - yield pos[1], funcname, strings + yield pos[1], funcname, strings, [] elif kind is SUB: subkind, substream = data messages = self.extract(substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf) - for lineno, funcname, text in messages: - yield lineno, funcname, text - + for lineno, funcname, text, comments in messages: + yield lineno, funcname, text, comments + class MessageBuffer(object): """Helper class for managing internationalized mixed content. @@ -352,14 +410,36 @@ :param lineno: the line number on which the first stream event belonging to the message was found """ - self.params = [name.strip() for name in params.split(',')] + if isinstance(params, (list, tuple)): + self.params = [] + for entry in params: + if entry[0] == 'TEXT': + for param in entry[1].split(','): + if param: + self.params.append(param.strip()) + elif entry[0] == 'EXPR': + self.choose_numeral = entry[1] + elif isinstance(params, basestring): + self.params = [p.strip() for p in params.split(',')] + else: + self.params = params + self.singular_params = self.params[:] + self.plural_params = self.params[:] self.lineno = lineno self.string = [] + self.singular = [] + self.plural = [] self.events = {} self.values = {} self.depth = 1 self.order = 1 + self.choose_order = 1 self.stack = [0] + self.comments = [] + self.i18n_choose_singular = I18N_NAMESPACE['singular'] + self.i18n_choose_plural = I18N_NAMESPACE['plural'] + self.choose_singular = False + self.choose_plural = False def append(self, kind, data, pos): """Append a stream event to the buffer. @@ -369,31 +449,78 @@ :param pos: the position of the event in the source """ if kind is TEXT: - self.string.append(data) + if self.choose_singular: + self.singular.append(data) + if self.choose_plural: + self.plural.append(data) + else: + self.string.append(data) self.events.setdefault(self.stack[-1], []).append(None) elif kind is EXPR: - param = self.params.pop(0) - self.string.append('%%(%s)s' % param) - self.events.setdefault(self.stack[-1], []).append(None) + if self.choose_singular: + param = self.singular_params.pop(0) + self.singular.append('%%(%s)s' % param) + elif self.choose_plural: + param = self.plural_params.pop(0) + self.plural.append('%%(%s)s' % param) + else: + param = self.params.pop(0) + self.string.append('%%(%s)s' % param) self.values[param] = (kind, data, pos) + self.events.setdefault(self.stack[-1], []).append(None) else: if kind is START: - self.string.append(u'[%d:' % self.order) - self.events.setdefault(self.order, []).append((kind, data, pos)) - self.stack.append(self.order) + if self.choose_singular: + self.singular.append(u'[%d:' % self.choose_order) + self.events.setdefault(self.choose_order, + []).append((kind, data, pos)) + self.stack.append(self.choose_order) + elif self.choose_plural: + self.plural.append(u'[%d:' % self.choose_order) + self.events.setdefault(self.choose_order, + []).append((kind, data, pos)) + self.stack.append(self.choose_order) + self.choose_order += 1 + elif self.i18n_choose_singular in data[1]: + self.choose_singular = True + self.choose_plural = False + elif self.i18n_choose_plural in data[1]: + self.choose_plural = True + self.choose_singular = False + else: + self.string.append(u'[%d:' % self.order) + self.events.setdefault(self.order, + []).append((kind, data, pos)) + self.stack.append(self.order) + self.order += 1 self.depth += 1 - self.order += 1 elif kind is END: self.depth -= 1 if self.depth: - self.events[self.stack[-1]].append((kind, data, pos)) - self.string.append(u']') - self.stack.pop() + if self.choose_singular: + self.choose_singular = False + self.events[self.stack[-1]].append((kind, data, pos)) + if self.choose_order > 1: + self.singular.append(u']') + self.stack.pop() + elif self.choose_plural: + self.events[self.stack[-1]].append((kind, data, pos)) + self.choose_plural = False + if self.choose_order > 1: + self.plural.append(u']') + self.stack.pop() + else: + self.string.append(u']') + self.events[self.stack[-1]].append((kind, data, pos)) + self.stack.pop() def format(self): """Return a message identifier representing the content in the buffer. """ + if self.singular or self.plural: + return (u''.join(self.singular).strip(), + u''.join(self.plural).strip(), self.choose_numeral) return u''.join(self.string).strip() def translate(self, string, regex=re.compile(r'%\((\w+)\)s')): @@ -545,6 +672,6 @@ tmpl = template_class(fileobj, filename=getattr(fileobj, 'name', None), encoding=encoding) translator = Translator(None, ignore_tags, include_attrs, extract_text) - for lineno, func, message in translator.extract(tmpl.stream, - gettext_functions=keywords): - yield lineno, func, message, [] + for lineno, func, message, comments in translator.extract( + tmpl.stream, gettext_functions=keywords): + yield lineno, func, message, comments Index: genshi/template/base.py =================================================================== --- genshi/template/base.py (revision 928) +++ genshi/template/base.py (working copy) @@ -560,6 +560,7 @@ template files. """ from genshi.template.loader import TemplateNotFound + from genshi.filters.i18n import Translator for event in stream: if event[0] is INCLUDE: @@ -574,6 +575,12 @@ try: tmpl = self.loader.load(href, relative_to=event[2][0], cls=cls or self.__class__) + if isinstance(tmpl.filters[0], Translator): + # Update xi:include template Translator.i18n_domains + # with the ones found on the template which calls + # this include + tmpl.filters[0].i18n_domains = \ + self.filters[0].i18n_domains for event in tmpl.generate(ctxt, **vars): yield event except TemplateNotFound: