Context Navigation

i18n.py

Last change on this file was 711, checked in by cmlenz, 16 years ago
Ported [710] to 0.4.x branch.
Property svn:eol-style set to `native`
File size: 13.9 KB

Line
1	# -- coding: utf-8 --
2	#
3	# Copyright (C) 2007 Edgewall Software
4	# All rights reserved.
5	#
6	# This software is licensed as described in the file COPYING, which
7	# you should have received as part of this distribution. The terms
8	# are also available at http://genshi.edgewall.org/wiki/License.
9	#
10	# This software consists of voluntary contributions made by many
11	# individuals. For the exact contribution history, see the revision
12	# history and logs, available at http://genshi.edgewall.org/log/.
13
14	"""Utilities for internationalization and localization of templates."""
15
16	try:
17	frozenset
18	except NameError:
19	from sets import ImmutableSet as frozenset
20	from gettext import gettext
21	from opcode import opmap
22	import re
23
24	from genshi.core import Attrs, Namespace, QName, START, END, TEXT, \
25	XML_NAMESPACE, _ensure
26	from genshi.template.base import Template, EXPR, SUB
27	from genshi.template.markup import MarkupTemplate, EXEC
28
29	__all__ = ['Translator', 'extract']
30	__docformat__ = 'restructuredtext en'
31
32	_LOAD_NAME = chr(opmap['LOAD_NAME'])
33	_LOAD_CONST = chr(opmap['LOAD_CONST'])
34	_CALL_FUNCTION = chr(opmap['CALL_FUNCTION'])
35	_BINARY_ADD = chr(opmap['BINARY_ADD'])
36
37
38	class Translator(object):
39	"""Can extract and translate localizable strings from markup streams and
40	templates.
41
42	For example, assume the followng template:
43
44	>>> from genshi.template import MarkupTemplate
45	>>>
46	>>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/">
47	... <head>
48	... <title>Example</title>
49	... </head>
50	... <body>
51	... <h1>Example</h1>
52	... <p>${_("Hello, %(name)s") % dict(name=username)}</p>
53	... </body>
54	... </html>''', filename='example.html')
55
56	For demonstration, we define a dummy ``gettext``-style function with a
57	hard-coded translation table, and pass that to the `Translator` initializer:
58
59	>>> def pseudo_gettext(string):
60	... return {
61	... 'Example': 'Beispiel',
62	... 'Hello, %(name)s': 'Hallo, %(name)s'
63	... }[string]
64	>>>
65	>>> translator = Translator(pseudo_gettext)
66
67	Next, the translator needs to be prepended to any already defined filters
68	on the template:
69
70	>>> tmpl.filters.insert(0, translator)
71
72	When generating the template output, our hard-coded translations should be
73	applied as expected:
74
75	>>> print tmpl.generate(username='Hans', _=pseudo_gettext)
76	<html>
77	<head>
78	<title>Beispiel</title>
79	</head>
80	<body>
81	<h1>Beispiel</h1>
82	<p>Hallo, Hans</p>
83	</body>
84	</html>
85
86	Note that elements defining ``xml:lang`` attributes that do not contain
87	variable expressions are ignored by this filter. That can be used to
88	exclude specific parts of a template from being extracted and translated.
89	"""
90
91	IGNORE_TAGS = frozenset([
92	QName('script'), QName('http://www.w3.org/1999/xhtml}script'),
93	QName('style'), QName('http://www.w3.org/1999/xhtml}style')
94	])
95	INCLUDE_ATTRS = frozenset(['abbr', 'alt', 'label', 'prompt', 'standby',
96	'summary', 'title'])
97
98	def __init__(self, translate=gettext, ignore_tags=IGNORE_TAGS,
99	include_attrs=INCLUDE_ATTRS, extract_text=True):
100	"""Initialize the translator.
101
102	:param translate: the translation function, for example ``gettext`` or
103	``ugettext``.
104	:param ignore_tags: a set of tag names that should not be localized
105	:param include_attrs: a set of attribute names should be localized
106	:param extract_text: whether the content of text nodes should be
107	extracted, or only text in explicit ``gettext``
108	function calls
109	"""
110	self.translate = translate
111	self.ignore_tags = ignore_tags
112	self.include_attrs = include_attrs
113	self.extract_text = extract_text
114
115	def __call__(self, stream, ctxt=None, search_text=True):
116	"""Translate any localizable strings in the given stream.
117
118	This function shouldn't be called directly. Instead, an instance of
119	the `Translator` class should be registered as a filter with the
120	`Template` or the `TemplateLoader`, or applied as a regular stream
121	filter. If used as a template filter, it should be inserted in front of
122	all the default filters.
123
124	:param stream: the markup event stream
125	:param ctxt: the template context (not used)
126	:param search_text: whether text nodes should be translated (used
127	internally)
128	:return: the localized stream
129	"""
130	ignore_tags = self.ignore_tags
131	include_attrs = self.include_attrs
132	translate = self.translate
133	if not self.extract_text:
134	search_text = False
135	skip = 0
136	xml_lang = XML_NAMESPACE['lang']
137
138	for kind, data, pos in stream:
139
140	# skip chunks that should not be localized
141	if skip:
142	if kind is START:
143	skip += 1
144	elif kind is END:
145	skip -= 1
146	yield kind, data, pos
147	continue
148
149	# handle different events that can be localized
150	if kind is START:
151	tag, attrs = data
152	if tag in self.ignore_tags or \
153	isinstance(attrs.get(xml_lang), basestring):
154	skip += 1
155	yield kind, data, pos
156	continue
157
158	new_attrs = []
159	changed = False
160	for name, value in attrs:
161	newval = value
162	if search_text and isinstance(value, basestring):
163	if name in include_attrs:
164	newval = self.translate(value)
165	else:
166	newval = list(self(_ensure(value), ctxt,
167	search_text=False)
168	)
169	if newval != value:
170	value = newval
171	changed = True
172	new_attrs.append((name, value))
173	if changed:
174	attrs = new_attrs
175
176	yield kind, (tag, attrs), pos
177
178	elif search_text and kind is TEXT:
179	text = data.strip()
180	if text:
181	data = data.replace(text, translate(text))
182	yield kind, data, pos
183
184	elif kind is SUB:
185	subkind, substream = data
186	new_substream = list(self(substream, ctxt))
187	yield kind, (subkind, new_substream), pos
188
189	else:
190	yield kind, data, pos
191
192	GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext', 'dgettext', 'dngettext',
193	'ugettext', 'ungettext')
194
195	def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS,
196	search_text=True):
197	"""Extract localizable strings from the given template stream.
198
199	For every string found, this function yields a ``(lineno, function,
200	message)`` tuple, where:
201
202	* ``lineno`` is the number of the line on which the string was found,
203	* ``function`` is the name of the ``gettext`` function used (if the
204	string was extracted from embedded Python code), and
205	* ``message`` is the string itself (a ``unicode`` object, or a tuple
206	of ``unicode`` objects for functions with multiple string arguments).
207
208	>>> from genshi.template import MarkupTemplate
209	>>>
210	>>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/">
211	... <head>
212	... <title>Example</title>
213	... </head>
214	... <body>
215	... <h1>Example</h1>
216	... <p>${_("Hello, %(name)s") % dict(name=username)}</p>
217	... <p>${ngettext("You have %d item", "You have %d items", num)}</p>
218	... </body>
219	... </html>''', filename='example.html')
220	>>>
221	>>> for lineno, funcname, message in Translator().extract(tmpl.stream):
222	... print "%d, %r, %r" % (lineno, funcname, message)
223	3, None, u'Example'
224	6, None, u'Example'
225	7, '_', u'Hello, %(name)s'
226	8, 'ngettext', (u'You have %d item', u'You have %d items')
227
228	:param stream: the event stream to extract strings from; can be a
229	regular stream or a template stream
230	:param gettext_functions: a sequence of function names that should be
231	treated as gettext-style localization
232	functions
233	:param search_text: whether the content of text nodes should be
234	extracted (used internally)
235
236	:note: Changed in 0.4.1: For a function with multiple string arguments
237	(such as ``ngettext``), a single item with a tuple of strings is
238	yielded, instead an item for each string argument.
239	"""
240	tagname = None
241	if not self.extract_text:
242	search_text = False
243	skip = 0
244	xml_lang = XML_NAMESPACE['lang']
245
246	for kind, data, pos in stream:
247
248	if skip:
249	if kind is START:
250	skip += 1
251	if kind is END:
252	skip -= 1
253
254	if kind is START and not skip:
255	tag, attrs = data
256	if tag in self.ignore_tags or \
257	isinstance(attrs.get(xml_lang), basestring):
258	skip += 1
259	continue
260
261	for name, value in attrs:
262	if search_text and isinstance(value, basestring):
263	if name in self.include_attrs:
264	text = value.strip()
265	if text:
266	yield pos[1], None, text
267	else:
268	for lineno, funcname, text in self.extract(
269	_ensure(value), gettext_functions,
270	search_text=False):
271	yield lineno, funcname, text
272
273	elif not skip and search_text and kind is TEXT:
274	text = data.strip()
275	if text and filter(None, [ch.isalpha() for ch in text]):
276	yield pos[1], None, text
277
278	elif kind is EXPR or kind is EXEC:
279	consts = dict([(n, chr(i) + '\x00') for i, n in
280	enumerate(data.code.co_consts)])
281	gettext_locs = [consts[n] for n in gettext_functions
282	if n in consts]
283	ops = [
284	_LOAD_CONST, '(', '\|'.join(gettext_locs), ')',
285	_CALL_FUNCTION, '.\x00',
286	'((?:', _BINARY_ADD, '\|', _LOAD_CONST, '.\x00)+)'
287	]
288	for loc, opcodes in re.findall(''.join(ops), data.code.co_code):
289	funcname = data.code.co_consts[ord(loc[0])]
290	strings = []
291	opcodes = iter(opcodes)
292	for opcode in opcodes:
293	if opcode == _BINARY_ADD:
294	arg = strings.pop()
295	strings[-1] += arg
296	else:
297	arg = data.code.co_consts[ord(opcodes.next())]
298	opcodes.next() # skip second byte
299	if not isinstance(arg, basestring):
300	break
301	strings.append(unicode(arg))
302	if len(strings) == 1:
303	strings = strings[0]
304	else:
305	strings = tuple(strings)
306	yield pos[1], funcname, strings
307
308	elif kind is SUB:
309	subkind, substream = data
310	messages = self.extract(substream, gettext_functions,
311	search_text=search_text and not skip)
312	for lineno, funcname, text in messages:
313	yield lineno, funcname, text
314
315
316	def extract(fileobj, keywords, comment_tags, options):
317	"""Babel extraction method for Genshi templates.
318
319	:param fileobj: the file-like object the messages should be extracted from
320	:param keywords: a list of keywords (i.e. function names) that should be
321	recognized as translation functions
322	:param comment_tags: a list of translator tags to search for and include
323	in the results
324	:param options: a dictionary of additional options (optional)
325	:return: an iterator over ``(lineno, funcname, message, comments)`` tuples
326	:rtype: ``iterator``
327	"""
328	template_class = options.get('template_class', MarkupTemplate)
329	if isinstance(template_class, basestring):
330	module, clsname = template_class.split(':', 1)
331	template_class = getattr(__import__(module, {}, {}, [clsname]), clsname)
332	encoding = options.get('encoding', None)
333
334	extract_text = options.get('extract_text', True)
335	if isinstance(extract_text, basestring):
336	extract_text = extract_text.lower() in ('1', 'on', 'yes', 'true')
337
338	ignore_tags = options.get('ignore_tags', Translator.IGNORE_TAGS)
339	if isinstance(ignore_tags, basestring):
340	ignore_tags = ignore_tags.split()
341	ignore_tags = [QName(tag) for tag in ignore_tags]
342
343	include_attrs = options.get('include_attrs', Translator.INCLUDE_ATTRS)
344	if isinstance(include_attrs, basestring):
345	include_attrs = include_attrs.split()
346	include_attrs = [QName(attr) for attr in include_attrs]
347
348	tmpl = template_class(fileobj, filename=getattr(fileobj, 'name', None),
349	encoding=encoding)
350	translator = Translator(None, ignore_tags, include_attrs, extract_text)
351	for lineno, func, message in translator.extract(tmpl.stream,
352	gettext_functions=keywords):
353	yield lineno, func, message, []

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: branches/stable/0.4.x/genshi/filters/i18n.py

Download in other formats: