Ticket #485: template-encoding-bugfix.diff

genshi/template/loader.py

@@ -14 +14 @@
 """Template loading and caching."""
 
 import os
+import codecs
 try:
     import threading
 except ImportError:
@@ -21 +22 @@
 
 from genshi.template.base import TemplateError
 from genshi.util import LRUCache
+from genshi.compat import StringIO
 
 __all__ = ['TemplateLoader', 'TemplateNotFound', 'directory', 'package',
            'prefixed']
@@ -120 +122 @@
         """Whether templates should be reloaded when the underlying file is
         changed"""
 
-        self.default_encoding = default_encoding
+        if default_encoding is None:
+            self.default_encoding = "utf-8"
+        else:
+            self.default_encoding = default_encoding
         self.default_class = default_class or MarkupTemplate
         self.variable_lookup = variable_lookup
         self.allow_exec = allow_exec
@@ -183 +188 @@
 
         filename = os.path.normpath(filename)
         cachekey = filename
+        if encoding is None:
+            encoding = self.default_encoding
 
         self._lock.acquire()
         try:
@@ -220 +227 @@
                 if isinstance(loadfunc, basestring):
                     loadfunc = directory(loadfunc)
                 try:
-                    filepath, filename, fileobj, uptodate = loadfunc(filename)
+                    filepath, filename, fileobj, uptodate = loadfunc(filename,
+                                                                     encoding)
                 except IOError:
                     continue
                 else:
@@ -233 +241 @@
                             # search path
                             filename = filepath
                         tmpl = self._instantiate(cls, fileobj, filepath,
-                                                 filename, encoding=encoding)
+                                                 filename)
                         if self.callback:
                             self.callback(tmpl)
                         self._cache[cachekey] = tmpl
@@ -248 +256 @@
         finally:
             self._lock.release()
 
-    def _instantiate(self, cls, fileobj, filepath, filename, encoding=None):
+    def _instantiate(self, cls, fileobj, filepath, filename):
         """Instantiate and return the `Template` object based on the given
         class and parameters.
         
@@ -262 +270 @@
         :param filepath: the absolute path to the template file
         :param filename: the path to the template file relative to the search
                          path
-        :param encoding: the encoding of the template to load; defaults to the
-                         ``default_encoding`` of the loader instance
         :return: the loaded `Template` instance
         :rtype: `Template`
         """
-        if encoding is None:
-            encoding = self.default_encoding
+        # encoding here is always None because the encoding has already
+        # been handled by opening the file using codec.open(...) under
+        # Python 2.x or open(...) under Python 3.x.
         return cls(fileobj, filepath=filepath, filename=filename, loader=self,
-                   encoding=encoding, lookup=self.variable_lookup,
+                   encoding=None, lookup=self.variable_lookup,
                    allow_exec=self.allow_exec)
 
     @staticmethod
@@ -281 +288 @@
         :return: the loader function to load templates from the given directory
         :rtype: ``function``
         """
-        def _load_from_directory(filename):
+        def _load_from_directory(filename, encoding):
             filepath = os.path.join(path, filename)
-            fileobj = open(filepath, 'rU')
+            fileobj = codecs.open(filepath, 'rU', encoding=encoding)
             mtime = os.path.getmtime(filepath)
             def _uptodate():
                 return mtime == os.path.getmtime(filepath)
@@ -300 +307 @@
         :rtype: ``function``
         """
         from pkg_resources import resource_stream
-        def _load_from_package(filename):
+        def _load_from_package(filename, encoding):
             filepath = os.path.join(path, filename)
-            return filepath, filename, resource_stream(name, filepath), None
+            # TODO: read file iteratively
+            bytes_stream = resource_stream(name, filepath)
+            file_stream = StringIO(bytes_stream.read().decode(encoding))
+            return filepath, filename, file_stream, None
         return _load_from_package
 
     @staticmethod
@@ -314 +324 @@
         request to the delegate.
         
         >>> load = prefixed(
-        ...     app1 = lambda filename: ('app1', filename, None, None),
-        ...     app2 = lambda filename: ('app2', filename, None, None)
+        ...     app1 = lambda filename, enc: ('app1', filename, None, None),
+        ...     app2 = lambda filename, enc: ('app2', filename, None, None)
         ... )
-        >>> print(load('app1/foo.html'))
+        >>> print(load('app1/foo.html', 'utf-8'))
         ('app1', 'app1/foo.html', None, None)
-        >>> print(load('app2/bar.html'))
+        >>> print(load('app2/bar.html', 'utf-8'))
         ('app2', 'app2/bar.html', None, None)
         
         :param delegates: mapping of path prefixes to loader functions
         :return: the loader function
         :rtype: ``function``
         """
-        def _dispatch_by_prefix(filename):
+        def _dispatch_by_prefix(filename, encoding):
             for prefix, delegate in delegates.items():
                 if filename.startswith(prefix):
                     if isinstance(delegate, basestring):
                         delegate = directory(delegate)
                     filepath, _, fileobj, uptodate = delegate(
-                        filename[len(prefix):].lstrip('/\\')
+                        filename[len(prefix):].lstrip('/\\'),
+                        encoding
                     )
                     return filepath, filename, fileobj, uptodate
             raise TemplateNotFound(filename, list(delegates.keys()))