Changeset 1189 for trunk/genshi/input.py

Timestamp:

Dec 29, 2012, 2:02:20 PM (11 years ago)

Author:

hodgestar

Message:

Fix parsing of multi-byte characters that occur on 4K boundaries of HTML files (fixes #538).

File:

• trunk/genshi/input.py (modified) (5 diffs)

trunk/genshi/input.py

@@ -17 +17 @@
 
 from itertools import chain
+import codecs
 import htmlentitydefs as entities
 import HTMLParser as html
@@ -318 +319 @@
         """
         def _generate():
+            if self.encoding:
+                reader = codecs.getreader(self.encoding)
+                source = reader(self.source)
+            else:
+                source = self.source
             try:
                 bufsize = 4 * 1024 # 4K
@@ -323 +329 @@
                 while 1:
                     while not done and len(self._queue) == 0:
-                        data = self.source.read(bufsize)
+                        data = source.read(bufsize)
                         if not data: # end of data
                             self.close()
@@ -329 +335 @@
                         else:
                             if not isinstance(data, unicode):
-                                # bytes
-                                if self.encoding:
-                                    data = data.decode(self.encoding)
-                                else:
-                                    raise UnicodeError("source returned bytes, but no encoding specified")
+                                raise UnicodeError("source returned bytes, but no encoding specified")
                             self.feed(data)
                     for kind, data, pos in self._queue:
@@ -433 +435 @@
     """
     if isinstance(text, unicode):
-        return Stream(list(HTMLParser(StringIO(text), encoding=encoding)))
+        # If it's unicode text the encoding should be set to None.
+        # The option to pass in an incorrect encoding is for ease
+        # of writing doctests that work in both Python 2.x and 3.x.
+        return Stream(list(HTMLParser(StringIO(text), encoding=None)))
     return Stream(list(HTMLParser(BytesIO(text), encoding=encoding)))