Merge pull request RustPython#3954 from oow214/unicode_escape_decode

Fix escape codecs

Author: youknowone

Lib/_pycodecs.py

@@ -93,10 +93,10 @@ def escape_encode( obj, errors='strict'):
         v = v.replace(b"'", b"\\'").replace(b'\\"', b'"')
     return v, len(obj)
 
-def raw_unicode_escape_decode( data, errors='strict'):
+def raw_unicode_escape_decode( data, errors='strict', final=False):
     """None
     """
-    res = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors)
+    res = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final)
     res = ''.join(res)
     return res, len(data)
 
@@ -131,10 +131,10 @@ def utf_16_decode( data, errors='strict', final=False):
     res = ''.join(res)
     return res, consumed
 
-def unicode_escape_decode( data, errors='strict'):
+def unicode_escape_decode( data, errors='strict', final=False):
     """None
     """
-    res = PyUnicode_DecodeUnicodeEscape(data, len(data), errors)
+    res = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final)
     res = ''.join(res)
     return res, len(data)
 
@@ -948,7 +948,7 @@ def hexescape(s, pos, digits, message, errors):
     res = p
     return res, pos
 
-def PyUnicode_DecodeUnicodeEscape(s, size, errors):
+def PyUnicode_DecodeUnicodeEscape(s, size, errors, final):
 
     if (size == 0):
         return ''
@@ -1159,7 +1159,7 @@ def PyUnicode_DecodeCharmap(s, size, mapping, errors):
         inpos += 1
     return p
 
-def PyUnicode_DecodeRawUnicodeEscape(s, size, errors):
+def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final):
 
     if (size == 0):
         return ''

Lib/encodings/__init__.py

@@ -61,7 +61,8 @@ def normalize_encoding(encoding):
         if c.isalnum() or c == '.':
             if punct and chars:
                 chars.append('_')
-            chars.append(c)
+            if c.isascii():
+                chars.append(c)
             punct = False
         else:
             punct = True

Lib/encodings/aliases.py

@@ -450,6 +450,7 @@
 
     # mac_latin2 codec
     'maccentraleurope'   : 'mac_latin2',
+    'mac_centeuro'       : 'mac_latin2',
     'maclatin2'          : 'mac_latin2',
 
     # mac_roman codec
@@ -493,9 +494,6 @@
     'sjisx0213'          : 'shift_jisx0213',
     's_jisx0213'         : 'shift_jisx0213',
 
-    # tactis codec
-    'tis260'             : 'tactis',
-
     # tis_620 codec
     'tis620'             : 'tis_620',
     'tis_620_0'          : 'tis_620',

Lib/encodings/raw_unicode_escape.py

@@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.IncrementalEncoder):
     def encode(self, input, final=False):
         return codecs.raw_unicode_escape_encode(input, self.errors)[0]
 
-class IncrementalDecoder(codecs.IncrementalDecoder):
-    def decode(self, input, final=False):
-        return codecs.raw_unicode_escape_decode(input, self.errors)[0]
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+    def _buffer_decode(self, input, errors, final):
+        return codecs.raw_unicode_escape_decode(input, errors, final)
 
 class StreamWriter(Codec,codecs.StreamWriter):
     pass
 
 class StreamReader(Codec,codecs.StreamReader):
-    pass
+    def decode(self, input, errors='strict'):
+        return codecs.raw_unicode_escape_decode(input, errors, False)
 
 ### encodings module API
 

Lib/encodings/unicode_escape.py

@@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.IncrementalEncoder):
     def encode(self, input, final=False):
         return codecs.unicode_escape_encode(input, self.errors)[0]
 
-class IncrementalDecoder(codecs.IncrementalDecoder):
-    def decode(self, input, final=False):
-        return codecs.unicode_escape_decode(input, self.errors)[0]
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+    def _buffer_decode(self, input, errors, final):
+        return codecs.unicode_escape_decode(input, errors, final)
 
 class StreamWriter(Codec,codecs.StreamWriter):
     pass
 
 class StreamReader(Codec,codecs.StreamReader):
-    pass
+    def decode(self, input, errors='strict'):
+        return codecs.unicode_escape_decode(input, errors, False)
 
 ### encodings module API