Be explicit about encoding file contents as UTF-8 in unit tests (issue #63)

Author: edschofield

future/tests/base.py

@@ -5,6 +5,7 @@
 import subprocess
 import re
 import warnings
+import io
 if not hasattr(unittest, 'skip'):
     import unittest2 as unittest
 from textwrap import dedent
@@ -172,10 +173,17 @@ def compare(self, output, expected, ignore_imports=True):
 
         If ignore_imports is True, passes the code blocks into the
         strip_future_imports method.
+
+        If one code block is a unicode string and the other a
+        byte-string, it assumes the byte-string is encoded as utf-8.
         """
         if ignore_imports:
             output = self.strip_future_imports(output)
             expected = self.strip_future_imports(expected)
+        if isinstance(output, bytes) and not isinstance(expected, bytes):
+            output = output.decode('utf-8')
+        if isinstance(expected, bytes) and not isinstance(output, bytes):
+            expected = expected.decode('utf-8')
         self.assertEqual(order_future_lines(output.rstrip()),
                          expected.rstrip())
 
@@ -236,7 +244,7 @@ def convert_check(self, before, expected, stages=(1, 2), all_imports=False,
             headers = ''
 
         self.compare(output, reformat_code(headers + expected),
-                    ignore_imports=ignore_imports)
+                     ignore_imports=ignore_imports)
 
     def unchanged(self, code, **kwargs):
         """
@@ -250,11 +258,14 @@ def _write_test_script(self, code, filename='mytestscript.py'):
         Dedents the given code (a multiline string) and writes it out to
         a file in a temporary folder like /tmp/tmpUDCn7x/mytestscript.py.
         """
-        with open(self.tempdir + filename, 'w') as f:
+        if isinstance(code, bytes):
+            code = code.decode('utf-8')
+        # Be explicit about encoding the temp file as UTF-8 (issue #63):
+        with io.open(self.tempdir + filename, 'wt', encoding='utf-8') as f:
             f.write(dedent(code))
 
     def _read_test_script(self, filename='mytestscript.py'):
-        with open(self.tempdir + filename) as f:
+        with io.open(self.tempdir + filename, 'rt', encoding='utf-8') as f:
             newsource = f.read()
         return newsource
 

future/tests/test_requests.py

@@ -9,6 +9,7 @@
 import textwrap
 import sys
 import os
+import io
 
 
 # Don't import requests first. This avoids the problem we want to expose:
@@ -31,7 +32,8 @@ def __init__(self, code, tempdir):
 
     def __enter__(self):
         print('Creating {0}test_imports_future_stdlib.py ...'.format(self.tempdir))
-        with open(self.tempdir + 'test_imports_future_stdlib.py', 'w') as f:
+        with io.open(self.tempdir + 'test_imports_future_stdlib.py', 'wt',
+                     encoding='utf-8') as f:
             f.write(textwrap.dedent(self.code))
         sys.path.insert(0, self.tempdir)
         return self

past/tests/test_translation.py

@@ -10,6 +10,7 @@
 import pprint
 import tempfile
 import os
+import io
 from subprocess import Popen, PIPE
 
 from past import utils
@@ -40,7 +41,10 @@ def test_common_substring(self):
     def write_and_import(self, code, modulename='mymodule'):
         self.assertTrue('.py' not in modulename)
         filename = modulename + '.py'
-        with open(self.tempdir + filename, 'w') as f:
+        if isinstance(code, bytes):
+            code = code.decode('utf-8')
+        # Be explicit about encoding the temp file as UTF-8 (issue #63):
+        with io.open(self.tempdir + filename, 'w', encoding='utf-8') as f:
             f.write(textwrap.dedent(code).strip() + '\n')
 
         # meta_path_len = len(sys.meta_path)