Merge pull request RustPython#4781 from MegasKomnenos/io

Add encoding="locale" support for TextIOWrapper

Author: youknowone

Lib/test/test_fileinput.py

@@ -391,8 +391,6 @@ def test__getitem__(self):
             retval2 = fi[1]
             self.assertEqual(retval2, "line2\n")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test__getitem___deprecation(self):
         t = self.writeTmp("line1\nline2\n")
         with self.assertWarnsRegex(DeprecationWarning,
@@ -914,8 +912,6 @@ def test_empty_string(self):
     def test_no_ext(self):
         self.do_test_use_builtin_open("abcd", 2)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @unittest.skipUnless(gzip, "Requires gzip and zlib")
     def test_gz_ext_fake(self):
         original_open = gzip.open

vm/src/stdlib/io.rs

@@ -2193,10 +2193,14 @@ mod _io {
             *data = None;
 
             let encoding = match args.encoding {
-                Some(enc) => enc,
-                None => {
-                    // TODO: try os.device_encoding(fileno) and then locale.getpreferredencoding()
-                    PyStr::from(crate::codecs::DEFAULT_ENCODING).into_ref(&vm.ctx)
+                None if vm.state.settings.utf8_mode > 0 => PyStr::from("utf-8").into_ref(&vm.ctx),
+                Some(enc) if enc.as_str() != "locale" => enc,
+                _ => {
+                    // None without utf8_mode or "locale" encoding
+                    vm.import("locale", None, 0)?
+                        .get_attr("getencoding", vm)?
+                        .call((), vm)?
+                        .try_into_value(vm)?
                 }
             };
 

vm/src/stdlib/sys.rs

@@ -728,7 +728,7 @@ mod sys {
                 hash_randomization: settings.hash_seed.is_none() as u8,
                 isolated: settings.isolated as u8,
                 dev_mode: settings.dev_mode,
-                utf8_mode: 1,
+                utf8_mode: settings.utf8_mode,
                 int_max_str_digits: -1,
                 safe_path: false,
                 warn_default_encoding: settings.warn_default_encoding as u8,

vm/src/vm/setting.rs

@@ -74,6 +74,8 @@ pub struct Settings {
     /// false for wasm. Not a command-line option
     pub allow_external_library: bool,
 
+    pub utf8_mode: u8,
+
     #[cfg(feature = "flame-it")]
     pub profile_output: Option<OsString>,
     #[cfg(feature = "flame-it")]
@@ -107,6 +109,7 @@ impl Default for Settings {
             stdio_unbuffered: false,
             check_hash_based_pycs: "default".to_owned(),
             allow_external_library: cfg!(feature = "importlib"),
+            utf8_mode: 1,
             #[cfg(feature = "flame-it")]
             profile_output: None,
             #[cfg(feature = "flame-it")]