Merge pull request RustPython#3668 from youknowone/interpreter-default

Remove Interpreter::default to alert better there is no stdlib

Author: youknowone

benches/execution.rs

@@ -23,7 +23,7 @@ fn bench_cpython_code(b: &mut Bencher, source: &str) {
 
 fn bench_rustpy_code(b: &mut Bencher, name: &str, source: &str) {
     // NOTE: Take long time.
-    Interpreter::default().enter(|vm| {
+    Interpreter::without_stdlib(Default::default()).enter(|vm| {
         // Note: bench_cpython is both compiling and executing the code.
         // As such we compile the code in the benchmark loop as well.
         b.iter(|| {

benches/microbenchmarks.rs

@@ -3,7 +3,7 @@ use criterion::{
     Criterion, Throughput,
 };
 use rustpython_compiler::Mode;
-use rustpython_vm::{common::ascii, InitParameter, Interpreter, PyResult, Settings};
+use rustpython_vm::{common::ascii, Interpreter, PyResult, Settings};
 use std::{
     ffi, fs, io,
     path::{Path, PathBuf},
@@ -114,11 +114,10 @@ fn bench_rustpy_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchmar
     settings.dont_write_bytecode = true;
     settings.no_user_site = true;
 
-    Interpreter::new_with_init(settings, |vm| {
+    Interpreter::with_init(settings, |vm| {
         for (name, init) in rustpython_stdlib::get_module_inits().into_iter() {
             vm.add_native_module(name, init);
         }
-        InitParameter::External
     })
     .enter(|vm| {
         let setup_code = vm

examples/freeze/main.rs

@@ -1,7 +1,7 @@
 use rustpython_vm as vm;
 
 fn main() -> vm::PyResult<()> {
-    vm::Interpreter::default().enter(run)
+    vm::Interpreter::without_stdlib(Default::default()).enter(run)
 }
 
 fn run(vm: &vm::VirtualMachine) -> vm::PyResult<()> {

examples/hello_embed.rs

@@ -1,7 +1,7 @@
 use rustpython_vm as vm;
 
 fn main() -> vm::PyResult<()> {
-    vm::Interpreter::default().enter(|vm| {
+    vm::Interpreter::without_stdlib(Default::default()).enter(|vm| {
         let scope = vm.new_scope_with_builtins();
 
         let code_obj = vm

examples/mini_repl.rs

@@ -29,7 +29,7 @@ fn on(b: bool) {
 }
 
 fn main() -> vm::PyResult<()> {
-    vm::Interpreter::default().enter(run)
+    vm::Interpreter::without_stdlib(Default::default()).enter(run)
 }
 
 fn run(vm: &vm::VirtualMachine) -> vm::PyResult<()> {

src/lib.rs

@@ -47,14 +47,13 @@ mod shell;
 
 use clap::{App, AppSettings, Arg, ArgMatches};
 use rustpython_vm::{
-    builtins::{PyDictRef, PyInt},
-    compile, match_class,
+    builtins::PyInt,
+    match_class,
     scope::Scope,
     stdlib::{atexit, sys},
-    AsObject, InitParameter, Interpreter, PyObjectRef, PyResult, Settings, TryFromObject,
-    VirtualMachine,
+    AsObject, Interpreter, PyResult, Settings, VirtualMachine,
 };
-use std::{env, path::Path, process, str::FromStr};
+use std::{env, process, str::FromStr};
 
 pub use rustpython_vm as vm;
 
@@ -84,10 +83,9 @@ where
         }
     }
 
-    let interp = Interpreter::new_with_init(settings, |vm| {
+    let interp = Interpreter::with_init(settings, |vm| {
         add_stdlib(vm);
         init(vm);
-        InitParameter::External
     });
 
     let exitcode = interp.enter(move |vm| {
@@ -574,7 +572,7 @@ __import__("io").TextIOWrapper(
         .downcast()
         .expect("TextIOWrapper.read() should return str");
     eprintln!("running get-pip.py...");
-    _run_string(vm, scope, getpip_code.as_str(), "get-pip.py".to_owned())
+    vm.run_code_string(scope, getpip_code.as_str(), "get-pip.py".to_owned())
 }
 
 #[cfg(not(feature = "ssl"))]
@@ -599,13 +597,16 @@ fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> {
 
     // Figure out if a -c option was given:
     if let Some(command) = matches.value_of("c") {
-        run_command(vm, scope, command.to_owned())?;
+        debug!("Running command {}", command);
+        vm.run_code_string(scope, command, "<stdin>".to_owned())?;
     } else if let Some(module) = matches.value_of("m") {
-        run_module(vm, module)?;
+        debug!("Running module {}", module);
+        vm.run_module(module)?;
     } else if matches.is_present("install_pip") {
         install_pip(scope, vm)?;
     } else if let Some(filename) = matches.value_of("script") {
-        run_script(vm, scope.clone(), filename)?;
+        debug!("Running file {}", filename);
+        vm.run_script(scope.clone(), filename)?;
         if matches.is_present("inspect") {
             shell::run_shell(vm, scope)?;
         }
@@ -620,97 +621,13 @@ fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> {
     Ok(())
 }
 
-fn _run_string(vm: &VirtualMachine, scope: Scope, source: &str, source_path: String) -> PyResult {
-    let code_obj = vm
-        .compile(source, compile::Mode::Exec, source_path.clone())
-        .map_err(|err| vm.new_syntax_error(&err))?;
-    // trace!("Code object: {:?}", code_obj.borrow());
-    scope
-        .globals
-        .set_item("__file__", vm.new_pyobj(source_path), vm)?;
-    vm.run_code_obj(code_obj, scope)
-}
-
-fn run_command(vm: &VirtualMachine, scope: Scope, source: String) -> PyResult<()> {
-    debug!("Running command {}", source);
-    _run_string(vm, scope, &source, "<stdin>".to_owned())?;
-    Ok(())
-}
-
-fn run_module(vm: &VirtualMachine, module: &str) -> PyResult<()> {
-    debug!("Running module {}", module);
-    let runpy = vm.import("runpy", None, 0)?;
-    let run_module_as_main = runpy.get_attr("_run_module_as_main", vm)?;
-    vm.invoke(&run_module_as_main, (module,))?;
-    Ok(())
-}
-
-fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> {
-    let path_importer_cache = vm.sys_module.get_attr("path_importer_cache", vm)?;
-    let path_importer_cache = PyDictRef::try_from_object(vm, path_importer_cache)?;
-    if let Some(importer) = path_importer_cache.get_item_opt(path, vm)? {
-        return Ok(Some(importer));
-    }
-    let path = vm.ctx.new_str(path);
-    let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?;
-    let mut importer = None;
-    let path_hooks: Vec<PyObjectRef> = path_hooks.try_into_value(vm)?;
-    for path_hook in path_hooks {
-        match vm.invoke(&path_hook, (path.clone(),)) {
-            Ok(imp) => {
-                importer = Some(imp);
-                break;
-            }
-            Err(e) if e.fast_isinstance(&vm.ctx.exceptions.import_error) => continue,
-            Err(e) => return Err(e),
-        }
-    }
-    Ok(if let Some(imp) = importer {
-        let imp = path_importer_cache.get_or_insert(vm, path.into(), || imp.clone())?;
-        Some(imp)
-    } else {
-        None
-    })
-}
-
-fn insert_sys_path(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<()> {
-    let sys_path = vm.sys_module.get_attr("path", vm).unwrap();
-    vm.call_method(&sys_path, "insert", (0, obj))?;
-    Ok(())
-}
-
-fn run_script(vm: &VirtualMachine, scope: Scope, script_file: &str) -> PyResult<()> {
-    debug!("Running file {}", script_file);
-    if get_importer(script_file, vm)?.is_some() {
-        insert_sys_path(vm, vm.ctx.new_str(script_file).into())?;
-        let runpy = vm.import("runpy", None, 0)?;
-        let run_module_as_main = runpy.get_attr("_run_module_as_main", vm)?;
-        vm.invoke(&run_module_as_main, (vm.ctx.new_str("__main__"), false))?;
-        return Ok(());
-    }
-    let dir = Path::new(script_file).parent().unwrap().to_str().unwrap();
-    insert_sys_path(vm, vm.ctx.new_str(dir).into())?;
-
-    match std::fs::read_to_string(script_file) {
-        Ok(source) => {
-            _run_string(vm, scope, &source, script_file.to_owned())?;
-        }
-        Err(err) => {
-            error!("Failed reading file '{}': {}", script_file, err);
-            process::exit(1);
-        }
-    }
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
 
     fn interpreter() -> Interpreter {
-        Interpreter::new_with_init(Settings::default(), |vm| {
+        Interpreter::with_init(Settings::default(), |vm| {
             add_stdlib(vm);
-            InitParameter::External
         })
     }
 
@@ -720,11 +637,11 @@ mod tests {
             vm.unwrap_pyresult((|| {
                 let scope = setup_main_module(vm)?;
                 // test file run
-                run_script(vm, scope, "extra_tests/snippets/dir_main/__main__.py")?;
+                vm.run_script(scope, "extra_tests/snippets/dir_main/__main__.py")?;
 
                 let scope = setup_main_module(vm)?;
                 // test module run
-                run_script(vm, scope, "extra_tests/snippets/dir_main")?;
+                vm.run_script(scope, "extra_tests/snippets/dir_main")?;
 
                 Ok(())
             })());

vm/src/builtins/str.rs

@@ -1559,7 +1559,7 @@ mod tests {
 
     #[test]
     fn str_maketrans_and_translate() {
-        Interpreter::default().enter(|vm| {
+        Interpreter::without_stdlib(Default::default()).enter(|vm| {
             let table = vm.ctx.new_dict();
             table
                 .set_item("a", vm.ctx.new_str("🎅").into(), &vm)

vm/src/dictdatatype.rs

@@ -872,7 +872,7 @@ mod tests {
 
     #[test]
     fn test_insert() {
-        Interpreter::default().enter(|vm| {
+        Interpreter::without_stdlib(Default::default()).enter(|vm| {
             let dict = Dict::default();
             assert_eq!(0, dict.len());
 
@@ -921,7 +921,7 @@ mod tests {
     }
 
     fn check_hash_equivalence(text: &str) {
-        Interpreter::default().enter(|vm| {
+        Interpreter::without_stdlib(Default::default()).enter(|vm| {
             let value1 = text;
             let value2 = vm.new_pyobj(value1.to_owned());
 

vm/src/eval.rs

@@ -20,7 +20,7 @@ mod tests {
 
     #[test]
     fn test_print_42() {
-        Interpreter::default().enter(|vm| {
+        Interpreter::without_stdlib(Default::default()).enter(|vm| {
             let source = String::from("print('Hello world')");
             let vars = vm.new_scope_with_builtins();
             let result = eval(&vm, &source, vars, "<unittest>").expect("this should pass");

vm/src/import.rs

@@ -7,16 +7,23 @@ use crate::{
     builtins::{code, code::CodeObject, list, traceback::PyTraceback, PyBaseExceptionRef},
     scope::Scope,
     version::get_git_revision,
-    vm::{InitParameter, VirtualMachine},
+    vm::{thread, VirtualMachine},
     AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject,
 };
 use rand::Rng;
 
 pub(crate) fn init_importlib(
     vm: &mut VirtualMachine,
-    initialize_parameter: InitParameter,
+    allow_external_library: bool,
 ) -> PyResult<()> {
-    use crate::vm::thread::enter_vm;
+    let importlib = init_importlib_base(vm)?;
+    if allow_external_library && cfg!(feature = "rustpython-compiler") {
+        init_importlib_package(vm, importlib)?;
+    }
+    Ok(())
+}
+
+pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult<PyObjectRef> {
     flame_guard!("init importlib");
 
     // importlib_bootstrap needs these and it inlines checks to sys.modules before calling into
@@ -26,53 +33,56 @@ pub(crate) fn init_importlib(
     import_builtin(vm, "_warnings")?;
     import_builtin(vm, "_weakref")?;
 
-    let importlib = enter_vm(vm, || {
+    let importlib = thread::enter_vm(vm, || {
         let importlib = import_frozen(vm, "_frozen_importlib")?;
         let impmod = import_builtin(vm, "_imp")?;
         let install = importlib.get_attr("_install", vm)?;
         vm.invoke(&install, (vm.sys_module.clone(), impmod))?;
         Ok(importlib)
     })?;
     vm.import_func = importlib.get_attr("__import__", vm)?;
+    Ok(importlib)
+}
 
-    if initialize_parameter == InitParameter::External && cfg!(feature = "rustpython-compiler") {
-        enter_vm(vm, || {
-            flame_guard!("install_external");
-
-            // same deal as imports above
-            #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
-            import_builtin(vm, crate::stdlib::os::MODULE_NAME)?;
-            #[cfg(windows)]
-            import_builtin(vm, "winreg")?;
-            import_builtin(vm, "_io")?;
-            import_builtin(vm, "marshal")?;
-
-            let install_external = importlib.get_attr("_install_external_importers", vm)?;
-            vm.invoke(&install_external, ())?;
-            // Set pyc magic number to commit hash. Should be changed when bytecode will be more stable.
-            let importlib_external = vm.import("_frozen_importlib_external", None, 0)?;
-            let mut magic = get_git_revision().into_bytes();
-            magic.truncate(4);
-            if magic.len() != 4 {
-                magic = rand::thread_rng().gen::<[u8; 4]>().to_vec();
-            }
-            let magic: PyObjectRef = vm.ctx.new_bytes(magic).into();
-            importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?;
-            let zipimport_res = (|| -> PyResult<()> {
-                let zipimport = vm.import("zipimport", None, 0)?;
-                let zipimporter = zipimport.get_attr("zipimporter", vm)?;
-                let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?;
-                let path_hooks = list::PyListRef::try_from_object(vm, path_hooks)?;
-                path_hooks.insert(0, zipimporter);
-                Ok(())
-            })();
-            if zipimport_res.is_err() {
-                warn!("couldn't init zipimport")
-            }
+pub(crate) fn init_importlib_package(
+    vm: &mut VirtualMachine,
+    importlib: PyObjectRef,
+) -> PyResult<()> {
+    thread::enter_vm(vm, || {
+        flame_guard!("install_external");
+
+        // same deal as imports above
+        #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
+        import_builtin(vm, crate::stdlib::os::MODULE_NAME)?;
+        #[cfg(windows)]
+        import_builtin(vm, "winreg")?;
+        import_builtin(vm, "_io")?;
+        import_builtin(vm, "marshal")?;
+
+        let install_external = importlib.get_attr("_install_external_importers", vm)?;
+        vm.invoke(&install_external, ())?;
+        // Set pyc magic number to commit hash. Should be changed when bytecode will be more stable.
+        let importlib_external = vm.import("_frozen_importlib_external", None, 0)?;
+        let mut magic = get_git_revision().into_bytes();
+        magic.truncate(4);
+        if magic.len() != 4 {
+            magic = rand::thread_rng().gen::<[u8; 4]>().to_vec();
+        }
+        let magic: PyObjectRef = vm.ctx.new_bytes(magic).into();
+        importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?;
+        let zipimport_res = (|| -> PyResult<()> {
+            let zipimport = vm.import("zipimport", None, 0)?;
+            let zipimporter = zipimport.get_attr("zipimporter", vm)?;
+            let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?;
+            let path_hooks = list::PyListRef::try_from_object(vm, path_hooks)?;
+            path_hooks.insert(0, zipimporter);
             Ok(())
-        })?
-    }
-    Ok(())
+        })();
+        if zipimport_res.is_err() {
+            warn!("couldn't init zipimport")
+        }
+        Ok(())
+    })
 }
 
 pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult {