Merge pull request RustPython#4133 from youknowone/pylib

Split pylib from vm

Author: youknowone

Cargo.lock

@@ -15,16 +15,16 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 resolver = "2"
 members = [
     "compiler", "compiler/ast", "compiler/core", "compiler/codegen", "compiler/parser",
-    ".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
+    ".", "common", "derive", "jit", "vm", "pylib", "stdlib", "wasm/lib",
 ]
 
 [features]
 default = ["threading", "stdlib", "zlib", "importlib", "encodings", "rustpython-parser/lalrpop"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
-stdlib = ["rustpython-stdlib"]
+stdlib = ["rustpython-stdlib", "rustpython-pylib"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
-freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
+freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
 threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
 zlib = ["stdlib", "rustpython-stdlib/zlib"]
@@ -35,7 +35,8 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
 [dependencies]
 rustpython-compiler = { path = "compiler", version = "0.1.1" }
 rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
-rustpython-stdlib = {path = "stdlib", optional = true, default-features = false}
+rustpython-pylib = { path = "pylib", optional = true, default-features = false }
+rustpython-stdlib = { path = "stdlib", optional = true, default-features = false }
 rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
 
 cfg-if = "1.0.0"

Cargo.toml

@@ -12,8 +12,8 @@ include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"
 freeze-stdlib = []
 
 [dependencies]
-rustpython-compiler-core = { version = "0.1.2", path = "../../compiler/core" }
-rustpython-derive = { version = "0.1.2", path = "../../derive" }
+rustpython-compiler-core = { version = "0.1.2", path = "../compiler/core" }
+rustpython-derive = { version = "0.1.2", path = "../derive" }
 
 [build-dependencies]
 glob = "0.3"

vm/pylib-crate/Cargo.toml pylib/Cargo.tomlvm/pylib-crate/Cargo.toml renamed to pylib/Cargo.toml

@@ -0,0 +1 @@
+../Lib

pylib/Lib

@@ -0,0 +1,15 @@
+//! This crate includes the compiled python bytecode of the RustPython standard library. The most
+//! common way to use this crate is to just add the `"freeze-stdlib"` feature to `rustpython-vm`,
+//! in order to automatically include the python part of the standard library into the binary.
+
+// windows needs to read the symlink out of `Lib` as git turns it into a text file,
+// so build.rs sets this env var
+pub const LIB_PATH: &str = match option_env!("win_lib_path") {
+    Some(s) => s,
+    None => concat!(env!("CARGO_MANIFEST_DIR"), "/../Lib"),
+};
+
+#[cfg(feature = "freeze-stdlib")]
+pub fn frozen_stdlib() -> impl Iterator<Item = (String, rustpython_compiler_core::FrozenModule)> {
+    rustpython_derive::py_freeze!(dir = "../Lib", crate_name = "rustpython_compiler_core")
+}

vm/pylib-crate/build.rs pylib/build.rsvm/pylib-crate/build.rs renamed to pylib/build.rs

@@ -248,6 +248,35 @@ fn add_stdlib(vm: &mut VirtualMachine) {
     let _ = vm;
     #[cfg(feature = "stdlib")]
     vm.add_native_modules(rustpython_stdlib::get_module_inits());
+
+    // if we're on freeze-stdlib, the core stdlib modules will be included anyway
+    #[cfg(feature = "freeze-stdlib")]
+    vm.add_frozen(rustpython_pylib::frozen_stdlib());
+
+    #[cfg(not(feature = "freeze-stdlib"))]
+    {
+        use rustpython_vm::common::rc::PyRc;
+        let state = PyRc::get_mut(&mut vm.state).unwrap();
+
+        #[allow(clippy::needless_collect)] // false positive
+        let path_list: Vec<_> = state.settings.path_list.drain(..).collect();
+
+        // BUILDTIME_RUSTPYTHONPATH should be set when distributing
+        if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") {
+            state
+                .settings
+                .path_list
+                .extend(split_paths(paths).map(|path| path.into_os_string().into_string().unwrap()))
+        } else {
+            #[cfg(feature = "rustpython-pylib")]
+            state
+                .settings
+                .path_list
+                .push(rustpython_pylib::LIB_PATH.to_owned())
+        }
+
+        state.settings.path_list.extend(path_list.into_iter());
+    }
 }
 
 /// Create settings by examining command line arguments and environment
@@ -264,21 +293,6 @@ fn create_settings(matches: &ArgMatches) -> Settings {
 
     let ignore_environment = settings.ignore_environment || settings.isolated;
 
-    // when rustpython-vm/pylib is enabled, Settings::default().path_list has pylib::LIB_PATH
-    let maybe_pylib = settings.path_list.pop();
-
-    // add the current directory to sys.path
-    settings.path_list.push("".to_owned());
-
-    // BUILDTIME_RUSTPYTHONPATH should be set when distributing
-    if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") {
-        settings
-            .path_list
-            .extend(split_paths(paths).map(|path| path.into_os_string().into_string().unwrap()))
-    } else {
-        settings.path_list.extend(maybe_pylib);
-    }
-
     if !ignore_environment {
         settings.path_list.extend(get_paths("RUSTPYTHONPATH"));
         settings.path_list.extend(get_paths("PYTHONPATH"));

pylib/src/lib.rs

@@ -14,7 +14,7 @@ importlib = []
 encodings = ["importlib"]
 vm-tracing-logging = []
 flame-it = ["flame", "flamer"]
-freeze-stdlib = ["rustpython-pylib/freeze-stdlib"]
+freeze-stdlib = []
 jit = ["rustpython-jit"]
 threading = ["rustpython-common/threading"]
 compiler = ["parser", "codegen", "rustpython-compiler"]
@@ -31,7 +31,6 @@ rustpython-compiler-core = { path = "../compiler/core", version = "0.1.2" }
 rustpython-common = { path = "../common" }
 rustpython-derive = { path = "../derive", version = "0.1.2" }
 rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" }
-rustpython-pylib = { path = "pylib-crate", version = "0.1.0" }
 
 num-complex = { version = "0.4.0", features = ["serde"] }
 num-bigint = { version = "0.4.3", features = ["serde"] }

src/lib.rs

@@ -1,6 +1,6 @@
 use crate::bytecode::FrozenModule;
 
-pub fn get_module_inits() -> impl Iterator<Item = (String, FrozenModule)> {
+pub fn core_frozen_inits() -> impl Iterator<Item = (String, FrozenModule)> {
     let iter = std::iter::empty();
     macro_rules! ext_modules {
         ($iter:ident, ($modules:expr)) => {
@@ -20,16 +20,24 @@ pub fn get_module_inits() -> impl Iterator<Item = (String, FrozenModule)> {
     // Python modules that the vm calls into, but are not actually part of the stdlib. They could
     // in theory be implemented in Rust, but are easiest to do in Python for one reason or another.
     // Includes _importlib_bootstrap and _importlib_bootstrap_external
-    ext_modules!(iter, (rustpython_pylib::frozen_builtins()));
+    ext_modules!(
+        iter,
+        (rustpython_derive::py_freeze!(
+            dir = "./Lib/python_builtins",
+            crate_name = "rustpython_compiler_core"
+        ))
+    );
 
-    #[cfg(not(feature = "freeze-stdlib"))]
     // core stdlib Python modules that the vm calls into, but are still used in Python
     // application code, e.g. copyreg
-    ext_modules!(iter, (rustpython_pylib::frozen_core()));
-
-    // if we're on freeze-stdlib, the core stdlib modules will be included anyway
-    #[cfg(feature = "freeze-stdlib")]
-    ext_modules!(iter, (rustpython_pylib::frozen_stdlib()));
+    #[cfg(not(feature = "freeze-stdlib"))]
+    ext_modules!(
+        iter,
+        (rustpython_derive::py_freeze!(
+            dir = "./Lib/core_modules",
+            crate_name = "rustpython_compiler_core"
+        ))
+    );
 
     iter
 }