diff --git a/ci/azure-pipelines-template.yml b/ci/azure-pipelines-template.yml index f3aa9d5f5..eb78205ee 100644 --- a/ci/azure-pipelines-template.yml +++ b/ci/azure-pipelines-template.yml @@ -47,14 +47,39 @@ jobs: displayName: Clippy condition: ne( variables['rustup_toolchain'], 'nightly' ) - - ${{ if ne(parameters.name, 'Windows') }}: + - ${{ if eq(parameters.name, 'Linux') }}: + - script: | + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 --pip-install gevent==1.4.0 ~/pyapp + cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp + displayName: Build Oxidized Application + + - ${{ if eq(parameters.name, 'macOS') }}: + - script: brew install openssl@1.1 + displayName: 'Install OpenSSL' + + - ${{ if eq(parameters.name, 'macOS') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 ~/pyapp + export CPPFLAGS="-I$(brew --prefix openssl@1.1)/include" + export LDFLAGS="-L$(brew --prefix openssl@1.1)/lib" + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 --pip-install gevent==1.4.0 ~/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp displayName: Build Oxidized Application + - ${{ if eq(parameters.name, 'Windows') }}: + - powershell: | + git clone 'https://github.com/Badgerati/Fudge' + .\Fudge\src\Fudge.ps1 install -adhoc openssl + displayName: 'Install OpenSSL' + + - ${{ if eq(parameters.name, 'Windows') }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + - ${{ if eq(parameters.name, 'Windows') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 %USERPROFILE%/pyapp + set INCLUDE=%ProgramFiles%\OpenSSL-Win64\include;%INCLUDE% + set LIB=%ProgramFiles%\OpenSSL-Win64\lib\;%LIB% + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 --pip-install gevent==1.4.0 %USERPROFILE%/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run %USERPROFILE%/pyapp displayName: Build Oxidized Application (Windows) diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index a0db3233b..7134c89d1 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -7,7 +7,7 @@ jobs: - template: azure-pipelines-template.yml parameters: name: macOS - vmImage: macOS-10.14 + vmImage: macOS-10.13 - template: azure-pipelines-template.yml parameters: diff --git a/ci/pyapp.py b/ci/pyapp.py index ae9a27407..3f610e9cc 100644 --- a/ci/pyapp.py +++ b/ci/pyapp.py @@ -1,11 +1,37 @@ import appdirs import _cffi_backend -# Allow AttributeError rather than ImportError -# as this indirectly triggers a missing __file__ -# https://github.com/dabeaz/ply/issues/216 -try: - import zero_buffer -except AttributeError: - pass + +import markupsafe._speedups +import simplejson._speedups + +import markupsafe +import simplejson print("hello, world") + +import cryptography.fernet + +m = b'does this work?' +s = 'original message:\n' + str(m) + '\n\n' + +k = cryptography.fernet.Fernet.generate_key() +fernet = cryptography.fernet.Fernet(k) +t = fernet.encrypt(m) +d = fernet.decrypt(t) + +s += 'key:\n' + str(k) + '\n\n' +s += 'token:\n' + str(t) + '\n\n' +s += 'decoded message:\n' + str(d) +print(s) +assert d == m + +# This only confirms the built objects are loadable. +import gevent +import gevent._queue +assert 'gevent' in dir(gevent._queue) + +import gevent._event +import gevent._greenlet +import gevent._local +import gevent.resolver.cares +import gevent.libev.corecext diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index ddfd7772c..9bcdd6ac4 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -20,6 +20,12 @@ path = "src/lib.rs" [build-dependencies] vergen = "3" +# Contains one fix needed for Windows +[dependencies.object] +git = "https://github.com/gimli-rs/object.git" +branch = "master" +features = ["read", "std", "write"] + [dependencies] byteorder = "1.2" cargo_toml = "0.6" @@ -27,6 +33,7 @@ cc = "1.0" clap = "2.32" codemap = "0.1" codemap-diagnostic = "0.1" +copy_dir = "0.1.2" encoding_rs = "0.8" fs2 = "0.4" git2 = "0.9" @@ -37,6 +44,7 @@ hex = "0.3" itertools = "0.8" lazy_static = "1.3" libc = "0.2" +memmap = "0.7" regex = "1" reqwest = "0.9" rustc_version = "0.2" @@ -53,6 +61,7 @@ url = "1.7" uuid = { version = "0.7", features = ["v4", "v5"] } version-compare = "0.0" walkdir = "2" +which = "3.1.0" xml-rs = "0.8" zip = "0.5" zstd = "0.4" diff --git a/pyoxidizer/src/app_packaging/config.rs b/pyoxidizer/src/app_packaging/config.rs index 80e0ce2cd..a18089914 100644 --- a/pyoxidizer/src/app_packaging/config.rs +++ b/pyoxidizer/src/app_packaging/config.rs @@ -26,6 +26,7 @@ pub enum InstallLocation { #[derive(Clone, Debug, PartialEq)] pub struct PackagingSetupPyInstall { pub path: String, + pub venv_path: Option, pub extra_env: HashMap, pub extra_global_arguments: Vec, pub optimize_level: i64, @@ -87,6 +88,7 @@ pub struct PackagingPackageRoot { #[derive(Clone, Debug, PartialEq)] pub struct PackagingPipInstallSimple { pub package: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub excludes: Vec, @@ -99,6 +101,7 @@ pub struct PackagingPipInstallSimple { pub struct PackagingPipRequirementsFile { // TODO resolve to a PathBuf. pub requirements_path: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub include_source: bool, diff --git a/pyoxidizer/src/app_packaging/packaging_rule.rs b/pyoxidizer/src/app_packaging/packaging_rule.rs index 032066e1d..bf66e57ac 100644 --- a/pyoxidizer/src/app_packaging/packaging_rule.rs +++ b/pyoxidizer/src/app_packaging/packaging_rule.rs @@ -15,12 +15,13 @@ use super::config::{ PackagingStdlibExtensionsPolicy, PackagingVirtualenv, PythonPackaging, }; use super::state::BuildContext; -use crate::py_packaging::distribution::{is_stdlib_test_package, ParsedPythonDistribution}; -use crate::py_packaging::distutils::{prepare_hacked_distutils, read_built_extensions}; +use crate::py_packaging::distribution::{ + is_stdlib_test_package, resolve_python_paths, ParsedPythonDistribution, +}; +use crate::py_packaging::distutils::read_built_extensions; use crate::py_packaging::fsscan::{ find_python_resources, is_package_from_path, PythonFileResource, }; -use crate::py_packaging::pip::pip_install; use crate::py_packaging::resource::{AppRelativeResources, PythonResource}; #[derive(Debug)] @@ -110,6 +111,7 @@ fn resource_full_name(resource: &PythonFileResource) -> &str { } } +// part of new approach from pip.rs; unused atm fn python_resource_full_name(resource: &PythonResource) -> String { match resource { PythonResource::ModuleSource { name, .. } => name.clone(), @@ -119,30 +121,6 @@ fn python_resource_full_name(resource: &PythonResource) -> String { } } -struct PythonPaths { - main: PathBuf, - site_packages: PathBuf, -} - -/// Resolve the location of Python modules given a base install path. -fn resolve_python_paths(base: &Path, python_version: &str, is_windows: bool) -> PythonPaths { - let mut p = base.to_path_buf(); - - if is_windows { - p.push("Lib"); - } else { - p.push("lib"); - p.push(format!("python{}", &python_version[0..3])); - } - - let site_packages = p.join("site-packages"); - - PythonPaths { - main: p, - site_packages, - } -} - fn resolve_built_extensions( state_dir: &Path, res: &mut Vec, @@ -159,6 +137,122 @@ fn resolve_built_extensions( Ok(()) } +/// Processes resources in a path +/// Args includes and excludes are ignored if None or an empty Vec. +fn process_resources( + logger: &slog::Logger, + path: &PathBuf, + location: &ResourceLocation, + state_dir: Option<&PathBuf>, + include_source: bool, + optimize_level: i64, + includes: Option<&Vec>, + excludes: Option<&Vec>, +) -> Vec { + let mut res = Vec::new(); + + let path_s = path.display().to_string(); + warn!(logger, "processing resources from {}", path_s); + + for resource in find_python_resources(path) { + let full_name = resource_full_name(&resource); + + let excluded = match includes { + Some(values) => values.iter().any(|v| { + let prefix = v.clone() + "."; + full_name != v && !full_name.starts_with(&prefix) + }), + None => false, + }; + + if excluded { + info!( + logger, + "whitelist skipping {}", full_name + ); + continue; + } + + let excluded = match excludes { + Some(values) => match values.is_empty() { + true => false, + false => values.iter().all(|v| { + let prefix = v.clone() + "."; + full_name == v || full_name.starts_with(&prefix) + }), + }, + None => false, + }; + + if excluded { + info!( + logger, + "blacklist skipping {}", full_name + ); + continue; + } + + match resource { + PythonFileResource::Source { + full_name, path, .. + } => { + let is_package = is_package_from_path(&path); + let source = fs::read(path).expect("error reading source file"); + + if include_source { + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleSource { + name: full_name.clone(), + source: source.clone(), + is_package, + }, + }); + } + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleBytecodeRequest { + name: full_name.clone(), + source, + optimize_level: optimize_level as i32, + is_package, + }, + }); + } + + PythonFileResource::Resource(resource) => { + let data = fs::read(resource.path).expect("error reading resource file"); + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::Resource { + package: resource.package.clone(), + name: resource.stem.clone(), + data, + }, + }); + } + + _ => {} + } + } + + match state_dir { + Some(dir) => { + if dir.exists() { + resolve_built_extensions(&dir, &mut res, &location).unwrap(); + } + } + None => {} + }; + + res +} + fn resolve_stdlib_extensions_policy( logger: &slog::Logger, dist: &ParsedPythonDistribution, @@ -343,165 +437,44 @@ fn resolve_stdlib( } fn resolve_virtualenv( + logger: &slog::Logger, dist: &ParsedPythonDistribution, rule: &PackagingVirtualenv, ) -> Vec { - let mut res = Vec::new(); - let location = ResourceLocation::new(&rule.install_location); let python_paths = - resolve_python_paths(&Path::new(&rule.path), &dist.version, dist.os == "windows"); - let packages_path = python_paths.site_packages; + resolve_python_paths(&Path::new(&rule.path), &dist.version); - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), + ) } -fn resolve_package_root(rule: &PackagingPackageRoot) -> Vec { - let mut res = Vec::new(); - +fn resolve_package_root( + logger: &slog::Logger, + rule: &PackagingPackageRoot, +) -> Vec { let location = ResourceLocation::new(&rule.install_location); let path = PathBuf::from(&rule.path); - for resource in find_python_resources(&path) { - let mut relevant = false; - let full_name = resource_full_name(&resource); - - for package in &rule.packages { - let prefix = package.clone() + "."; - - if full_name == package || full_name.starts_with(&prefix) { - relevant = true; - } - } - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &path, + &location, + None, + rule.include_source, + rule.optimize_level, + Some(&rule.packages), + None, + ) } fn resolve_pip_install_simple( @@ -510,97 +483,61 @@ fn resolve_pip_install_simple( rule: &PackagingPipInstallSimple, verbose: bool, ) -> Vec { - let mut install_args = vec![ - "--no-binary".to_string(), - ":all:".to_string(), - rule.package.clone(), - ]; - - if let Some(ref args) = rule.extra_args { - install_args.extend(args.clone()); - } - - let resources = pip_install(logger, dist, verbose, &install_args, &rule.extra_env).unwrap(); - - let mut res = Vec::new(); let location = ResourceLocation::new(&rule.install_location); - for resource in resources { - let mut relevant = true; - - let full_name = python_resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); - if &full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } + let mut pip_args: Vec = vec![ + "-m".to_string(), + "pip".to_string(), + "--disable-pip-version-check".to_string(), + ]; - if !relevant { - continue; - } + if verbose { + pip_args.push("--verbose".to_string()); + } - match resource { - PythonResource::ModuleSource { - name, - is_package, - source, - } => { - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: name.clone(), - source: source.clone(), - is_package, - }, - }); - } + pip_args.extend(vec![ + "install".to_string(), + "--no-binary".to_string(), + ":all:".to_string(), + rule.package.clone(), + ]); - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name, - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } + if let Some(ref args) = rule.extra_args { + pip_args.extend(args.clone()); + } - PythonResource::Resource { - package, - name, - data, - } => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package, - name, - data, - }, - }); - } + for (key, value) in rule.extra_env.iter() { + extra_envs.insert(key.clone(), value.clone()); + } - PythonResource::BuiltExtensionModule(em) => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::BuiltExtensionModule(em), - }); - } + // TODO send stderr to stdout. + let mut cmd = std::process::Command::new(&python_paths.python_exe) + .args(&pip_args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("error running pip"); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); - _ => {} + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); } } - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), + ) } fn resolve_pip_requirements_file( @@ -609,83 +546,69 @@ fn resolve_pip_requirements_file( rule: &PackagingPipRequirementsFile, verbose: bool, ) -> Vec { - let mut install_args = vec![ + let location = ResourceLocation::new(&rule.install_location); + + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); + + let mut pip_args: Vec = vec![ + "-m".to_string(), + "pip".to_string(), + "--disable-pip-version-check".to_string(), + ]; + + if verbose { + pip_args.push("--verbose".to_string()); + } + + pip_args.extend(vec![ + "install".to_string(), "--no-binary".to_string(), ":all:".to_string(), "--requirement".to_string(), rule.requirements_path.to_string(), - ]; + ]); if let Some(ref args) = rule.extra_args { - install_args.extend(args.clone()); + pip_args.extend(args.clone()); } - let resources = pip_install(logger, dist, verbose, &install_args, &rule.extra_env).unwrap(); - - let location = ResourceLocation::new(&rule.install_location); - - let mut res = Vec::new(); - - for resource in resources { - match resource { - PythonResource::ModuleSource { - name, - is_package, - source, - } => { - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name, - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } + for (key, value) in rule.extra_env.iter() { + extra_envs.insert(key.clone(), value.clone()); + } - PythonResource::Resource { - package, - name, - data, - } => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package, - name, - data, - }, - }); - } + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + pip_args.join(" ") + ); - PythonResource::BuiltExtensionModule(em) => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::BuiltExtensionModule(em), - }); - } + // TODO send stderr to stdout. + let mut cmd = std::process::Command::new(&python_paths.python_exe) + .args(&pip_args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("error running pip"); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); - _ => {} + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); } } - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + None, + ) } fn resolve_setup_py_install( @@ -695,8 +618,6 @@ fn resolve_setup_py_install( rule: &PackagingSetupPyInstall, verbose: bool, ) -> Vec { - let mut res = Vec::new(); - // Execution directory is resolved relative to the active configuration // file unless it is absolute. let rule_path = PathBuf::from(&rule.path); @@ -709,31 +630,12 @@ fn resolve_setup_py_install( let location = ResourceLocation::new(&rule.install_location); - let temp_dir = tempdir::TempDir::new("pyoxidizer-setup-py-install") - .expect("could not create temp directory"); - - let target_dir_path = temp_dir.path().join("install"); - let target_dir_s = target_dir_path.display().to_string(); - - let python_paths = resolve_python_paths(&target_dir_path, &dist.version, dist.os == "windows"); - - std::fs::create_dir_all(&python_paths.site_packages) - .expect("unable to create site-packages directory"); - - let mut extra_envs = prepare_hacked_distutils( - logger, - dist, - temp_dir.path(), - &[&python_paths.site_packages, &python_paths.main], - ) - .expect("unable to hack distutils"); + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); for (key, value) in rule.extra_env.iter() { extra_envs.insert(key.clone(), value.clone()); } - warn!(logger, "python setup.py installing to {}", target_dir_s); - let mut args = vec!["setup.py"]; for arg in &rule.extra_global_arguments { @@ -744,10 +646,19 @@ fn resolve_setup_py_install( args.push("--verbose"); } - args.extend(&["install", "--prefix", &target_dir_s, "--no-compile"]); + let prefix_dir_s = python_paths.prefix.display().to_string(); + + args.extend(&["install", "--prefix", &prefix_dir_s, "--no-compile"]); + + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); // TODO send stderr to stdout. - let mut cmd = std::process::Command::new(&dist.python_exe) + let mut cmd = std::process::Command::new(&python_paths.python_exe) .current_dir(cwd) .args(&args) .envs(&extra_envs) @@ -768,81 +679,16 @@ fn resolve_setup_py_install( panic!("error running setup.py"); } - let packages_path = python_paths.site_packages; - - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - resolve_built_extensions( - &PathBuf::from(extra_envs.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()), - &mut res, + process_resources( + &logger, + &python_paths.site_packages, &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), ) - .unwrap(); - - res } /// Resolves a Python packaging rule to resources to package. @@ -872,9 +718,9 @@ pub fn resolve_python_packaging( PythonPackaging::Stdlib(rule) => resolve_stdlib(logger, dist, &rule), - PythonPackaging::Virtualenv(rule) => resolve_virtualenv(dist, &rule), + PythonPackaging::Virtualenv(rule) => resolve_virtualenv(logger, dist, &rule), - PythonPackaging::PackageRoot(rule) => resolve_package_root(&rule), + PythonPackaging::PackageRoot(rule) => resolve_package_root(logger, &rule), PythonPackaging::PipInstallSimple(rule) => { resolve_pip_install_simple(logger, dist, &rule, verbose) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 88e482aad..34fabb4f2 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -14,10 +14,12 @@ # ported to VS 2015 by Steve Dower import json +import pathlib import os import shutil import stat import subprocess +import sys import winreg from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ @@ -265,7 +267,7 @@ def initialize(self, plat_name=None): # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib # later to dynamically link to ucrtbase but not vcruntime. self.compile_options = [ - '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG' + '/nologo', '/Ox', '/W3', '/DNDEBUG' ] self.compile_options.append('/MD' if self._vcruntime_redist else '/MT') @@ -426,6 +428,18 @@ def compile(self, sources, except DistutilsExecError as msg: raise CompileError(msg) + obj_index = args.index("/Fo" + obj) + args.remove("/Fo" + obj) + args.insert(obj_index, "/Fo" + obj + '.static') + # This is needed to activate specific symbol visibility / linking + # behavior on Windows. It isn't needed on UNIX but should be harmless. + args.append('/DPy_BUILD_CORE_BUILTIN=1') + + try: + self.spawn(args) + except DistutilsExecError as msg: + raise CompileError(msg) + return objects @@ -562,30 +576,28 @@ def extension_link_shared_object(self, package=None, ): - if 'PYOXIDIZER_DISTUTILS_STATE_DIR' not in os.environ: - raise Exception('PYOXIDIZER_DISTUTILS_STATE_DIR not defined') - - # The extension is compiled as a built-in, so linking a shared library - # won't work due to symbol visibility/export issues. The extension is - # expecting all CPython symbols to be available in the current binary, - # which they won't be for a shared library. PyOxidizer doesn't use the - # library anyway (at least not yet), so don't even bother with any - # linking. - #self.link(CCompiler.SHARED_OBJECT, objects, - # output_filename, output_dir, - # libraries, library_dirs, runtime_library_dirs, - # export_symbols, debug, - # extra_preargs, extra_postargs, build_temp, target_lang) - # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. - dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + dest_path = str(pathlib.Path(sys.prefix) / 'state' / 'pyoxidizer') + + if not os.path.exists(dest_path): + os.makedirs(dest_path) - # We need to copy the object files because they may be in a temp + # We need to copy the static object files because they may be in a temp # directory that doesn't outlive this process. object_paths = [] for i, o in enumerate(objects): + if 'libffi_msvc' in o: + print('Ignored static {}'.format(o)) + # https://github.com/indygreg/python-build-standalone/issues/23 + # cffi includes a near replica of CPython 3.7's custom libffi. + continue + # If there are pre-compiled .obj files provided + # they will not have a .static version, and the + # original should be used. greenlet does this. + if os.path.exists(o + '.static'): + o = o + '.static' p = os.path.join(dest_path, '%s.%d.o' % (name, i)) shutil.copyfile(o, p) object_paths.append(p) @@ -604,6 +616,21 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) + + # This is the shared library, which would only be used if this is a + # build dependencies during the packaging rule processing, and even + # then it is possible that the package is usable as a build dependency + # without this extension. + try: + self.link(CCompiler.SHARED_OBJECT, objects, + output_filename, output_dir, + libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, + extra_preargs, extra_postargs, build_temp, target_lang) + except Exception as e: + print('Linking shared {} failed: {!r}'.format(name, e), + file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/distutils/command/build_ext.py b/pyoxidizer/src/distutils/command/build_ext.py index 156ce30a9..9b0f854e2 100644 --- a/pyoxidizer/src/distutils/command/build_ext.py +++ b/pyoxidizer/src/distutils/command/build_ext.py @@ -525,10 +525,6 @@ def build_extension(self, ext): for undef in ext.undef_macros: macros.append((undef,)) - # This is needed to activate specific symbol visibility / linking - # behavior on Windows. It isn't needed on UNIX but should be harmless. - macros.append(('Py_BUILD_CORE_BUILTIN', '1')) - objects = self.compiler.compile(sources, output_dir=self.build_temp, macros=macros, diff --git a/pyoxidizer/src/distutils/unixccompiler.py b/pyoxidizer/src/distutils/unixccompiler.py index 26f575fe0..971072115 100644 --- a/pyoxidizer/src/distutils/unixccompiler.py +++ b/pyoxidizer/src/distutils/unixccompiler.py @@ -14,6 +14,7 @@ """ import json, os, sys, re, shutil +import pathlib from distutils import sysconfig from distutils.dep_util import newer @@ -113,6 +114,18 @@ def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): if sys.platform == 'darwin': compiler_so = _osx_support.compiler_fixup(compiler_so, cc_args + extra_postargs) + + # If the PyInit symbol need to be renamed, any syms in common section + # may cause a breakage on Linux, and -fno-common avoids that. + # https://github.com/gimli-rs/object/issues/149 + # It may also help with macOS + # https://github.com/gimli-rs/object/issues/158 + # gevent on Linux could be solved this way, but it isnt, as rewriting + # the stdlib _queue is the solution ofr as macOS and Windows and it is + # better for all three platforms to use the same solution. + if '-fno-common' not in compiler_so: + compiler_so.insert(1, '-fno-common') + try: self.spawn(compiler_so + cc_args + [src, '-o', obj] + extra_postargs) @@ -224,19 +237,25 @@ def extension_link_shared_object(self, package=None, ): - if 'PYOXIDIZER_DISTUTILS_STATE_DIR' not in os.environ: - raise Exception('PYOXIDIZER_DISTUTILS_STATE_DIR not defined') + libs=libraries.copy() + if 'crypto' in libs: + libs.remove('crypto') + if 'ssl' in libs: + libs.remove('ssl') self.link(CCompiler.SHARED_OBJECT, objects, output_filename, output_dir, - libraries, library_dirs, runtime_library_dirs, + libs, library_dirs, runtime_library_dirs, export_symbols, debug, extra_preargs, extra_postargs, build_temp, target_lang) # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. - dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + dest_path = str(pathlib.Path(sys.prefix) / 'state' / 'pyoxidizer') + + if not os.path.exists(dest_path): + os.makedirs(dest_path) # We need to copy the object files because they may be in a temp # directory that doesn't outlive this process. @@ -260,6 +279,7 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/projectmgmt.rs b/pyoxidizer/src/projectmgmt.rs index bd8883f87..012bd54a0 100644 --- a/pyoxidizer/src/projectmgmt.rs +++ b/pyoxidizer/src/projectmgmt.rs @@ -498,6 +498,10 @@ pub fn build_project(logger: &slog::Logger, context: &mut BuildContext) -> Resul // https://github.com/rust-lang/rust/issues/37403 is resolved. if cfg!(windows) { envs.push(("RUSTC_BOOTSTRAP", "1".to_string())); + // Allow multiple definitions when using link.exe + if context.target_triple.contains("msvc") { + envs.push(("RUSTFLAGS", "-C link-args=/FORCE:MULTIPLE".to_string())); + } } match process::Command::new("cargo") diff --git a/pyoxidizer/src/py_packaging/distribution.rs b/pyoxidizer/src/py_packaging/distribution.rs index 8a2f36a6e..9328c3ebb 100644 --- a/pyoxidizer/src/py_packaging/distribution.rs +++ b/pyoxidizer/src/py_packaging/distribution.rs @@ -2,31 +2,45 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use copy_dir::copy_dir; use fs2::FileExt; use itertools::Itertools; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use slog::warn; -use std::collections::BTreeMap; +use slog::{info, warn}; +use std::collections::{BTreeMap, HashMap}; +use std::env; use std::fs; use std::fs::{create_dir_all, File}; -use std::io::{Cursor, Read}; +use std::io::{BufRead, BufReader, Cursor, Read}; use std::path::{Path, PathBuf}; use url::Url; use uuid::Uuid; +use which::which; +use super::distutils::prepare_hacked_distutils; use super::fsscan::{ find_python_resources, is_package_from_path, walk_tree_files, PythonFileResource, }; use super::resource::{ResourceData, SourceModule}; + use crate::licensing::NON_GPL_LICENSES; +#[cfg(windows)] +const PYTHON_EXE_BASENAME: &str = "python.exe"; + +#[cfg(unix)] +const PYTHON_EXE_BASENAME: &str = "python3"; + #[cfg(windows)] const PIP_EXE_BASENAME: &str = "pip3.exe"; #[cfg(unix)] const PIP_EXE_BASENAME: &str = "pip3"; +// This needs to be kept in sync with *compiler.py +const PYOXIDIZER_STATE_DIR: &str = "state/pyoxidizer"; + const STDLIB_TEST_PACKAGES: &[&str] = &[ "bsddb.test", "ctypes.test", @@ -311,6 +325,9 @@ pub struct ParsedPythonDistribution { /// Describes license info for things in this distribution. pub license_infos: BTreeMap>, + + /// Path to copy of hacked dist to use for packaging rules venvs + pub venv_base: PathBuf, } #[derive(Debug)] @@ -333,6 +350,102 @@ pub enum ExtensionModuleFilter { NoGPL, } +pub struct PythonPaths { + pub prefix: PathBuf, + pub bin_dir: PathBuf, + pub python_exe: PathBuf, + pub stdlib: PathBuf, + pub site_packages: PathBuf, + pub pyoxidizer_state_dir: PathBuf, +} + +/// Resolve the location of Python modules given a base install path. +pub fn resolve_python_paths(base: &Path, python_version: &str) -> PythonPaths { + let prefix = base.to_path_buf().canonicalize().unwrap(); + + let p = prefix.clone(); + + let bin_dir = if p.join("Scripts").exists() { + p.join("Scripts") + } else { + p.join("bin") + }; + + let python_exe = if bin_dir.join(PYTHON_EXE_BASENAME).exists() { + bin_dir.join(PYTHON_EXE_BASENAME) + } else { + p.join(PYTHON_EXE_BASENAME) + }; + + let mut pyoxidizer_state_dir = p.clone(); + pyoxidizer_state_dir.extend(PYOXIDIZER_STATE_DIR.split('/')); + + let unix_lib_dir = p.join("lib").join(format!("python{}", &python_version[0..3])); + + let stdlib = if unix_lib_dir.exists() { + unix_lib_dir.clone() + } else { + p.join("Lib") + }.canonicalize().unwrap(); + + let site_packages = stdlib.join("site-packages"); + + PythonPaths { + prefix, + bin_dir, + python_exe, + stdlib, + site_packages, + pyoxidizer_state_dir, + } +} + +pub fn invoke_python(python_paths: &PythonPaths, logger: &slog::Logger, args: &[&str]) { + let mut site_packages_s = python_paths + .site_packages + .canonicalize() + .unwrap() + .display() + .to_string(); + + if site_packages_s.starts_with("\\\\?\\") { + site_packages_s = site_packages_s[4..].to_string(); + } + + info!(logger, "setting PYTHONPATH {}", site_packages_s); + + let mut extra_envs = HashMap::new(); + extra_envs.insert("PYTHONPATH".to_string(), site_packages_s); + + info!( + logger, + "running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); + + let mut cmd = std::process::Command::new(&python_paths.python_exe.canonicalize().unwrap()) + .args(args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect( + format!( + "failed to run {} {}", + python_paths.python_exe.display(), + args.join(" ") + ) + .as_str(), + ); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); + } + } +} + impl ParsedPythonDistribution { pub fn from_path( logger: &slog::Logger, @@ -362,24 +475,142 @@ impl ParsedPythonDistribution { } /// Ensure pip is available to run in the distribution. - pub fn ensure_pip(&self) -> PathBuf { - let pip_path = self - .python_exe - .parent() - .expect("could not derive parent") - .to_path_buf() - .join(PIP_EXE_BASENAME); + /// This is only used by pip.rs + pub fn ensure_pip(&self, logger: &slog::Logger) -> PathBuf { + let dist_prefix = self.base_dir.join("python").join("install"); + let python_paths = resolve_python_paths(&dist_prefix, &self.version); + + let pip_path = python_paths.bin_dir.join(PIP_EXE_BASENAME); if !pip_path.exists() { - std::process::Command::new(&self.python_exe) - .args(&["-m", "ensurepip"]) - .status() - .expect("failed to run ensurepip"); + warn!(logger, "{} doesnt exist", pip_path.display().to_string()); + invoke_python(&python_paths, &logger, &["-m", "ensurepip"]); } pip_path } + /// Duplicate the python distribution, with distutils hacked + pub fn create_hacked_base(&self, logger: &slog::Logger) -> PythonPaths { + let venv_base = self.venv_base.clone(); + + let venv_dir_s = self.venv_base.display().to_string(); + + if !venv_base.exists() { + if self.os == "windows" { + let external_python = which("python").unwrap().clone(); + warn!(logger, "python={}", external_python.display().to_string()); + // TODO: check python version + + let external_dist_prefix = external_python.parent().unwrap(); + + copy_dir(&external_dist_prefix, &venv_base).unwrap(); + + let external_dist_prefix_s = external_dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", external_dist_prefix_s, venv_dir_s + ); + } else { + let dist_prefix = self.base_dir.join("python").join("install"); + + copy_dir(&dist_prefix, &venv_base).unwrap(); + + let dist_prefix_s = dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", dist_prefix_s, venv_dir_s + ); + }; + } + + let python_paths = resolve_python_paths(&venv_base, &self.version); + + invoke_python(&python_paths, &logger, &["-m", "ensurepip"]); + + prepare_hacked_distutils(logger, &python_paths); + + python_paths + } + + /// Create a venv from the distribution at path. + pub fn create_venv(&self, logger: &slog::Logger, path: &PathBuf) -> PythonPaths { + let venv_dir_s = path.display().to_string(); + + // This will recreate it, if it was deleted + let python_paths = self.create_hacked_base(&logger); + + if path.exists() { + warn!(logger, "re-using {} {}", "venv", venv_dir_s); + } else { + warn!(logger, "creating {} {}", "venv", venv_dir_s); + invoke_python(&python_paths, &logger, &["-m", "venv", venv_dir_s.as_str()]); + } + + resolve_python_paths(&path, &self.version) + } + + /// Create or re-use an existing venv + pub fn prepare_venv( + &self, + logger: &slog::Logger, + venv_path: Option<&String>, + ) -> (PythonPaths, HashMap) { + let venv_dir_path = match venv_path { + Some(path_str) => PathBuf::from(path_str), + None => tempdir::TempDir::new("pyoxidizer-temp-venv") + .expect("could not create temp directory") + .path() + .join("venv"), + }; + + let python_paths = self.create_venv(logger, &venv_dir_path); + + let mut extra_envs = HashMap::new(); + + let prefix_s = python_paths + .prefix + .canonicalize() + .unwrap() + .display() + .to_string(); + + let venv_path_bin_s = python_paths + .bin_dir + .canonicalize() + .unwrap() + .display() + .to_string(); + + let path_separator = if cfg!(windows) { ";" } else { ":" }; + + let process_path_s = env::var("PATH").unwrap(); + + extra_envs.insert( + "PATH".to_string(), + format!("{}{}{}", venv_path_bin_s, path_separator, process_path_s), + ); + + let mut site_packages_s = python_paths + .site_packages + .canonicalize() + .unwrap() + .display() + .to_string(); + if site_packages_s.starts_with("\\\\?\\") { + site_packages_s = site_packages_s[4..].to_string(); + } + + extra_envs.insert("VIRTUAL_ENV".to_string(), prefix_s); + extra_envs.insert("PYTHONPATH".to_string(), site_packages_s); + + extra_envs.insert("PYOXIDIZER".to_string(), "1".to_string()); + + fs::create_dir_all(&python_paths.pyoxidizer_state_dir).unwrap(); + + (python_paths, extra_envs) + } + /// Obtain resolved `SourceModule` instances for this distribution. /// /// This effectively resolves the raw file content for .py files into @@ -689,6 +920,8 @@ pub fn analyze_python_distribution_data( }; } + let venv_base = dist_dir.parent().unwrap().join("hacked_base"); + Ok(ParsedPythonDistribution { flavor: pi.python_flavor.clone(), version: pi.python_version.clone(), @@ -715,6 +948,7 @@ pub fn analyze_python_distribution_data( py_modules, resources, license_infos, + venv_base, }) } diff --git a/pyoxidizer/src/py_packaging/distutils.rs b/pyoxidizer/src/py_packaging/distutils.rs index 11a932f38..d629e8c77 100644 --- a/pyoxidizer/src/py_packaging/distutils.rs +++ b/pyoxidizer/src/py_packaging/distutils.rs @@ -5,11 +5,11 @@ use lazy_static::lazy_static; use serde::Deserialize; use slog::warn; -use std::collections::{BTreeMap, HashMap}; -use std::fs::{create_dir_all, read_dir, read_to_string}; +use std::collections::BTreeMap; +use std::fs::{read_dir, read_to_string}; use std::path::{Path, PathBuf}; -use super::distribution::ParsedPythonDistribution; +use super::distribution::PythonPaths; use super::resource::BuiltExtensionModule; lazy_static! { @@ -46,70 +46,22 @@ lazy_static! { /// modified distutils will survive multiple process invocations, unlike a /// monkeypatch. People do weird things in setup.py scripts and we want to /// support as many as possible. -pub fn prepare_hacked_distutils( - logger: &slog::Logger, - dist: &ParsedPythonDistribution, - dest_dir: &Path, - extra_python_paths: &[&Path], -) -> Result, String> { - let extra_sys_path = dest_dir.join("packages"); - +pub fn prepare_hacked_distutils(logger: &slog::Logger, target: &PythonPaths) { warn!( logger, "installing modified distutils to {}", - extra_sys_path.display() + target.stdlib.display() ); - let orig_distutils_path = dist.stdlib_path.join("distutils"); - let dest_distutils_path = extra_sys_path.join("distutils"); - - for entry in walkdir::WalkDir::new(&orig_distutils_path) { - match entry { - Ok(entry) => { - if entry.path().is_dir() { - continue; - } - - let source_path = entry.path(); - let rel_path = source_path - .strip_prefix(&orig_distutils_path) - .or_else(|_| Err("unable to strip prefix"))?; - let dest_path = dest_distutils_path.join(rel_path); - - let dest_dir = dest_path.parent().unwrap(); - std::fs::create_dir_all(&dest_dir).or_else(|e| Err(e.to_string()))?; - std::fs::copy(&source_path, &dest_path).or_else(|e| Err(e.to_string()))?; - } - Err(e) => return Err(e.to_string()), - } - } + let dest_distutils_path = target.stdlib.join("distutils"); for (path, data) in MODIFIED_DISTUTILS_FILES.iter() { - let dest_path = dest_distutils_path.join(path); + let mut dest_path = dest_distutils_path.clone(); + dest_path.extend(path.split('/')); warn!(logger, "modifying distutils/{} for oxidation", path); - std::fs::write(dest_path, data).or_else(|e| Err(e.to_string()))?; + std::fs::write(dest_path, data).unwrap(); } - - let state_dir = dest_dir.join("pyoxidizer-build-state"); - create_dir_all(&state_dir).or_else(|e| Err(e.to_string()))?; - - let mut python_paths = vec![extra_sys_path.display().to_string()]; - python_paths.extend(extra_python_paths.iter().map(|p| p.display().to_string())); - - let path_separator = if cfg!(windows) { ";" } else { ":" }; - - let python_path = python_paths.join(path_separator); - - let mut res = HashMap::new(); - res.insert("PYTHONPATH".to_string(), python_path); - res.insert( - "PYOXIDIZER_DISTUTILS_STATE_DIR".to_string(), - state_dir.display().to_string(), - ); - res.insert("PYOXIDIZER".to_string(), "1".to_string()); - - Ok(res) } #[derive(Debug, Deserialize)] diff --git a/pyoxidizer/src/py_packaging/libpython.rs b/pyoxidizer/src/py_packaging/libpython.rs index 6d559201d..f9c18be6f 100644 --- a/pyoxidizer/src/py_packaging/libpython.rs +++ b/pyoxidizer/src/py_packaging/libpython.rs @@ -4,6 +4,7 @@ use itertools::Itertools; use lazy_static::lazy_static; +use memmap::Mmap; use slog::{info, warn}; use std::collections::{BTreeMap, BTreeSet}; use std::fs; @@ -13,6 +14,7 @@ use std::path::{Path, PathBuf}; use super::bytecode::{BytecodeCompiler, CompileMode}; use super::distribution::{ExtensionModule, LicenseInfo, ParsedPythonDistribution}; use super::embedded_resource::EmbeddedPythonResources; +use super::object::rename_init; use super::resource::BuiltExtensionModule; pub const PYTHON_IMPORTER: &[u8] = include_bytes!("memoryimporter.py"); @@ -102,28 +104,57 @@ pub fn make_config_c( continue; } - lines.push(format!("extern PyObject* {}(void);", init_fn)); + if init_fn == "PyInit__queue" && built_extension_modules.contains_key("gevent._queue") { + lines.push("extern PyObject* PyInit_stdlib_queue(void);".to_string()); + } else { + lines.push(format!("extern PyObject* {}(void);", init_fn)); + } } } for em in built_extension_modules.values() { - lines.push(format!("extern PyObject* {}(void);", em.init_fn)); + let ambiguous_line = format!("extern PyObject* {}(void);", em.init_fn); + + if lines.contains(&ambiguous_line) { + lines.push(format!( + "extern PyObject* PyInit_{}(void);", + em.name.replace(".", "_") + )); + } else { + lines.push(ambiguous_line); + } } lines.push(String::from("struct _inittab _PyImport_Inittab[] = {")); + let mut ambiguous_init_fns: Vec = Vec::new(); + for em in extension_modules.values() { if let Some(init_fn) = &em.init_fn { if init_fn == "NULL" { continue; } - lines.push(format!("{{\"{}\", {}}},", em.module, init_fn)); + if init_fn == "PyInit__queue" && built_extension_modules.contains_key("gevent._queue") { + lines.push("{\"_queue\", PyInit_stdlib_queue},".to_string()); + } else { + lines.push(format!("{{\"{}\", {}}},", em.module, init_fn)); + ambiguous_init_fns.push(init_fn.to_string()); + } } } for em in built_extension_modules.values() { - lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn)); + if ambiguous_init_fns.contains(&em.init_fn) { + lines.push(format!( + "{{\"{}\", PyInit_{}}},", + em.name, + em.name.replace(".", "_") + )); + } else { + lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn)); + ambiguous_init_fns.push(em.init_fn.clone()); + } } lines.push(String::from("{0, 0}")); @@ -265,12 +296,22 @@ pub fn link_libpython( // TODO handle static/dynamic libraries. } + let mut ambiguous_init_fns: Vec = Vec::new(); + warn!( logger, "resolving inputs for {} extension modules...", extension_modules.len() + built_extension_modules.len() ); for (name, em) in extension_modules { + if let Some(init_fn) = &em.init_fn { + if init_fn == "PyInit__queue" && built_extension_modules.contains_key("gevent._queue") { + ambiguous_init_fns.push("PyInit_stdlib_queue".to_string()); + } else if init_fn != "NULL" { + ambiguous_init_fns.push(init_fn.to_string()); + } + } + if em.builtin_default { continue; } @@ -283,7 +324,23 @@ pub fn link_libpython( em.object_paths ); for path in &em.object_paths { - build.object(path); + let mut out_path = path.clone(); + if (path.ends_with("_queuemodule.o") || path.ends_with("_queuemodule.obj")) + && built_extension_modules.contains_key("gevent._queue") + { + out_path = temp_dir_path.join(format!("{}_stdlib_prefixed.o", path.display())); + + let file = fs::File::open(&path).unwrap(); + let object_data = unsafe { Mmap::map(&file).unwrap() }; + + match rename_init(logger, &"stdlib_queue".to_string(), &object_data) { + Ok(val) => fs::write(&out_path, val).expect("unable to write object file"), + Err(err) => { + println!("Failed to rename symbol in '{}': {}", path.display(), err); + } + }; + } + build.object(out_path); } for entry in &em.links { @@ -319,10 +376,21 @@ pub fn link_libpython( em.object_file_data.len(), name ); + for (i, object_data) in em.object_file_data.iter().enumerate() { let out_path = temp_dir_path.join(format!("{}.{}.o", name, i)); - fs::write(&out_path, object_data).expect("unable to write object file"); + if i == em.object_file_data.len() - 1 && ambiguous_init_fns.contains(&em.init_fn) { + match rename_init(logger, name, object_data) { + Ok(val) => fs::write(&out_path, val).expect("unable to write object file"), + Err(_) => { + fs::write(&out_path, object_data).expect("unable to write object file") + } + }; + } else { + fs::write(&out_path, object_data).expect("unable to write object file"); + } + build.object(&out_path); } @@ -331,6 +399,10 @@ pub fn link_libpython( needed_libraries_external.insert(&library); } + if !ambiguous_init_fns.contains(&em.init_fn) { + ambiguous_init_fns.push(em.init_fn.clone()); + } + // TODO do something with library_dirs. } diff --git a/pyoxidizer/src/py_packaging/mod.rs b/pyoxidizer/src/py_packaging/mod.rs index 48c5ec9df..e08dc16a5 100644 --- a/pyoxidizer/src/py_packaging/mod.rs +++ b/pyoxidizer/src/py_packaging/mod.rs @@ -9,6 +9,7 @@ pub mod distutils; pub mod embedded_resource; pub mod fsscan; pub mod libpython; +pub mod object; pub mod pip; pub mod pyembed; pub mod resource; diff --git a/pyoxidizer/src/py_packaging/object.rs b/pyoxidizer/src/py_packaging/object.rs new file mode 100644 index 000000000..e77dc6b22 --- /dev/null +++ b/pyoxidizer/src/py_packaging/object.rs @@ -0,0 +1,175 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use object::{write, Object, ObjectSection, RelocationTarget, SectionKind, SymbolKind}; +use slog::{info, warn}; +use std::collections::HashMap; +use std::error::Error; +use std::fmt; + +#[derive(Debug, Clone)] +pub struct NoRewriteError; + +impl fmt::Display for NoRewriteError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "no object rewriting was performed") + } +} + +impl Error for NoRewriteError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + // Generic error, underlying cause isn't tracked. + None + } +} + +/// Rename object syn PyInit_foo to PyInit_ to avoid clashes +pub fn rename_init( + logger: &slog::Logger, + name: &String, + object_data: &[u8], +) -> Result, NoRewriteError> { + let mut rewritten = false; + + let name_prefix = name.split('.').next().unwrap(); + + let in_object = match object::File::parse(object_data) { + Ok(object) => object, + Err(err) => { + let magic = [ + object_data[0], + object_data[1], + object_data[2], + object_data[3], + ]; + warn!( + logger, + "Failed to parse compiled object for {} (magic {:x?}): {}", name, magic, err + ); + return Err(NoRewriteError); + } + }; + + let mut out_object = write::Object::new(in_object.format(), in_object.architecture()); + + let mut out_sections = HashMap::new(); + for in_section in in_object.sections() { + if in_section.kind() == SectionKind::Metadata { + continue; + } + let section_id = out_object.add_section( + in_section.segment_name().unwrap_or("").as_bytes().to_vec(), + in_section.name().unwrap_or("").as_bytes().to_vec(), + in_section.kind(), + ); + let out_section = out_object.section_mut(section_id); + if out_section.is_bss() { + out_section.append_bss(in_section.size(), in_section.align()); + } else { + out_section.set_data(in_section.uncompressed_data().into(), in_section.align()); + } + out_sections.insert(in_section.index(), section_id); + } + + let mut out_symbols = HashMap::new(); + for (symbol_index, in_symbol) in in_object.symbols() { + if in_symbol.kind() == SymbolKind::Null { + // This is normal in ELF + info!( + logger, + "object symbol name kind 'null' discarded", + ); + continue; + } + let in_sym_name = in_symbol.name().unwrap_or(""); + if in_symbol.kind() == SymbolKind::Unknown { + warn!( + logger, + "object symbol name {} kind 'unknown' encountered", in_sym_name, + ); + } + let (section, value) = match in_symbol.section_index() { + Some(index) => ( + Some(*out_sections.get(&index).unwrap()), + in_symbol.address() - in_object.section_by_index(index).unwrap().address(), + ), + None => (None, in_symbol.address()), + }; + let sym_name = if !in_sym_name.starts_with("$") + && in_sym_name.contains("PyInit_") + && !in_sym_name.contains(name_prefix) + { + "PyInit_".to_string() + &name.replace(".", "_") + } else { + String::from(in_sym_name) + }; + if sym_name != in_sym_name { + warn!( + logger, + "renaming object symbol name {} to {}", in_sym_name, sym_name, + ); + + rewritten = true; + } + + let out_symbol = write::Symbol { + name: sym_name.as_bytes().to_vec(), + value, + size: in_symbol.size(), + kind: in_symbol.kind(), + scope: in_symbol.scope(), + weak: in_symbol.is_weak(), + section, + }; + + let symbol_id = out_object.add_symbol(out_symbol); + out_symbols.insert(symbol_index, symbol_id); + info!( + logger, + "added object symbol name {} kind {:?}", sym_name, in_symbol, + ); + } + + if !rewritten { + warn!(logger, "no symbol name rewriting occurred for {}", name); + return Err(NoRewriteError); + } + + for in_section in in_object.sections() { + if in_section.kind() == SectionKind::Metadata { + continue; + } + let out_section = *out_sections.get(&in_section.index()).unwrap(); + for (offset, in_relocation) in in_section.relocations() { + let symbol = match in_relocation.target() { + RelocationTarget::Symbol(symbol) => *out_symbols.get(&symbol).unwrap(), + RelocationTarget::Section(section) => { + out_object.section_symbol(*out_sections.get(§ion).unwrap()) + } + }; + let out_relocation = write::Relocation { + offset, + size: in_relocation.size(), + kind: in_relocation.kind(), + encoding: in_relocation.encoding(), + symbol, + addend: in_relocation.addend(), + }; + out_object + .add_relocation(out_section, out_relocation) + .unwrap(); + } + } + + info!(logger, "serialising object for {} ..", name); + + match out_object.write() { + Ok(obj) => Ok(obj), + Err(err) => { + warn!(logger, "object {} serialisation failed: {}", name, err); + + Err(NoRewriteError) + } + } +} diff --git a/pyoxidizer/src/py_packaging/pip.rs b/pyoxidizer/src/py_packaging/pip.rs index d2044b17d..9482f935a 100644 --- a/pyoxidizer/src/py_packaging/pip.rs +++ b/pyoxidizer/src/py_packaging/pip.rs @@ -6,14 +6,18 @@ use slog::warn; use std::collections::HashMap; use std::convert::TryFrom; use std::io::{BufRead, BufReader}; -use std::path::PathBuf; -use super::distribution::ParsedPythonDistribution; +use super::distribution::{ParsedPythonDistribution}; use super::distutils::{prepare_hacked_distutils, read_built_extensions}; use super::fsscan::{find_python_resources, PythonFileResource}; use super::resource::PythonResource; /// Run `pip install` and return found resources. +/// This is unused at the moment as it is based around pip install --target +/// instead of `venv`s. +/// It doesnt use prepare_hacked_distutils as that isnt needed for its test +/// case, and a rewrite is needed to support core Python modules like +/// cffi, cryptography, Cython, PyNaCl, etc. pub fn pip_install( logger: &slog::Logger, dist: &ParsedPythonDistribution, @@ -24,9 +28,9 @@ pub fn pip_install( let temp_dir = tempdir::TempDir::new("pyoxidizer-pip-install") .or_else(|_| Err("could not create temp directory".to_string()))?; - dist.ensure_pip(); + dist.ensure_pip(&logger); - let mut env = prepare_hacked_distutils(logger, dist, temp_dir.path(), &[])?; + let mut env = HashMap::new(); for (key, value) in extra_envs.iter() { env.insert(key.clone(), value.clone()); @@ -98,10 +102,5 @@ pub fn pip_install( } } - let state_dir = PathBuf::from(env.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()); - for ext in read_built_extensions(&state_dir)? { - res.push(PythonResource::BuiltExtensionModule(ext)); - } - Ok(res) } diff --git a/pyoxidizer/src/starlark/python_packaging.rs b/pyoxidizer/src/starlark/python_packaging.rs index 65516d8b3..d6710f2eb 100644 --- a/pyoxidizer/src/starlark/python_packaging.rs +++ b/pyoxidizer/src/starlark/python_packaging.rs @@ -16,8 +16,8 @@ use std::cmp::Ordering; use std::collections::HashMap; use super::env::{ - optional_dict_arg, optional_list_arg, required_bool_arg, required_list_arg, required_str_arg, - required_type_arg, + optional_dict_arg, optional_list_arg, optional_str_arg, required_bool_arg, required_list_arg, + required_str_arg, required_type_arg, }; use crate::app_packaging::config::{ resolve_install_location, PackagingFilterInclude, PackagingPackageRoot, @@ -519,6 +519,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] PipInstallSimple( package, + venv_path=None, extra_env=None, optimize_level=0, excludes=None, @@ -527,6 +528,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let package = required_str_arg("package", &package)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; optional_list_arg("excludes", "string", &excludes)?; @@ -565,6 +567,7 @@ starlark_module! { python_packaging_env => let rule = PackagingPipInstallSimple { package, + venv_path, extra_env, optimize_level, excludes, @@ -579,6 +582,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] PipRequirementsFile( requirements_path, + venv_path=None, extra_env=None, optimize_level=0, include_source=true, @@ -586,6 +590,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let requirements_path = required_str_arg("path", &requirements_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; let include_source = required_bool_arg("include_source", &include_source)?; @@ -618,6 +623,7 @@ starlark_module! { python_packaging_env => }; let rule = PackagingPipRequirementsFile { + venv_path, requirements_path, extra_env, optimize_level, @@ -632,6 +638,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] SetupPyInstall( package_path, + venv_path=None, extra_env=None, extra_global_arguments=None, optimize_level=0, @@ -640,6 +647,7 @@ starlark_module! { python_packaging_env => excludes=None ) { let package_path = required_str_arg("package_path", &package_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; optional_list_arg("extra_global_arguments", "string", &extra_global_arguments)?; required_type_arg("optimize_level", "int", &optimize_level)?; @@ -676,6 +684,7 @@ starlark_module! { python_packaging_env => let rule = PackagingSetupPyInstall { path: package_path, + venv_path, extra_env, extra_global_arguments, optimize_level: optimize_level.to_int().unwrap(), @@ -890,6 +899,7 @@ mod tests { let v = starlark_ok("PipInstallSimple('foo')"); let wanted = PackagingPipInstallSimple { package: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, excludes: Vec::new(), @@ -914,6 +924,7 @@ mod tests { let v = starlark_ok("PipRequirementsFile('path')"); let wanted = PackagingPipRequirementsFile { requirements_path: "path".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, include_source: true, @@ -928,6 +939,7 @@ mod tests { fn test_pip_requirements_file_extra_args() { let v = starlark_ok("PipRequirementsFile('path', extra_args=['foo'])"); let wanted = PackagingPipRequirementsFile { + venv_path: None, requirements_path: "path".to_string(), extra_env: HashMap::new(), optimize_level: 0, @@ -950,6 +962,7 @@ mod tests { let v = starlark_ok("SetupPyInstall('foo')"); let wanted = PackagingSetupPyInstall { path: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), extra_global_arguments: Vec::new(), optimize_level: 0,