From c99b271b48f31f6447197ec1f2598aa4a8ffcd40 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 22 Nov 2019 20:17:30 +0700 Subject: [PATCH 01/20] Remove cl.exe option /GL This causes the .obj files to be unparsable by editbin, dumpbin, nm, objcopy, goblin and object. Related to https://github.com/indygreg/PyOxidizer/issues/169 --- pyoxidizer/src/distutils/_msvccompiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 88e482aad..b09896139 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -265,7 +265,7 @@ def initialize(self, plat_name=None): # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib # later to dynamically link to ucrtbase but not vcruntime. self.compile_options = [ - '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG' + '/nologo', '/Ox', '/W3', '/DNDEBUG' ] self.compile_options.append('/MD' if self._vcruntime_redist else '/MT') From 0ca173bbf3ce992374101af0ef69bb03ecd055d4 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 8 Nov 2019 19:30:11 +0700 Subject: [PATCH 02/20] Rename PyInit syms to avoid clashes Built extensions in packages often have common names like speedups, utils, _objects, cpython, etc. which reside inside the package namespace. The compiled extensions each have a PyInit_ which needs to be renamed to PyInit__ to avoid clashes when combined into a static binary. Fixes https://github.com/indygreg/PyOxidizer/issues/169 --- ci/azure-pipelines-template.yml | 4 +- ci/pyapp.py | 6 + pyoxidizer/Cargo.toml | 1 + pyoxidizer/src/py_packaging/libpython.rs | 51 ++++++- pyoxidizer/src/py_packaging/mod.rs | 1 + pyoxidizer/src/py_packaging/object.rs | 175 +++++++++++++++++++++++ 6 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 pyoxidizer/src/py_packaging/object.rs diff --git a/ci/azure-pipelines-template.yml b/ci/azure-pipelines-template.yml index f3aa9d5f5..1bc324470 100644 --- a/ci/azure-pipelines-template.yml +++ b/ci/azure-pipelines-template.yml @@ -49,12 +49,12 @@ jobs: - ${{ if ne(parameters.name, 'Windows') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 ~/pyapp + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 ~/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp displayName: Build Oxidized Application - ${{ if eq(parameters.name, 'Windows') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 %USERPROFILE%/pyapp + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 %USERPROFILE%/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run %USERPROFILE%/pyapp displayName: Build Oxidized Application (Windows) diff --git a/ci/pyapp.py b/ci/pyapp.py index ae9a27407..ed171d1aa 100644 --- a/ci/pyapp.py +++ b/ci/pyapp.py @@ -8,4 +8,10 @@ except AttributeError: pass +import markupsafe._speedups +import simplejson._speedups + +import markupsafe +import simplejson + print("hello, world") diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index ddfd7772c..ace8124a5 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -37,6 +37,7 @@ hex = "0.3" itertools = "0.8" lazy_static = "1.3" libc = "0.2" +object = { version = "0.16.0", features = ["read", "std", "write"] } regex = "1" reqwest = "0.9" rustc_version = "0.2" diff --git a/pyoxidizer/src/py_packaging/libpython.rs b/pyoxidizer/src/py_packaging/libpython.rs index 6d559201d..fd02bc343 100644 --- a/pyoxidizer/src/py_packaging/libpython.rs +++ b/pyoxidizer/src/py_packaging/libpython.rs @@ -13,6 +13,7 @@ use std::path::{Path, PathBuf}; use super::bytecode::{BytecodeCompiler, CompileMode}; use super::distribution::{ExtensionModule, LicenseInfo, ParsedPythonDistribution}; use super::embedded_resource::EmbeddedPythonResources; +use super::object::rename_init; use super::resource::BuiltExtensionModule; pub const PYTHON_IMPORTER: &[u8] = include_bytes!("memoryimporter.py"); @@ -107,11 +108,22 @@ pub fn make_config_c( } for em in built_extension_modules.values() { - lines.push(format!("extern PyObject* {}(void);", em.init_fn)); + let ambiguous_line = format!("extern PyObject* {}(void);", em.init_fn); + + if lines.contains(&ambiguous_line) { + lines.push(format!( + "extern PyObject* PyInit_{}(void);", + em.name.replace(".", "_") + )); + } else { + lines.push(ambiguous_line); + } } lines.push(String::from("struct _inittab _PyImport_Inittab[] = {")); + let mut ambiguous_init_fns: Vec = Vec::new(); + for em in extension_modules.values() { if let Some(init_fn) = &em.init_fn { if init_fn == "NULL" { @@ -119,11 +131,21 @@ pub fn make_config_c( } lines.push(format!("{{\"{}\", {}}},", em.module, init_fn)); + ambiguous_init_fns.push(init_fn.to_string()); } } for em in built_extension_modules.values() { - lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn)); + if ambiguous_init_fns.contains(&em.init_fn) { + lines.push(format!( + "{{\"{}\", PyInit_{}}},", + em.name, + em.name.replace(".", "_") + )); + } else { + lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn)); + ambiguous_init_fns.push(em.init_fn.clone()); + } } lines.push(String::from("{0, 0}")); @@ -265,12 +287,20 @@ pub fn link_libpython( // TODO handle static/dynamic libraries. } + let mut ambiguous_init_fns: Vec = Vec::new(); + warn!( logger, "resolving inputs for {} extension modules...", extension_modules.len() + built_extension_modules.len() ); for (name, em) in extension_modules { + if let Some(init_fn) = &em.init_fn { + if init_fn != "NULL" { + ambiguous_init_fns.push(init_fn.to_string()); + } + } + if em.builtin_default { continue; } @@ -319,10 +349,21 @@ pub fn link_libpython( em.object_file_data.len(), name ); + for (i, object_data) in em.object_file_data.iter().enumerate() { let out_path = temp_dir_path.join(format!("{}.{}.o", name, i)); - fs::write(&out_path, object_data).expect("unable to write object file"); + if i == em.object_file_data.len() - 1 && ambiguous_init_fns.contains(&em.init_fn) { + match rename_init(logger, name, object_data) { + Ok(val) => fs::write(&out_path, val).expect("unable to write object file"), + Err(_) => { + fs::write(&out_path, object_data).expect("unable to write object file") + } + }; + } else { + fs::write(&out_path, object_data).expect("unable to write object file"); + } + build.object(&out_path); } @@ -331,6 +372,10 @@ pub fn link_libpython( needed_libraries_external.insert(&library); } + if !ambiguous_init_fns.contains(&em.init_fn) { + ambiguous_init_fns.push(em.init_fn.clone()); + } + // TODO do something with library_dirs. } diff --git a/pyoxidizer/src/py_packaging/mod.rs b/pyoxidizer/src/py_packaging/mod.rs index 48c5ec9df..e08dc16a5 100644 --- a/pyoxidizer/src/py_packaging/mod.rs +++ b/pyoxidizer/src/py_packaging/mod.rs @@ -9,6 +9,7 @@ pub mod distutils; pub mod embedded_resource; pub mod fsscan; pub mod libpython; +pub mod object; pub mod pip; pub mod pyembed; pub mod resource; diff --git a/pyoxidizer/src/py_packaging/object.rs b/pyoxidizer/src/py_packaging/object.rs new file mode 100644 index 000000000..e77dc6b22 --- /dev/null +++ b/pyoxidizer/src/py_packaging/object.rs @@ -0,0 +1,175 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use object::{write, Object, ObjectSection, RelocationTarget, SectionKind, SymbolKind}; +use slog::{info, warn}; +use std::collections::HashMap; +use std::error::Error; +use std::fmt; + +#[derive(Debug, Clone)] +pub struct NoRewriteError; + +impl fmt::Display for NoRewriteError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "no object rewriting was performed") + } +} + +impl Error for NoRewriteError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + // Generic error, underlying cause isn't tracked. + None + } +} + +/// Rename object syn PyInit_foo to PyInit_ to avoid clashes +pub fn rename_init( + logger: &slog::Logger, + name: &String, + object_data: &[u8], +) -> Result, NoRewriteError> { + let mut rewritten = false; + + let name_prefix = name.split('.').next().unwrap(); + + let in_object = match object::File::parse(object_data) { + Ok(object) => object, + Err(err) => { + let magic = [ + object_data[0], + object_data[1], + object_data[2], + object_data[3], + ]; + warn!( + logger, + "Failed to parse compiled object for {} (magic {:x?}): {}", name, magic, err + ); + return Err(NoRewriteError); + } + }; + + let mut out_object = write::Object::new(in_object.format(), in_object.architecture()); + + let mut out_sections = HashMap::new(); + for in_section in in_object.sections() { + if in_section.kind() == SectionKind::Metadata { + continue; + } + let section_id = out_object.add_section( + in_section.segment_name().unwrap_or("").as_bytes().to_vec(), + in_section.name().unwrap_or("").as_bytes().to_vec(), + in_section.kind(), + ); + let out_section = out_object.section_mut(section_id); + if out_section.is_bss() { + out_section.append_bss(in_section.size(), in_section.align()); + } else { + out_section.set_data(in_section.uncompressed_data().into(), in_section.align()); + } + out_sections.insert(in_section.index(), section_id); + } + + let mut out_symbols = HashMap::new(); + for (symbol_index, in_symbol) in in_object.symbols() { + if in_symbol.kind() == SymbolKind::Null { + // This is normal in ELF + info!( + logger, + "object symbol name kind 'null' discarded", + ); + continue; + } + let in_sym_name = in_symbol.name().unwrap_or(""); + if in_symbol.kind() == SymbolKind::Unknown { + warn!( + logger, + "object symbol name {} kind 'unknown' encountered", in_sym_name, + ); + } + let (section, value) = match in_symbol.section_index() { + Some(index) => ( + Some(*out_sections.get(&index).unwrap()), + in_symbol.address() - in_object.section_by_index(index).unwrap().address(), + ), + None => (None, in_symbol.address()), + }; + let sym_name = if !in_sym_name.starts_with("$") + && in_sym_name.contains("PyInit_") + && !in_sym_name.contains(name_prefix) + { + "PyInit_".to_string() + &name.replace(".", "_") + } else { + String::from(in_sym_name) + }; + if sym_name != in_sym_name { + warn!( + logger, + "renaming object symbol name {} to {}", in_sym_name, sym_name, + ); + + rewritten = true; + } + + let out_symbol = write::Symbol { + name: sym_name.as_bytes().to_vec(), + value, + size: in_symbol.size(), + kind: in_symbol.kind(), + scope: in_symbol.scope(), + weak: in_symbol.is_weak(), + section, + }; + + let symbol_id = out_object.add_symbol(out_symbol); + out_symbols.insert(symbol_index, symbol_id); + info!( + logger, + "added object symbol name {} kind {:?}", sym_name, in_symbol, + ); + } + + if !rewritten { + warn!(logger, "no symbol name rewriting occurred for {}", name); + return Err(NoRewriteError); + } + + for in_section in in_object.sections() { + if in_section.kind() == SectionKind::Metadata { + continue; + } + let out_section = *out_sections.get(&in_section.index()).unwrap(); + for (offset, in_relocation) in in_section.relocations() { + let symbol = match in_relocation.target() { + RelocationTarget::Symbol(symbol) => *out_symbols.get(&symbol).unwrap(), + RelocationTarget::Section(section) => { + out_object.section_symbol(*out_sections.get(§ion).unwrap()) + } + }; + let out_relocation = write::Relocation { + offset, + size: in_relocation.size(), + kind: in_relocation.kind(), + encoding: in_relocation.encoding(), + symbol, + addend: in_relocation.addend(), + }; + out_object + .add_relocation(out_section, out_relocation) + .unwrap(); + } + } + + info!(logger, "serialising object for {} ..", name); + + match out_object.write() { + Ok(obj) => Ok(obj), + Err(err) => { + warn!(logger, "object {} serialisation failed: {}", name, err); + + Err(NoRewriteError) + } + } +} From e0b31b320d7d82822fcf7bee95b12f02f5bab832 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 23 Nov 2019 09:20:52 +0700 Subject: [PATCH 03/20] CI: Replace image macOS-10.14 with macOS-10.13 Python import of simplejson fails with a UnicodeDecodeError on macOS-10.14 when its PyInit symbol is renamed. --- ci/azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index a0db3233b..7134c89d1 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -7,7 +7,7 @@ jobs: - template: azure-pipelines-template.yml parameters: name: macOS - vmImage: macOS-10.14 + vmImage: macOS-10.13 - template: azure-pipelines-template.yml parameters: From 27e7805732b97df52ba5279cd47c064e9e9de312 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 25 Nov 2019 07:29:44 +0700 Subject: [PATCH 04/20] distutils: Discard cffi's libffi on MSVC cffi includes a near replica of the CPython 3.7's libffi on MSVC, which needs to be discard to avoid duplicate syms. --- pyoxidizer/src/distutils/_msvccompiler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index b09896139..503b7b632 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -586,6 +586,11 @@ def extension_link_shared_object(self, # directory that doesn't outlive this process. object_paths = [] for i, o in enumerate(objects): + if 'libffi_msvc' in o: + print('Ignored static {}'.format(o)) + # https://github.com/indygreg/python-build-standalone/issues/23 + # cffi includes a near replica of CPython's custom libffi. + continue p = os.path.join(dest_path, '%s.%d.o' % (name, i)) shutil.copyfile(o, p) object_paths.append(p) From 873e9c7068546e3656bf51360b2930c521d98c25 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 16 Nov 2019 20:52:47 +0700 Subject: [PATCH 05/20] Use reusable venvs Replace pip install --target with venvs. As setuptools breakes out of the venv to load distutils build_ext command, first create a copy of the Python distribution and hack it. This also allows venv's to be populated manually outside of PyOxidizer's config, while retaining the ability for PyOxidizer to pick up and include any built object files. To facilitate that, the PyOxidizer state dir where objects are stored is a constant in the distutils compiler modules and in the Rust source. Also add venv_path to PipRequirementsFile, allowing the same venv to be incrementally populated in multiple rules, and the venv re-used across PyOxidizer build runs. Fixes https://github.com/indygreg/PyOxidizer/issues/162 Fixes https://github.com/indygreg/PyOxidizer/issues/170 Closes https://github.com/indygreg/PyOxidizer/issues/194 --- pyoxidizer/Cargo.toml | 1 + pyoxidizer/src/app_packaging/config.rs | 3 + .../src/app_packaging/packaging_rule.rs | 675 +++++++----------- pyoxidizer/src/distutils/_msvccompiler.py | 10 +- pyoxidizer/src/distutils/unixccompiler.py | 10 +- pyoxidizer/src/py_packaging/distribution.rs | 246 ++++++- pyoxidizer/src/py_packaging/distutils.rs | 63 +- pyoxidizer/src/starlark/python_packaging.rs | 17 +- 8 files changed, 530 insertions(+), 495 deletions(-) diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index ddfd7772c..caff3c44d 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -27,6 +27,7 @@ cc = "1.0" clap = "2.32" codemap = "0.1" codemap-diagnostic = "0.1" +copy_dir = "0.1.2" encoding_rs = "0.8" fs2 = "0.4" git2 = "0.9" diff --git a/pyoxidizer/src/app_packaging/config.rs b/pyoxidizer/src/app_packaging/config.rs index 80e0ce2cd..a18089914 100644 --- a/pyoxidizer/src/app_packaging/config.rs +++ b/pyoxidizer/src/app_packaging/config.rs @@ -26,6 +26,7 @@ pub enum InstallLocation { #[derive(Clone, Debug, PartialEq)] pub struct PackagingSetupPyInstall { pub path: String, + pub venv_path: Option, pub extra_env: HashMap, pub extra_global_arguments: Vec, pub optimize_level: i64, @@ -87,6 +88,7 @@ pub struct PackagingPackageRoot { #[derive(Clone, Debug, PartialEq)] pub struct PackagingPipInstallSimple { pub package: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub excludes: Vec, @@ -99,6 +101,7 @@ pub struct PackagingPipInstallSimple { pub struct PackagingPipRequirementsFile { // TODO resolve to a PathBuf. pub requirements_path: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub include_source: bool, diff --git a/pyoxidizer/src/app_packaging/packaging_rule.rs b/pyoxidizer/src/app_packaging/packaging_rule.rs index 032066e1d..4c4557d2a 100644 --- a/pyoxidizer/src/app_packaging/packaging_rule.rs +++ b/pyoxidizer/src/app_packaging/packaging_rule.rs @@ -15,12 +15,13 @@ use super::config::{ PackagingStdlibExtensionsPolicy, PackagingVirtualenv, PythonPackaging, }; use super::state::BuildContext; -use crate::py_packaging::distribution::{is_stdlib_test_package, ParsedPythonDistribution}; -use crate::py_packaging::distutils::{prepare_hacked_distutils, read_built_extensions}; +use crate::py_packaging::distribution::{ + is_stdlib_test_package, resolve_python_paths, ParsedPythonDistribution, +}; +use crate::py_packaging::distutils::read_built_extensions; use crate::py_packaging::fsscan::{ find_python_resources, is_package_from_path, PythonFileResource, }; -use crate::py_packaging::pip::pip_install; use crate::py_packaging::resource::{AppRelativeResources, PythonResource}; #[derive(Debug)] @@ -119,30 +120,6 @@ fn python_resource_full_name(resource: &PythonResource) -> String { } } -struct PythonPaths { - main: PathBuf, - site_packages: PathBuf, -} - -/// Resolve the location of Python modules given a base install path. -fn resolve_python_paths(base: &Path, python_version: &str, is_windows: bool) -> PythonPaths { - let mut p = base.to_path_buf(); - - if is_windows { - p.push("Lib"); - } else { - p.push("lib"); - p.push(format!("python{}", &python_version[0..3])); - } - - let site_packages = p.join("site-packages"); - - PythonPaths { - main: p, - site_packages, - } -} - fn resolve_built_extensions( state_dir: &Path, res: &mut Vec, @@ -159,6 +136,122 @@ fn resolve_built_extensions( Ok(()) } +/// Processes resources in a path +/// Args includes and excludes are ignored if None or an empty Vec. +fn process_resources( + logger: &slog::Logger, + path: &PathBuf, + location: &ResourceLocation, + state_dir: Option<&PathBuf>, + include_source: bool, + optimize_level: i64, + includes: Option<&Vec>, + excludes: Option<&Vec>, +) -> Vec { + let mut res = Vec::new(); + + let path_s = path.display().to_string(); + warn!(logger, "processing resources from {}", path_s); + + for resource in find_python_resources(path) { + let full_name = resource_full_name(&resource); + + let excluded = match includes { + Some(values) => values.iter().any(|v| { + let prefix = v.clone() + "."; + full_name != v && !full_name.starts_with(&prefix) + }), + None => false, + }; + + if excluded { + info!( + logger, + "whitelist skipping {}", full_name + ); + continue; + } + + let excluded = match excludes { + Some(values) => match values.is_empty() { + true => false, + false => values.iter().all(|v| { + let prefix = v.clone() + "."; + full_name == v || full_name.starts_with(&prefix) + }), + }, + None => false, + }; + + if excluded { + info!( + logger, + "blacklist skipping {}", full_name + ); + continue; + } + + match resource { + PythonFileResource::Source { + full_name, path, .. + } => { + let is_package = is_package_from_path(&path); + let source = fs::read(path).expect("error reading source file"); + + if include_source { + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleSource { + name: full_name.clone(), + source: source.clone(), + is_package, + }, + }); + } + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleBytecodeRequest { + name: full_name.clone(), + source, + optimize_level: optimize_level as i32, + is_package, + }, + }); + } + + PythonFileResource::Resource(resource) => { + let data = fs::read(resource.path).expect("error reading resource file"); + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::Resource { + package: resource.package.clone(), + name: resource.stem.clone(), + data, + }, + }); + } + + _ => {} + } + } + + match state_dir { + Some(dir) => { + if dir.exists() { + resolve_built_extensions(&dir, &mut res, &location).unwrap(); + } + } + None => {} + }; + + res +} + fn resolve_stdlib_extensions_policy( logger: &slog::Logger, dist: &ParsedPythonDistribution, @@ -343,165 +436,44 @@ fn resolve_stdlib( } fn resolve_virtualenv( + logger: &slog::Logger, dist: &ParsedPythonDistribution, rule: &PackagingVirtualenv, ) -> Vec { - let mut res = Vec::new(); - let location = ResourceLocation::new(&rule.install_location); let python_paths = - resolve_python_paths(&Path::new(&rule.path), &dist.version, dist.os == "windows"); - let packages_path = python_paths.site_packages; + resolve_python_paths(&Path::new(&rule.path), &dist.version); - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), + ) } -fn resolve_package_root(rule: &PackagingPackageRoot) -> Vec { - let mut res = Vec::new(); - +fn resolve_package_root( + logger: &slog::Logger, + rule: &PackagingPackageRoot, +) -> Vec { let location = ResourceLocation::new(&rule.install_location); let path = PathBuf::from(&rule.path); - for resource in find_python_resources(&path) { - let mut relevant = false; - let full_name = resource_full_name(&resource); - - for package in &rule.packages { - let prefix = package.clone() + "."; - - if full_name == package || full_name.starts_with(&prefix) { - relevant = true; - } - } - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &path, + &location, + None, + rule.include_source, + rule.optimize_level, + Some(&rule.packages), + None, + ) } fn resolve_pip_install_simple( @@ -510,97 +482,61 @@ fn resolve_pip_install_simple( rule: &PackagingPipInstallSimple, verbose: bool, ) -> Vec { - let mut install_args = vec![ - "--no-binary".to_string(), - ":all:".to_string(), - rule.package.clone(), - ]; - - if let Some(ref args) = rule.extra_args { - install_args.extend(args.clone()); - } - - let resources = pip_install(logger, dist, verbose, &install_args, &rule.extra_env).unwrap(); - - let mut res = Vec::new(); let location = ResourceLocation::new(&rule.install_location); - for resource in resources { - let mut relevant = true; - - let full_name = python_resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); - if &full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } + let mut pip_args: Vec = vec![ + "-m".to_string(), + "pip".to_string(), + "--disable-pip-version-check".to_string(), + ]; - if !relevant { - continue; - } + if verbose { + pip_args.push("--verbose".to_string()); + } - match resource { - PythonResource::ModuleSource { - name, - is_package, - source, - } => { - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: name.clone(), - source: source.clone(), - is_package, - }, - }); - } + pip_args.extend(vec![ + "install".to_string(), + "--no-binary".to_string(), + ":all:".to_string(), + rule.package.clone(), + ]); - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name, - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } + if let Some(ref args) = rule.extra_args { + pip_args.extend(args.clone()); + } - PythonResource::Resource { - package, - name, - data, - } => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package, - name, - data, - }, - }); - } + for (key, value) in rule.extra_env.iter() { + extra_envs.insert(key.clone(), value.clone()); + } - PythonResource::BuiltExtensionModule(em) => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::BuiltExtensionModule(em), - }); - } + // TODO send stderr to stdout. + let mut cmd = std::process::Command::new(&python_paths.python_exe) + .args(&pip_args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("error running pip"); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); - _ => {} + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); } } - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), + ) } fn resolve_pip_requirements_file( @@ -609,83 +545,69 @@ fn resolve_pip_requirements_file( rule: &PackagingPipRequirementsFile, verbose: bool, ) -> Vec { - let mut install_args = vec![ + let location = ResourceLocation::new(&rule.install_location); + + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); + + let mut pip_args: Vec = vec![ + "-m".to_string(), + "pip".to_string(), + "--disable-pip-version-check".to_string(), + ]; + + if verbose { + pip_args.push("--verbose".to_string()); + } + + pip_args.extend(vec![ + "install".to_string(), "--no-binary".to_string(), ":all:".to_string(), "--requirement".to_string(), rule.requirements_path.to_string(), - ]; + ]); if let Some(ref args) = rule.extra_args { - install_args.extend(args.clone()); + pip_args.extend(args.clone()); } - let resources = pip_install(logger, dist, verbose, &install_args, &rule.extra_env).unwrap(); - - let location = ResourceLocation::new(&rule.install_location); - - let mut res = Vec::new(); - - for resource in resources { - match resource { - PythonResource::ModuleSource { - name, - is_package, - source, - } => { - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name, - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } + for (key, value) in rule.extra_env.iter() { + extra_envs.insert(key.clone(), value.clone()); + } - PythonResource::Resource { - package, - name, - data, - } => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package, - name, - data, - }, - }); - } + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + pip_args.join(" ") + ); - PythonResource::BuiltExtensionModule(em) => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::BuiltExtensionModule(em), - }); - } + // TODO send stderr to stdout. + let mut cmd = std::process::Command::new(&python_paths.python_exe) + .args(&pip_args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("error running pip"); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); - _ => {} + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); } } - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + None, + ) } fn resolve_setup_py_install( @@ -695,8 +617,6 @@ fn resolve_setup_py_install( rule: &PackagingSetupPyInstall, verbose: bool, ) -> Vec { - let mut res = Vec::new(); - // Execution directory is resolved relative to the active configuration // file unless it is absolute. let rule_path = PathBuf::from(&rule.path); @@ -709,31 +629,12 @@ fn resolve_setup_py_install( let location = ResourceLocation::new(&rule.install_location); - let temp_dir = tempdir::TempDir::new("pyoxidizer-setup-py-install") - .expect("could not create temp directory"); - - let target_dir_path = temp_dir.path().join("install"); - let target_dir_s = target_dir_path.display().to_string(); - - let python_paths = resolve_python_paths(&target_dir_path, &dist.version, dist.os == "windows"); - - std::fs::create_dir_all(&python_paths.site_packages) - .expect("unable to create site-packages directory"); - - let mut extra_envs = prepare_hacked_distutils( - logger, - dist, - temp_dir.path(), - &[&python_paths.site_packages, &python_paths.main], - ) - .expect("unable to hack distutils"); + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); for (key, value) in rule.extra_env.iter() { extra_envs.insert(key.clone(), value.clone()); } - warn!(logger, "python setup.py installing to {}", target_dir_s); - let mut args = vec!["setup.py"]; for arg in &rule.extra_global_arguments { @@ -744,10 +645,19 @@ fn resolve_setup_py_install( args.push("--verbose"); } - args.extend(&["install", "--prefix", &target_dir_s, "--no-compile"]); + let prefix_dir_s = python_paths.prefix.display().to_string(); + + args.extend(&["install", "--prefix", &prefix_dir_s, "--no-compile"]); + + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); // TODO send stderr to stdout. - let mut cmd = std::process::Command::new(&dist.python_exe) + let mut cmd = std::process::Command::new(&python_paths.python_exe) .current_dir(cwd) .args(&args) .envs(&extra_envs) @@ -768,81 +678,16 @@ fn resolve_setup_py_install( panic!("error running setup.py"); } - let packages_path = python_paths.site_packages; - - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - resolve_built_extensions( - &PathBuf::from(extra_envs.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()), - &mut res, + process_resources( + &logger, + &python_paths.site_packages, &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), ) - .unwrap(); - - res } /// Resolves a Python packaging rule to resources to package. @@ -872,9 +717,9 @@ pub fn resolve_python_packaging( PythonPackaging::Stdlib(rule) => resolve_stdlib(logger, dist, &rule), - PythonPackaging::Virtualenv(rule) => resolve_virtualenv(dist, &rule), + PythonPackaging::Virtualenv(rule) => resolve_virtualenv(logger, dist, &rule), - PythonPackaging::PackageRoot(rule) => resolve_package_root(&rule), + PythonPackaging::PackageRoot(rule) => resolve_package_root(logger, &rule), PythonPackaging::PipInstallSimple(rule) => { resolve_pip_install_simple(logger, dist, &rule, verbose) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 88e482aad..9bb1c4a98 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -14,6 +14,7 @@ # ported to VS 2015 by Steve Dower import json +import pathlib import os import shutil import stat @@ -562,9 +563,6 @@ def extension_link_shared_object(self, package=None, ): - if 'PYOXIDIZER_DISTUTILS_STATE_DIR' not in os.environ: - raise Exception('PYOXIDIZER_DISTUTILS_STATE_DIR not defined') - # The extension is compiled as a built-in, so linking a shared library # won't work due to symbol visibility/export issues. The extension is # expecting all CPython symbols to be available in the current binary, @@ -580,7 +578,10 @@ def extension_link_shared_object(self, # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. - dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + dest_path = str(pathlib.Path(sys.prefix) / 'state' / 'pyoxidizer') + + if not os.path.exists(dest_path): + os.makedirs(dest_path) # We need to copy the object files because they may be in a temp # directory that doesn't outlive this process. @@ -604,6 +605,7 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/distutils/unixccompiler.py b/pyoxidizer/src/distutils/unixccompiler.py index 26f575fe0..55e615e12 100644 --- a/pyoxidizer/src/distutils/unixccompiler.py +++ b/pyoxidizer/src/distutils/unixccompiler.py @@ -14,6 +14,7 @@ """ import json, os, sys, re, shutil +import pathlib from distutils import sysconfig from distutils.dep_util import newer @@ -224,9 +225,6 @@ def extension_link_shared_object(self, package=None, ): - if 'PYOXIDIZER_DISTUTILS_STATE_DIR' not in os.environ: - raise Exception('PYOXIDIZER_DISTUTILS_STATE_DIR not defined') - self.link(CCompiler.SHARED_OBJECT, objects, output_filename, output_dir, libraries, library_dirs, runtime_library_dirs, @@ -236,7 +234,10 @@ def extension_link_shared_object(self, # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. - dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + dest_path = str(pathlib.Path(sys.prefix) / 'state' / 'pyoxidizer') + + if not os.path.exists(dest_path): + os.makedirs(dest_path) # We need to copy the object files because they may be in a temp # directory that doesn't outlive this process. @@ -260,6 +261,7 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/py_packaging/distribution.rs b/pyoxidizer/src/py_packaging/distribution.rs index 8a2f36a6e..2bdb04b4f 100644 --- a/pyoxidizer/src/py_packaging/distribution.rs +++ b/pyoxidizer/src/py_packaging/distribution.rs @@ -2,31 +2,45 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use copy_dir::copy_dir; use fs2::FileExt; use itertools::Itertools; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use slog::warn; -use std::collections::BTreeMap; +use slog::{info, warn}; +use std::collections::{BTreeMap, HashMap}; +use std::env; use std::fs; use std::fs::{create_dir_all, File}; -use std::io::{Cursor, Read}; +use std::io::{BufRead, BufReader, Cursor, Read}; use std::path::{Path, PathBuf}; use url::Url; use uuid::Uuid; +use which::which; +use super::distutils::prepare_hacked_distutils; use super::fsscan::{ find_python_resources, is_package_from_path, walk_tree_files, PythonFileResource, }; use super::resource::{ResourceData, SourceModule}; + use crate::licensing::NON_GPL_LICENSES; +#[cfg(windows)] +const PYTHON_EXE_BASENAME: &str = "python.exe"; + +#[cfg(unix)] +const PYTHON_EXE_BASENAME: &str = "python3"; + #[cfg(windows)] const PIP_EXE_BASENAME: &str = "pip3.exe"; #[cfg(unix)] const PIP_EXE_BASENAME: &str = "pip3"; +// This needs to be kept in sync with *compiler.py +const PYOXIDIZER_STATE_DIR: &str = "state/pyoxidizer"; + const STDLIB_TEST_PACKAGES: &[&str] = &[ "bsddb.test", "ctypes.test", @@ -311,6 +325,9 @@ pub struct ParsedPythonDistribution { /// Describes license info for things in this distribution. pub license_infos: BTreeMap>, + + /// Path to copy of hacked dist to use for packaging rules venvs + pub venv_base: PathBuf, } #[derive(Debug)] @@ -333,6 +350,102 @@ pub enum ExtensionModuleFilter { NoGPL, } +pub struct PythonPaths { + pub prefix: PathBuf, + pub bin_dir: PathBuf, + pub python_exe: PathBuf, + pub stdlib: PathBuf, + pub site_packages: PathBuf, + pub pyoxidizer_state_dir: PathBuf, +} + +/// Resolve the location of Python modules given a base install path. +pub fn resolve_python_paths(base: &Path, python_version: &str) -> PythonPaths { + let prefix = base.to_path_buf().canonicalize().unwrap(); + + let p = prefix.clone(); + + let bin_dir = if p.join("Scripts").exists() { + p.join("Scripts") + } else { + p.join("bin") + }; + + let python_exe = if bin_dir.join(PYTHON_EXE_BASENAME).exists() { + bin_dir.join(PYTHON_EXE_BASENAME) + } else { + p.join(PYTHON_EXE_BASENAME) + }; + + let mut pyoxidizer_state_dir = p.clone(); + pyoxidizer_state_dir.extend(PYOXIDIZER_STATE_DIR.split('/')); + + let unix_lib_dir = p.join("lib").join(format!("python{}", &python_version[0..3])); + + let stdlib = if unix_lib_dir.exists() { + unix_lib_dir.clone() + } else { + p.join("Lib") + }.canonicalize().unwrap(); + + let site_packages = stdlib.join("site-packages"); + + PythonPaths { + prefix, + bin_dir, + python_exe, + stdlib, + site_packages, + pyoxidizer_state_dir, + } +} + +pub fn invoke_python(python_paths: &PythonPaths, logger: &slog::Logger, args: &[&str]) { + let mut site_packages_s = python_paths + .site_packages + .canonicalize() + .unwrap() + .display() + .to_string(); + + if site_packages_s.starts_with("\\\\?\\") { + site_packages_s = site_packages_s[4..].to_string(); + } + + info!(logger, "setting PYTHONPATH {}", site_packages_s); + + let mut extra_envs = HashMap::new(); + extra_envs.insert("PYTHONPATH".to_string(), site_packages_s); + + info!( + logger, + "running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); + + let mut cmd = std::process::Command::new(&python_paths.python_exe.canonicalize().unwrap()) + .args(args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect( + format!( + "failed to run {} {}", + python_paths.python_exe.display(), + args.join(" ") + ) + .as_str(), + ); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); + } + } +} + impl ParsedPythonDistribution { pub fn from_path( logger: &slog::Logger, @@ -362,24 +475,126 @@ impl ParsedPythonDistribution { } /// Ensure pip is available to run in the distribution. - pub fn ensure_pip(&self) -> PathBuf { - let pip_path = self - .python_exe - .parent() - .expect("could not derive parent") - .to_path_buf() - .join(PIP_EXE_BASENAME); + /// This is only used by pip.rs + pub fn ensure_pip(&self, logger: &slog::Logger) -> PathBuf { + let dist_prefix = self.base_dir.join("python").join("install"); + let python_paths = resolve_python_paths(&dist_prefix, &self.version); + + let pip_path = python_paths.bin_dir.join(PIP_EXE_BASENAME); if !pip_path.exists() { - std::process::Command::new(&self.python_exe) - .args(&["-m", "ensurepip"]) - .status() - .expect("failed to run ensurepip"); + warn!(logger, "{} doesnt exist", pip_path.display().to_string()); + invoke_python(&python_paths, &logger, &["-m", "ensurepip"]); } pip_path } + /// Duplicate the python distribution, with distutils hacked + pub fn create_hacked_base(&self, logger: &slog::Logger) -> PythonPaths { + let venv_base = self.venv_base.clone(); + + let venv_dir_s = self.venv_base.display().to_string(); + + if !venv_base.exists() { + let dist_prefix = self.base_dir.join("python").join("install"); + + copy_dir(&dist_prefix, &venv_base).unwrap(); + + let dist_prefix_s = dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", dist_prefix_s, venv_dir_s + ); + } + + let python_paths = resolve_python_paths(&venv_base, &self.version); + + invoke_python(&python_paths, &logger, &["-m", "ensurepip"]); + + prepare_hacked_distutils(logger, &python_paths); + + python_paths + } + + /// Create a venv from the distribution at path. + pub fn create_venv(&self, logger: &slog::Logger, path: &PathBuf) -> PythonPaths { + let venv_dir_s = path.display().to_string(); + + // This will recreate it, if it was deleted + let python_paths = self.create_hacked_base(&logger); + + if path.exists() { + warn!(logger, "re-using {} {}", "venv", venv_dir_s); + } else { + warn!(logger, "creating {} {}", "venv", venv_dir_s); + invoke_python(&python_paths, &logger, &["-m", "venv", venv_dir_s.as_str()]); + } + + resolve_python_paths(&path, &self.version) + } + + /// Create or re-use an existing venv + pub fn prepare_venv( + &self, + logger: &slog::Logger, + venv_path: Option<&String>, + ) -> (PythonPaths, HashMap) { + let venv_dir_path = match venv_path { + Some(path_str) => PathBuf::from(path_str), + None => tempdir::TempDir::new("pyoxidizer-temp-venv") + .expect("could not create temp directory") + .path() + .join("venv"), + }; + + let python_paths = self.create_venv(logger, &venv_dir_path); + + let mut extra_envs = HashMap::new(); + + let prefix_s = python_paths + .prefix + .canonicalize() + .unwrap() + .display() + .to_string(); + + let venv_path_bin_s = python_paths + .bin_dir + .canonicalize() + .unwrap() + .display() + .to_string(); + + let path_separator = if cfg!(windows) { ";" } else { ":" }; + + let process_path_s = env::var("PATH").unwrap(); + + extra_envs.insert( + "PATH".to_string(), + format!("{}{}{}", venv_path_bin_s, path_separator, process_path_s), + ); + + let mut site_packages_s = python_paths + .site_packages + .canonicalize() + .unwrap() + .display() + .to_string(); + if site_packages_s.starts_with("\\\\?\\") { + site_packages_s = site_packages_s[4..].to_string(); + } + + extra_envs.insert("VIRTUAL_ENV".to_string(), prefix_s); + extra_envs.insert("PYTHONPATH".to_string(), site_packages_s); + + extra_envs.insert("PYOXIDIZER".to_string(), "1".to_string()); + + fs::create_dir_all(&python_paths.pyoxidizer_state_dir).unwrap(); + + (python_paths, extra_envs) + } + /// Obtain resolved `SourceModule` instances for this distribution. /// /// This effectively resolves the raw file content for .py files into @@ -689,6 +904,8 @@ pub fn analyze_python_distribution_data( }; } + let venv_base = dist_dir.parent().unwrap().join("hacked_base"); + Ok(ParsedPythonDistribution { flavor: pi.python_flavor.clone(), version: pi.python_version.clone(), @@ -715,6 +932,7 @@ pub fn analyze_python_distribution_data( py_modules, resources, license_infos, + venv_base, }) } diff --git a/pyoxidizer/src/py_packaging/distutils.rs b/pyoxidizer/src/py_packaging/distutils.rs index 11a932f38..2e6049f8f 100644 --- a/pyoxidizer/src/py_packaging/distutils.rs +++ b/pyoxidizer/src/py_packaging/distutils.rs @@ -5,11 +5,11 @@ use lazy_static::lazy_static; use serde::Deserialize; use slog::warn; -use std::collections::{BTreeMap, HashMap}; -use std::fs::{create_dir_all, read_dir, read_to_string}; +use std::collections::BTreeMap; +use std::fs::{read_dir, read_to_string}; use std::path::{Path, PathBuf}; -use super::distribution::ParsedPythonDistribution; +use super::distribution::PythonPaths; use super::resource::BuiltExtensionModule; lazy_static! { @@ -46,70 +46,21 @@ lazy_static! { /// modified distutils will survive multiple process invocations, unlike a /// monkeypatch. People do weird things in setup.py scripts and we want to /// support as many as possible. -pub fn prepare_hacked_distutils( - logger: &slog::Logger, - dist: &ParsedPythonDistribution, - dest_dir: &Path, - extra_python_paths: &[&Path], -) -> Result, String> { - let extra_sys_path = dest_dir.join("packages"); - +pub fn prepare_hacked_distutils(logger: &slog::Logger, target: &PythonPaths) { warn!( logger, "installing modified distutils to {}", - extra_sys_path.display() + target.stdlib.display() ); - let orig_distutils_path = dist.stdlib_path.join("distutils"); - let dest_distutils_path = extra_sys_path.join("distutils"); - - for entry in walkdir::WalkDir::new(&orig_distutils_path) { - match entry { - Ok(entry) => { - if entry.path().is_dir() { - continue; - } - - let source_path = entry.path(); - let rel_path = source_path - .strip_prefix(&orig_distutils_path) - .or_else(|_| Err("unable to strip prefix"))?; - let dest_path = dest_distutils_path.join(rel_path); - - let dest_dir = dest_path.parent().unwrap(); - std::fs::create_dir_all(&dest_dir).or_else(|e| Err(e.to_string()))?; - std::fs::copy(&source_path, &dest_path).or_else(|e| Err(e.to_string()))?; - } - Err(e) => return Err(e.to_string()), - } - } + let dest_distutils_path = target.stdlib.join("distutils"); for (path, data) in MODIFIED_DISTUTILS_FILES.iter() { let dest_path = dest_distutils_path.join(path); warn!(logger, "modifying distutils/{} for oxidation", path); - std::fs::write(dest_path, data).or_else(|e| Err(e.to_string()))?; + std::fs::write(dest_path, data).unwrap(); } - - let state_dir = dest_dir.join("pyoxidizer-build-state"); - create_dir_all(&state_dir).or_else(|e| Err(e.to_string()))?; - - let mut python_paths = vec![extra_sys_path.display().to_string()]; - python_paths.extend(extra_python_paths.iter().map(|p| p.display().to_string())); - - let path_separator = if cfg!(windows) { ";" } else { ":" }; - - let python_path = python_paths.join(path_separator); - - let mut res = HashMap::new(); - res.insert("PYTHONPATH".to_string(), python_path); - res.insert( - "PYOXIDIZER_DISTUTILS_STATE_DIR".to_string(), - state_dir.display().to_string(), - ); - res.insert("PYOXIDIZER".to_string(), "1".to_string()); - - Ok(res) } #[derive(Debug, Deserialize)] diff --git a/pyoxidizer/src/starlark/python_packaging.rs b/pyoxidizer/src/starlark/python_packaging.rs index 65516d8b3..d6710f2eb 100644 --- a/pyoxidizer/src/starlark/python_packaging.rs +++ b/pyoxidizer/src/starlark/python_packaging.rs @@ -16,8 +16,8 @@ use std::cmp::Ordering; use std::collections::HashMap; use super::env::{ - optional_dict_arg, optional_list_arg, required_bool_arg, required_list_arg, required_str_arg, - required_type_arg, + optional_dict_arg, optional_list_arg, optional_str_arg, required_bool_arg, required_list_arg, + required_str_arg, required_type_arg, }; use crate::app_packaging::config::{ resolve_install_location, PackagingFilterInclude, PackagingPackageRoot, @@ -519,6 +519,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] PipInstallSimple( package, + venv_path=None, extra_env=None, optimize_level=0, excludes=None, @@ -527,6 +528,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let package = required_str_arg("package", &package)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; optional_list_arg("excludes", "string", &excludes)?; @@ -565,6 +567,7 @@ starlark_module! { python_packaging_env => let rule = PackagingPipInstallSimple { package, + venv_path, extra_env, optimize_level, excludes, @@ -579,6 +582,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] PipRequirementsFile( requirements_path, + venv_path=None, extra_env=None, optimize_level=0, include_source=true, @@ -586,6 +590,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let requirements_path = required_str_arg("path", &requirements_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; let include_source = required_bool_arg("include_source", &include_source)?; @@ -618,6 +623,7 @@ starlark_module! { python_packaging_env => }; let rule = PackagingPipRequirementsFile { + venv_path, requirements_path, extra_env, optimize_level, @@ -632,6 +638,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] SetupPyInstall( package_path, + venv_path=None, extra_env=None, extra_global_arguments=None, optimize_level=0, @@ -640,6 +647,7 @@ starlark_module! { python_packaging_env => excludes=None ) { let package_path = required_str_arg("package_path", &package_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; optional_list_arg("extra_global_arguments", "string", &extra_global_arguments)?; required_type_arg("optimize_level", "int", &optimize_level)?; @@ -676,6 +684,7 @@ starlark_module! { python_packaging_env => let rule = PackagingSetupPyInstall { path: package_path, + venv_path, extra_env, extra_global_arguments, optimize_level: optimize_level.to_int().unwrap(), @@ -890,6 +899,7 @@ mod tests { let v = starlark_ok("PipInstallSimple('foo')"); let wanted = PackagingPipInstallSimple { package: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, excludes: Vec::new(), @@ -914,6 +924,7 @@ mod tests { let v = starlark_ok("PipRequirementsFile('path')"); let wanted = PackagingPipRequirementsFile { requirements_path: "path".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, include_source: true, @@ -928,6 +939,7 @@ mod tests { fn test_pip_requirements_file_extra_args() { let v = starlark_ok("PipRequirementsFile('path', extra_args=['foo'])"); let wanted = PackagingPipRequirementsFile { + venv_path: None, requirements_path: "path".to_string(), extra_env: HashMap::new(), optimize_level: 0, @@ -950,6 +962,7 @@ mod tests { let v = starlark_ok("SetupPyInstall('foo')"); let wanted = PackagingSetupPyInstall { path: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), extra_global_arguments: Vec::new(), optimize_level: 0, From aef54160186d9c0432f7bdf146068abb3f2f265c Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 25 Nov 2019 16:48:31 +0700 Subject: [PATCH 06/20] distutils.rs: Split and re-join Unsure why this is necessary; as it wasnt needed before --- pyoxidizer/src/py_packaging/distutils.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyoxidizer/src/py_packaging/distutils.rs b/pyoxidizer/src/py_packaging/distutils.rs index 2e6049f8f..d629e8c77 100644 --- a/pyoxidizer/src/py_packaging/distutils.rs +++ b/pyoxidizer/src/py_packaging/distutils.rs @@ -56,7 +56,8 @@ pub fn prepare_hacked_distutils(logger: &slog::Logger, target: &PythonPaths) { let dest_distutils_path = target.stdlib.join("distutils"); for (path, data) in MODIFIED_DISTUTILS_FILES.iter() { - let dest_path = dest_distutils_path.join(path); + let mut dest_path = dest_distutils_path.clone(); + dest_path.extend(path.split('/')); warn!(logger, "modifying distutils/{} for oxidation", path); std::fs::write(dest_path, data).unwrap(); From 03b4eebab511dcc15707d1f3bc5585459734b724 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 19 Nov 2019 08:55:21 +0700 Subject: [PATCH 07/20] Unix CI: Verify building cryptography --- ci/azure-pipelines-template.yml | 18 +++++++++++++++--- ci/pyapp.py | 23 ++++++++++++++++------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/ci/azure-pipelines-template.yml b/ci/azure-pipelines-template.yml index f3aa9d5f5..6926f2428 100644 --- a/ci/azure-pipelines-template.yml +++ b/ci/azure-pipelines-template.yml @@ -47,14 +47,26 @@ jobs: displayName: Clippy condition: ne( variables['rustup_toolchain'], 'nightly' ) - - ${{ if ne(parameters.name, 'Windows') }}: + - ${{ if eq(parameters.name, 'Linux') }}: + - script: | + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography ~/pyapp + cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp + displayName: Build Oxidized Application + + - ${{ if eq(parameters.name, 'macOS') }}: + - script: brew install openssl@1.1 + displayName: 'Install OpenSSL' + + - ${{ if eq(parameters.name, 'macOS') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 ~/pyapp + export CPPFLAGS="-I$(brew --prefix openssl@1.1)/include" + export LDFLAGS="-L$(brew --prefix openssl@1.1)/lib" + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 ~/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp displayName: Build Oxidized Application - ${{ if eq(parameters.name, 'Windows') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 %USERPROFILE%/pyapp + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 %USERPROFILE%/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run %USERPROFILE%/pyapp displayName: Build Oxidized Application (Windows) diff --git a/ci/pyapp.py b/ci/pyapp.py index ae9a27407..6556bbc66 100644 --- a/ci/pyapp.py +++ b/ci/pyapp.py @@ -1,11 +1,20 @@ import appdirs import _cffi_backend -# Allow AttributeError rather than ImportError -# as this indirectly triggers a missing __file__ -# https://github.com/dabeaz/ply/issues/216 -try: - import zero_buffer -except AttributeError: - pass print("hello, world") + +import cryptography.fernet + +m = b'does this work?' +s = 'original message:\n' + str(m) + '\n\n' + +k = cryptography.fernet.Fernet.generate_key() +fernet = cryptography.fernet.Fernet(k) +t = fernet.encrypt(m) +d = fernet.decrypt(t) + +s += 'key:\n' + str(k) + '\n\n' +s += 'token:\n' + str(t) + '\n\n' +s += 'decoded message:\n' + str(d) +print(s) +assert d == m From 65f29b8eea7ac4ecdf58431d8d63da7e11f66b12 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 24 Nov 2019 15:11:26 +0700 Subject: [PATCH 08/20] Mac: distutils: Exclude openssl libs These libs are already in the static python3 binary so shouldnt be linked in shared libraries used during build dependencies. --- pyoxidizer/src/distutils/unixccompiler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyoxidizer/src/distutils/unixccompiler.py b/pyoxidizer/src/distutils/unixccompiler.py index 55e615e12..ed25dc376 100644 --- a/pyoxidizer/src/distutils/unixccompiler.py +++ b/pyoxidizer/src/distutils/unixccompiler.py @@ -225,9 +225,15 @@ def extension_link_shared_object(self, package=None, ): + libs=libraries.copy() + if 'crypto' in libs: + libs.remove('crypto') + if 'ssl' in libs: + libs.remove('ssl') + self.link(CCompiler.SHARED_OBJECT, objects, output_filename, output_dir, - libraries, library_dirs, runtime_library_dirs, + libs, library_dirs, runtime_library_dirs, export_symbols, debug, extra_preargs, extra_postargs, build_temp, target_lang) From 1c9e24e956a22f5017e98ce9b2225ab13f16768e Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 19 Nov 2019 13:13:14 +0700 Subject: [PATCH 09/20] Win: distribution.rs: Use external python --- pyoxidizer/Cargo.toml | 1 + pyoxidizer/src/py_packaging/distribution.rs | 34 +++++++++++++++------ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index caff3c44d..319d675c0 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -54,6 +54,7 @@ url = "1.7" uuid = { version = "0.7", features = ["v4", "v5"] } version-compare = "0.0" walkdir = "2" +which = "3.1.0" xml-rs = "0.8" zip = "0.5" zstd = "0.4" diff --git a/pyoxidizer/src/py_packaging/distribution.rs b/pyoxidizer/src/py_packaging/distribution.rs index 2bdb04b4f..9328c3ebb 100644 --- a/pyoxidizer/src/py_packaging/distribution.rs +++ b/pyoxidizer/src/py_packaging/distribution.rs @@ -497,15 +497,31 @@ impl ParsedPythonDistribution { let venv_dir_s = self.venv_base.display().to_string(); if !venv_base.exists() { - let dist_prefix = self.base_dir.join("python").join("install"); - - copy_dir(&dist_prefix, &venv_base).unwrap(); - - let dist_prefix_s = dist_prefix.display().to_string(); - warn!( - logger, - "copied {} to create hacked base {}", dist_prefix_s, venv_dir_s - ); + if self.os == "windows" { + let external_python = which("python").unwrap().clone(); + warn!(logger, "python={}", external_python.display().to_string()); + // TODO: check python version + + let external_dist_prefix = external_python.parent().unwrap(); + + copy_dir(&external_dist_prefix, &venv_base).unwrap(); + + let external_dist_prefix_s = external_dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", external_dist_prefix_s, venv_dir_s + ); + } else { + let dist_prefix = self.base_dir.join("python").join("install"); + + copy_dir(&dist_prefix, &venv_base).unwrap(); + + let dist_prefix_s = dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", dist_prefix_s, venv_dir_s + ); + }; } let python_paths = resolve_python_paths(&venv_base, &self.version); From 56774fd82ea646c63d649bcd0c05fafd0f224104 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 19 Nov 2019 08:55:21 +0700 Subject: [PATCH 10/20] Win CI: Verify building cryptography The Windows standalone distribution doesnt include venv and presents other difficulties when running setup.py in the venv's due to various build dependencies fail. --- ci/azure-pipelines-template.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/azure-pipelines-template.yml b/ci/azure-pipelines-template.yml index 6926f2428..21cbdb0d7 100644 --- a/ci/azure-pipelines-template.yml +++ b/ci/azure-pipelines-template.yml @@ -65,8 +65,19 @@ jobs: cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp displayName: Build Oxidized Application + - ${{ if eq(parameters.name, 'Windows') }}: + - script: choco install openssl + displayName: 'Install OpenSSL' + + - ${{ if eq(parameters.name, 'Windows') }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + - ${{ if eq(parameters.name, 'Windows') }}: - script: | + set INCLUDE=%ProgramFiles%\OpenSSL-Win64\include;%INCLUDE% + set LIB=%ProgramFiles%\OpenSSL-Win64\lib\;%LIB% cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 %USERPROFILE%/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run %USERPROFILE%/pyapp displayName: Build Oxidized Application (Windows) From 0968207c8bc937b15f0e397f2eef4216da83b1f1 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 19 Nov 2019 08:55:21 +0700 Subject: [PATCH 11/20] Win: distutils: Build shared libs and static objs --- pyoxidizer/src/distutils/_msvccompiler.py | 42 +++++++++++++------ pyoxidizer/src/distutils/command/build_ext.py | 4 -- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 9bb1c4a98..6103a68f0 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -19,6 +19,7 @@ import shutil import stat import subprocess +import sys import winreg from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ @@ -427,6 +428,18 @@ def compile(self, sources, except DistutilsExecError as msg: raise CompileError(msg) + obj_index = args.index("/Fo" + obj) + args.remove("/Fo" + obj) + args.insert(obj_index, "/Fo" + obj + '.static') + # This is needed to activate specific symbol visibility / linking + # behavior on Windows. It isn't needed on UNIX but should be harmless. + args.append('/DPy_BUILD_CORE_BUILTIN=1') + + try: + self.spawn(args) + except DistutilsExecError as msg: + raise CompileError(msg) + return objects @@ -563,18 +576,6 @@ def extension_link_shared_object(self, package=None, ): - # The extension is compiled as a built-in, so linking a shared library - # won't work due to symbol visibility/export issues. The extension is - # expecting all CPython symbols to be available in the current binary, - # which they won't be for a shared library. PyOxidizer doesn't use the - # library anyway (at least not yet), so don't even bother with any - # linking. - #self.link(CCompiler.SHARED_OBJECT, objects, - # output_filename, output_dir, - # libraries, library_dirs, runtime_library_dirs, - # export_symbols, debug, - # extra_preargs, extra_postargs, build_temp, target_lang) - # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. @@ -583,10 +584,11 @@ def extension_link_shared_object(self, if not os.path.exists(dest_path): os.makedirs(dest_path) - # We need to copy the object files because they may be in a temp + # We need to copy the static object files because they may be in a temp # directory that doesn't outlive this process. object_paths = [] for i, o in enumerate(objects): + o = o + '.static' p = os.path.join(dest_path, '%s.%d.o' % (name, i)) shutil.copyfile(o, p) object_paths.append(p) @@ -607,6 +609,20 @@ def extension_link_shared_object(self, print('Wrote {}'.format(json_path), file=sys.stderr) + # This is the shared library, which would only be used if this is a + # build dependencies during the packaging rule processing, and even + # then it is possible that the package is usable as a build dependency + # without this extension. + try: + self.link(CCompiler.SHARED_OBJECT, objects, + output_filename, output_dir, + libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, + extra_preargs, extra_postargs, build_temp, target_lang) + except Exception as e: + print('Linking shared {} failed: {!r}'.format(name, e), + file=sys.stderr) + # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in # ccompiler.py. diff --git a/pyoxidizer/src/distutils/command/build_ext.py b/pyoxidizer/src/distutils/command/build_ext.py index 156ce30a9..9b0f854e2 100644 --- a/pyoxidizer/src/distutils/command/build_ext.py +++ b/pyoxidizer/src/distutils/command/build_ext.py @@ -525,10 +525,6 @@ def build_extension(self, ext): for undef in ext.undef_macros: macros.append((undef,)) - # This is needed to activate specific symbol visibility / linking - # behavior on Windows. It isn't needed on UNIX but should be harmless. - macros.append(('Py_BUILD_CORE_BUILTIN', '1')) - objects = self.compiler.compile(sources, output_dir=self.build_temp, macros=macros, From bc54fba9d356fa642e4960e3787f51c313ecc552 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 25 Nov 2019 07:29:44 +0700 Subject: [PATCH 12/20] distutils: Discard cffi's libffi on MSVC cffi includes a near replica of the CPython 3.7's libffi on MSVC, which needs to be discard to avoid duplicate syms. --- pyoxidizer/src/distutils/_msvccompiler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 6103a68f0..4c805031c 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -588,6 +588,11 @@ def extension_link_shared_object(self, # directory that doesn't outlive this process. object_paths = [] for i, o in enumerate(objects): + if 'libffi_msvc' in o: + print('Ignored static {}'.format(o)) + # https://github.com/indygreg/python-build-standalone/issues/23 + # cffi includes a near replica of CPython 3.7's custom libffi. + continue o = o + '.static' p = os.path.join(dest_path, '%s.%d.o' % (name, i)) shutil.copyfile(o, p) From 4b600a2f0aa52c0d5adcd815db79f7ffdc696408 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 26 Nov 2019 20:48:51 +0700 Subject: [PATCH 13/20] Fix pip.rs by removing built ext support It isnt being used at the moment, as it doesnt align with a venv based solution, and pip.rs doesnt yet support the setup.py rule which is critical for working around various problems. --- pyoxidizer/src/app_packaging/packaging_rule.rs | 1 + pyoxidizer/src/py_packaging/pip.rs | 17 ++++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyoxidizer/src/app_packaging/packaging_rule.rs b/pyoxidizer/src/app_packaging/packaging_rule.rs index 4c4557d2a..bf66e57ac 100644 --- a/pyoxidizer/src/app_packaging/packaging_rule.rs +++ b/pyoxidizer/src/app_packaging/packaging_rule.rs @@ -111,6 +111,7 @@ fn resource_full_name(resource: &PythonFileResource) -> &str { } } +// part of new approach from pip.rs; unused atm fn python_resource_full_name(resource: &PythonResource) -> String { match resource { PythonResource::ModuleSource { name, .. } => name.clone(), diff --git a/pyoxidizer/src/py_packaging/pip.rs b/pyoxidizer/src/py_packaging/pip.rs index d2044b17d..9482f935a 100644 --- a/pyoxidizer/src/py_packaging/pip.rs +++ b/pyoxidizer/src/py_packaging/pip.rs @@ -6,14 +6,18 @@ use slog::warn; use std::collections::HashMap; use std::convert::TryFrom; use std::io::{BufRead, BufReader}; -use std::path::PathBuf; -use super::distribution::ParsedPythonDistribution; +use super::distribution::{ParsedPythonDistribution}; use super::distutils::{prepare_hacked_distutils, read_built_extensions}; use super::fsscan::{find_python_resources, PythonFileResource}; use super::resource::PythonResource; /// Run `pip install` and return found resources. +/// This is unused at the moment as it is based around pip install --target +/// instead of `venv`s. +/// It doesnt use prepare_hacked_distutils as that isnt needed for its test +/// case, and a rewrite is needed to support core Python modules like +/// cffi, cryptography, Cython, PyNaCl, etc. pub fn pip_install( logger: &slog::Logger, dist: &ParsedPythonDistribution, @@ -24,9 +28,9 @@ pub fn pip_install( let temp_dir = tempdir::TempDir::new("pyoxidizer-pip-install") .or_else(|_| Err("could not create temp directory".to_string()))?; - dist.ensure_pip(); + dist.ensure_pip(&logger); - let mut env = prepare_hacked_distutils(logger, dist, temp_dir.path(), &[])?; + let mut env = HashMap::new(); for (key, value) in extra_envs.iter() { env.insert(key.clone(), value.clone()); @@ -98,10 +102,5 @@ pub fn pip_install( } } - let state_dir = PathBuf::from(env.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()); - for ext in read_built_extensions(&state_dir)? { - res.push(PythonResource::BuiltExtensionModule(ext)); - } - Ok(res) } From 365f5acc3e090a1bfae785ffc760dad5da0d72ec Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 26 Nov 2019 23:50:02 +0700 Subject: [PATCH 14/20] _msvccompiler.py: Use pre-built .obj files greenlet includes a pre-built .obj that is not re-built during the package build, and needs to be included in the static build process. --- pyoxidizer/src/distutils/_msvccompiler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 68a5a059a..34fabb4f2 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -593,7 +593,11 @@ def extension_link_shared_object(self, # https://github.com/indygreg/python-build-standalone/issues/23 # cffi includes a near replica of CPython 3.7's custom libffi. continue - o = o + '.static' + # If there are pre-compiled .obj files provided + # they will not have a .static version, and the + # original should be used. greenlet does this. + if os.path.exists(o + '.static'): + o = o + '.static' p = os.path.join(dest_path, '%s.%d.o' % (name, i)) shutil.copyfile(o, p) object_paths.append(p) From a40d4b09a6cc240cec91a34efd9ae56b6d09b1de Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 22 Nov 2019 21:50:04 +0700 Subject: [PATCH 15/20] libpython: Special case for gevent symbol clash gevent._queue conflicts with stdlib _queue. One of their PyInit__queue symbols needs to be renamed. Due to missing features in the rust crate `object`, the gevent objects can not be processed. So the stdlib _queuemodule object is modified instead. --- pyoxidizer/Cargo.toml | 1 + pyoxidizer/src/py_packaging/libpython.rs | 37 ++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index 419cb06d9..584a0db9f 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -39,6 +39,7 @@ itertools = "0.8" lazy_static = "1.3" libc = "0.2" object = { version = "0.16.0", features = ["read", "std", "write"] } +memmap = "0.7" regex = "1" reqwest = "0.9" rustc_version = "0.2" diff --git a/pyoxidizer/src/py_packaging/libpython.rs b/pyoxidizer/src/py_packaging/libpython.rs index fd02bc343..f9c18be6f 100644 --- a/pyoxidizer/src/py_packaging/libpython.rs +++ b/pyoxidizer/src/py_packaging/libpython.rs @@ -4,6 +4,7 @@ use itertools::Itertools; use lazy_static::lazy_static; +use memmap::Mmap; use slog::{info, warn}; use std::collections::{BTreeMap, BTreeSet}; use std::fs; @@ -103,7 +104,11 @@ pub fn make_config_c( continue; } - lines.push(format!("extern PyObject* {}(void);", init_fn)); + if init_fn == "PyInit__queue" && built_extension_modules.contains_key("gevent._queue") { + lines.push("extern PyObject* PyInit_stdlib_queue(void);".to_string()); + } else { + lines.push(format!("extern PyObject* {}(void);", init_fn)); + } } } @@ -130,8 +135,12 @@ pub fn make_config_c( continue; } - lines.push(format!("{{\"{}\", {}}},", em.module, init_fn)); - ambiguous_init_fns.push(init_fn.to_string()); + if init_fn == "PyInit__queue" && built_extension_modules.contains_key("gevent._queue") { + lines.push("{\"_queue\", PyInit_stdlib_queue},".to_string()); + } else { + lines.push(format!("{{\"{}\", {}}},", em.module, init_fn)); + ambiguous_init_fns.push(init_fn.to_string()); + } } } @@ -296,7 +305,9 @@ pub fn link_libpython( ); for (name, em) in extension_modules { if let Some(init_fn) = &em.init_fn { - if init_fn != "NULL" { + if init_fn == "PyInit__queue" && built_extension_modules.contains_key("gevent._queue") { + ambiguous_init_fns.push("PyInit_stdlib_queue".to_string()); + } else if init_fn != "NULL" { ambiguous_init_fns.push(init_fn.to_string()); } } @@ -313,7 +324,23 @@ pub fn link_libpython( em.object_paths ); for path in &em.object_paths { - build.object(path); + let mut out_path = path.clone(); + if (path.ends_with("_queuemodule.o") || path.ends_with("_queuemodule.obj")) + && built_extension_modules.contains_key("gevent._queue") + { + out_path = temp_dir_path.join(format!("{}_stdlib_prefixed.o", path.display())); + + let file = fs::File::open(&path).unwrap(); + let object_data = unsafe { Mmap::map(&file).unwrap() }; + + match rename_init(logger, &"stdlib_queue".to_string(), &object_data) { + Ok(val) => fs::write(&out_path, val).expect("unable to write object file"), + Err(err) => { + println!("Failed to rename symbol in '{}': {}", path.display(), err); + } + }; + } + build.object(out_path); } for entry in &em.links { From 9aa36275738512cacb7cdf17195005cfcf883b1d Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 29 Nov 2019 21:42:47 +0700 Subject: [PATCH 16/20] Win: Use object master Need unreleased fix for COFF. --- pyoxidizer/Cargo.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index 584a0db9f..9bcdd6ac4 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -20,6 +20,12 @@ path = "src/lib.rs" [build-dependencies] vergen = "3" +# Contains one fix needed for Windows +[dependencies.object] +git = "https://github.com/gimli-rs/object.git" +branch = "master" +features = ["read", "std", "write"] + [dependencies] byteorder = "1.2" cargo_toml = "0.6" @@ -38,7 +44,6 @@ hex = "0.3" itertools = "0.8" lazy_static = "1.3" libc = "0.2" -object = { version = "0.16.0", features = ["read", "std", "write"] } memmap = "0.7" regex = "1" reqwest = "0.9" From b311bb7499b5d2b66e08835e763d61aa7b61edae Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 29 Nov 2019 21:57:34 +0700 Subject: [PATCH 17/20] distutils: Workaround bug with common syms This is not needed for gevent, as we are rewriting the stdlib symbol, however it is a preventative measure for other symbol clashes which might require rewriting. --- pyoxidizer/src/distutils/unixccompiler.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyoxidizer/src/distutils/unixccompiler.py b/pyoxidizer/src/distutils/unixccompiler.py index ed25dc376..971072115 100644 --- a/pyoxidizer/src/distutils/unixccompiler.py +++ b/pyoxidizer/src/distutils/unixccompiler.py @@ -114,6 +114,18 @@ def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): if sys.platform == 'darwin': compiler_so = _osx_support.compiler_fixup(compiler_so, cc_args + extra_postargs) + + # If the PyInit symbol need to be renamed, any syms in common section + # may cause a breakage on Linux, and -fno-common avoids that. + # https://github.com/gimli-rs/object/issues/149 + # It may also help with macOS + # https://github.com/gimli-rs/object/issues/158 + # gevent on Linux could be solved this way, but it isnt, as rewriting + # the stdlib _queue is the solution ofr as macOS and Windows and it is + # better for all three platforms to use the same solution. + if '-fno-common' not in compiler_so: + compiler_so.insert(1, '-fno-common') + try: self.spawn(compiler_so + cc_args + [src, '-o', obj] + extra_postargs) From f8fcb0abbf31b68ed202efd567472c62270105aa Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 29 Nov 2019 17:24:47 +0700 Subject: [PATCH 18/20] projectmgmt: Add link.exe /FORCE:MULTIPLE Specifially needed to workaround multiple __real@ caused by using rust crate object processing of stdlib queuemodule.obj. It also solves conflicts like stdlib ffi.obj vs cffi libffi. --- pyoxidizer/src/projectmgmt.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyoxidizer/src/projectmgmt.rs b/pyoxidizer/src/projectmgmt.rs index bd8883f87..012bd54a0 100644 --- a/pyoxidizer/src/projectmgmt.rs +++ b/pyoxidizer/src/projectmgmt.rs @@ -498,6 +498,10 @@ pub fn build_project(logger: &slog::Logger, context: &mut BuildContext) -> Resul // https://github.com/rust-lang/rust/issues/37403 is resolved. if cfg!(windows) { envs.push(("RUSTC_BOOTSTRAP", "1".to_string())); + // Allow multiple definitions when using link.exe + if context.target_triple.contains("msvc") { + envs.push(("RUSTFLAGS", "-C link-args=/FORCE:MULTIPLE".to_string())); + } } match process::Command::new("cargo") From f5ef0933b1cd657d485914a3307aa153c089e49d Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 30 Nov 2019 10:52:58 +0700 Subject: [PATCH 19/20] CI: Install and import gevent --- ci/azure-pipelines-template.yml | 6 +++--- ci/pyapp.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ci/azure-pipelines-template.yml b/ci/azure-pipelines-template.yml index 253cbcbf0..fca510d25 100644 --- a/ci/azure-pipelines-template.yml +++ b/ci/azure-pipelines-template.yml @@ -49,7 +49,7 @@ jobs: - ${{ if eq(parameters.name, 'Linux') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 ~/pyapp + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 --pip-install gevent==1.4.0 ~/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp displayName: Build Oxidized Application @@ -61,7 +61,7 @@ jobs: - script: | export CPPFLAGS="-I$(brew --prefix openssl@1.1)/include" export LDFLAGS="-L$(brew --prefix openssl@1.1)/lib" - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 ~/pyapp + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 --pip-install gevent==1.4.0 ~/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp displayName: Build Oxidized Application @@ -78,6 +78,6 @@ jobs: - script: | set INCLUDE=%ProgramFiles%\OpenSSL-Win64\include;%INCLUDE% set LIB=%ProgramFiles%\OpenSSL-Win64\lib\;%LIB% - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 %USERPROFILE%/pyapp + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 --pip-install markupsafe==1.1.1 --pip-install simplejson==3.17.0 --pip-install gevent==1.4.0 %USERPROFILE%/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run %USERPROFILE%/pyapp displayName: Build Oxidized Application (Windows) diff --git a/ci/pyapp.py b/ci/pyapp.py index 259992edf..3f610e9cc 100644 --- a/ci/pyapp.py +++ b/ci/pyapp.py @@ -24,3 +24,14 @@ s += 'decoded message:\n' + str(d) print(s) assert d == m + +# This only confirms the built objects are loadable. +import gevent +import gevent._queue +assert 'gevent' in dir(gevent._queue) + +import gevent._event +import gevent._greenlet +import gevent._local +import gevent.resolver.cares +import gevent.libev.corecext From 93415838b0b9b7e80999a0b93efa4ecebf03322b Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 30 Nov 2019 18:26:31 +0700 Subject: [PATCH 20/20] Win: Use Fudge to install openssl --- ci/azure-pipelines-template.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/azure-pipelines-template.yml b/ci/azure-pipelines-template.yml index fca510d25..eb78205ee 100644 --- a/ci/azure-pipelines-template.yml +++ b/ci/azure-pipelines-template.yml @@ -66,7 +66,9 @@ jobs: displayName: Build Oxidized Application - ${{ if eq(parameters.name, 'Windows') }}: - - script: choco install openssl + - powershell: | + git clone 'https://github.com/Badgerati/Fudge' + .\Fudge\src\Fudge.ps1 install -adhoc openssl displayName: 'Install OpenSSL' - ${{ if eq(parameters.name, 'Windows') }}: