From 873e9c7068546e3656bf51360b2930c521d98c25 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 16 Nov 2019 20:52:47 +0700 Subject: [PATCH 1/9] Use reusable venvs Replace pip install --target with venvs. As setuptools breakes out of the venv to load distutils build_ext command, first create a copy of the Python distribution and hack it. This also allows venv's to be populated manually outside of PyOxidizer's config, while retaining the ability for PyOxidizer to pick up and include any built object files. To facilitate that, the PyOxidizer state dir where objects are stored is a constant in the distutils compiler modules and in the Rust source. Also add venv_path to PipRequirementsFile, allowing the same venv to be incrementally populated in multiple rules, and the venv re-used across PyOxidizer build runs. Fixes https://github.com/indygreg/PyOxidizer/issues/162 Fixes https://github.com/indygreg/PyOxidizer/issues/170 Closes https://github.com/indygreg/PyOxidizer/issues/194 --- pyoxidizer/Cargo.toml | 1 + pyoxidizer/src/app_packaging/config.rs | 3 + .../src/app_packaging/packaging_rule.rs | 675 +++++++----------- pyoxidizer/src/distutils/_msvccompiler.py | 10 +- pyoxidizer/src/distutils/unixccompiler.py | 10 +- pyoxidizer/src/py_packaging/distribution.rs | 246 ++++++- pyoxidizer/src/py_packaging/distutils.rs | 63 +- pyoxidizer/src/starlark/python_packaging.rs | 17 +- 8 files changed, 530 insertions(+), 495 deletions(-) diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index ddfd7772c..caff3c44d 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -27,6 +27,7 @@ cc = "1.0" clap = "2.32" codemap = "0.1" codemap-diagnostic = "0.1" +copy_dir = "0.1.2" encoding_rs = "0.8" fs2 = "0.4" git2 = "0.9" diff --git a/pyoxidizer/src/app_packaging/config.rs b/pyoxidizer/src/app_packaging/config.rs index 80e0ce2cd..a18089914 100644 --- a/pyoxidizer/src/app_packaging/config.rs +++ b/pyoxidizer/src/app_packaging/config.rs @@ -26,6 +26,7 @@ pub enum InstallLocation { #[derive(Clone, Debug, PartialEq)] pub struct PackagingSetupPyInstall { pub path: String, + pub venv_path: Option, pub extra_env: HashMap, pub extra_global_arguments: Vec, pub optimize_level: i64, @@ -87,6 +88,7 @@ pub struct PackagingPackageRoot { #[derive(Clone, Debug, PartialEq)] pub struct PackagingPipInstallSimple { pub package: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub excludes: Vec, @@ -99,6 +101,7 @@ pub struct PackagingPipInstallSimple { pub struct PackagingPipRequirementsFile { // TODO resolve to a PathBuf. pub requirements_path: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub include_source: bool, diff --git a/pyoxidizer/src/app_packaging/packaging_rule.rs b/pyoxidizer/src/app_packaging/packaging_rule.rs index 032066e1d..4c4557d2a 100644 --- a/pyoxidizer/src/app_packaging/packaging_rule.rs +++ b/pyoxidizer/src/app_packaging/packaging_rule.rs @@ -15,12 +15,13 @@ use super::config::{ PackagingStdlibExtensionsPolicy, PackagingVirtualenv, PythonPackaging, }; use super::state::BuildContext; -use crate::py_packaging::distribution::{is_stdlib_test_package, ParsedPythonDistribution}; -use crate::py_packaging::distutils::{prepare_hacked_distutils, read_built_extensions}; +use crate::py_packaging::distribution::{ + is_stdlib_test_package, resolve_python_paths, ParsedPythonDistribution, +}; +use crate::py_packaging::distutils::read_built_extensions; use crate::py_packaging::fsscan::{ find_python_resources, is_package_from_path, PythonFileResource, }; -use crate::py_packaging::pip::pip_install; use crate::py_packaging::resource::{AppRelativeResources, PythonResource}; #[derive(Debug)] @@ -119,30 +120,6 @@ fn python_resource_full_name(resource: &PythonResource) -> String { } } -struct PythonPaths { - main: PathBuf, - site_packages: PathBuf, -} - -/// Resolve the location of Python modules given a base install path. -fn resolve_python_paths(base: &Path, python_version: &str, is_windows: bool) -> PythonPaths { - let mut p = base.to_path_buf(); - - if is_windows { - p.push("Lib"); - } else { - p.push("lib"); - p.push(format!("python{}", &python_version[0..3])); - } - - let site_packages = p.join("site-packages"); - - PythonPaths { - main: p, - site_packages, - } -} - fn resolve_built_extensions( state_dir: &Path, res: &mut Vec, @@ -159,6 +136,122 @@ fn resolve_built_extensions( Ok(()) } +/// Processes resources in a path +/// Args includes and excludes are ignored if None or an empty Vec. +fn process_resources( + logger: &slog::Logger, + path: &PathBuf, + location: &ResourceLocation, + state_dir: Option<&PathBuf>, + include_source: bool, + optimize_level: i64, + includes: Option<&Vec>, + excludes: Option<&Vec>, +) -> Vec { + let mut res = Vec::new(); + + let path_s = path.display().to_string(); + warn!(logger, "processing resources from {}", path_s); + + for resource in find_python_resources(path) { + let full_name = resource_full_name(&resource); + + let excluded = match includes { + Some(values) => values.iter().any(|v| { + let prefix = v.clone() + "."; + full_name != v && !full_name.starts_with(&prefix) + }), + None => false, + }; + + if excluded { + info!( + logger, + "whitelist skipping {}", full_name + ); + continue; + } + + let excluded = match excludes { + Some(values) => match values.is_empty() { + true => false, + false => values.iter().all(|v| { + let prefix = v.clone() + "."; + full_name == v || full_name.starts_with(&prefix) + }), + }, + None => false, + }; + + if excluded { + info!( + logger, + "blacklist skipping {}", full_name + ); + continue; + } + + match resource { + PythonFileResource::Source { + full_name, path, .. + } => { + let is_package = is_package_from_path(&path); + let source = fs::read(path).expect("error reading source file"); + + if include_source { + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleSource { + name: full_name.clone(), + source: source.clone(), + is_package, + }, + }); + } + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleBytecodeRequest { + name: full_name.clone(), + source, + optimize_level: optimize_level as i32, + is_package, + }, + }); + } + + PythonFileResource::Resource(resource) => { + let data = fs::read(resource.path).expect("error reading resource file"); + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::Resource { + package: resource.package.clone(), + name: resource.stem.clone(), + data, + }, + }); + } + + _ => {} + } + } + + match state_dir { + Some(dir) => { + if dir.exists() { + resolve_built_extensions(&dir, &mut res, &location).unwrap(); + } + } + None => {} + }; + + res +} + fn resolve_stdlib_extensions_policy( logger: &slog::Logger, dist: &ParsedPythonDistribution, @@ -343,165 +436,44 @@ fn resolve_stdlib( } fn resolve_virtualenv( + logger: &slog::Logger, dist: &ParsedPythonDistribution, rule: &PackagingVirtualenv, ) -> Vec { - let mut res = Vec::new(); - let location = ResourceLocation::new(&rule.install_location); let python_paths = - resolve_python_paths(&Path::new(&rule.path), &dist.version, dist.os == "windows"); - let packages_path = python_paths.site_packages; + resolve_python_paths(&Path::new(&rule.path), &dist.version); - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), + ) } -fn resolve_package_root(rule: &PackagingPackageRoot) -> Vec { - let mut res = Vec::new(); - +fn resolve_package_root( + logger: &slog::Logger, + rule: &PackagingPackageRoot, +) -> Vec { let location = ResourceLocation::new(&rule.install_location); let path = PathBuf::from(&rule.path); - for resource in find_python_resources(&path) { - let mut relevant = false; - let full_name = resource_full_name(&resource); - - for package in &rule.packages { - let prefix = package.clone() + "."; - - if full_name == package || full_name.starts_with(&prefix) { - relevant = true; - } - } - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &path, + &location, + None, + rule.include_source, + rule.optimize_level, + Some(&rule.packages), + None, + ) } fn resolve_pip_install_simple( @@ -510,97 +482,61 @@ fn resolve_pip_install_simple( rule: &PackagingPipInstallSimple, verbose: bool, ) -> Vec { - let mut install_args = vec![ - "--no-binary".to_string(), - ":all:".to_string(), - rule.package.clone(), - ]; - - if let Some(ref args) = rule.extra_args { - install_args.extend(args.clone()); - } - - let resources = pip_install(logger, dist, verbose, &install_args, &rule.extra_env).unwrap(); - - let mut res = Vec::new(); let location = ResourceLocation::new(&rule.install_location); - for resource in resources { - let mut relevant = true; - - let full_name = python_resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); - if &full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } + let mut pip_args: Vec = vec![ + "-m".to_string(), + "pip".to_string(), + "--disable-pip-version-check".to_string(), + ]; - if !relevant { - continue; - } + if verbose { + pip_args.push("--verbose".to_string()); + } - match resource { - PythonResource::ModuleSource { - name, - is_package, - source, - } => { - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: name.clone(), - source: source.clone(), - is_package, - }, - }); - } + pip_args.extend(vec![ + "install".to_string(), + "--no-binary".to_string(), + ":all:".to_string(), + rule.package.clone(), + ]); - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name, - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } + if let Some(ref args) = rule.extra_args { + pip_args.extend(args.clone()); + } - PythonResource::Resource { - package, - name, - data, - } => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package, - name, - data, - }, - }); - } + for (key, value) in rule.extra_env.iter() { + extra_envs.insert(key.clone(), value.clone()); + } - PythonResource::BuiltExtensionModule(em) => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::BuiltExtensionModule(em), - }); - } + // TODO send stderr to stdout. + let mut cmd = std::process::Command::new(&python_paths.python_exe) + .args(&pip_args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("error running pip"); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); - _ => {} + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); } } - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), + ) } fn resolve_pip_requirements_file( @@ -609,83 +545,69 @@ fn resolve_pip_requirements_file( rule: &PackagingPipRequirementsFile, verbose: bool, ) -> Vec { - let mut install_args = vec![ + let location = ResourceLocation::new(&rule.install_location); + + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); + + let mut pip_args: Vec = vec![ + "-m".to_string(), + "pip".to_string(), + "--disable-pip-version-check".to_string(), + ]; + + if verbose { + pip_args.push("--verbose".to_string()); + } + + pip_args.extend(vec![ + "install".to_string(), "--no-binary".to_string(), ":all:".to_string(), "--requirement".to_string(), rule.requirements_path.to_string(), - ]; + ]); if let Some(ref args) = rule.extra_args { - install_args.extend(args.clone()); + pip_args.extend(args.clone()); } - let resources = pip_install(logger, dist, verbose, &install_args, &rule.extra_env).unwrap(); - - let location = ResourceLocation::new(&rule.install_location); - - let mut res = Vec::new(); - - for resource in resources { - match resource { - PythonResource::ModuleSource { - name, - is_package, - source, - } => { - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name, - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } + for (key, value) in rule.extra_env.iter() { + extra_envs.insert(key.clone(), value.clone()); + } - PythonResource::Resource { - package, - name, - data, - } => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package, - name, - data, - }, - }); - } + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + pip_args.join(" ") + ); - PythonResource::BuiltExtensionModule(em) => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::BuiltExtensionModule(em), - }); - } + // TODO send stderr to stdout. + let mut cmd = std::process::Command::new(&python_paths.python_exe) + .args(&pip_args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("error running pip"); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); - _ => {} + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); } } - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + None, + ) } fn resolve_setup_py_install( @@ -695,8 +617,6 @@ fn resolve_setup_py_install( rule: &PackagingSetupPyInstall, verbose: bool, ) -> Vec { - let mut res = Vec::new(); - // Execution directory is resolved relative to the active configuration // file unless it is absolute. let rule_path = PathBuf::from(&rule.path); @@ -709,31 +629,12 @@ fn resolve_setup_py_install( let location = ResourceLocation::new(&rule.install_location); - let temp_dir = tempdir::TempDir::new("pyoxidizer-setup-py-install") - .expect("could not create temp directory"); - - let target_dir_path = temp_dir.path().join("install"); - let target_dir_s = target_dir_path.display().to_string(); - - let python_paths = resolve_python_paths(&target_dir_path, &dist.version, dist.os == "windows"); - - std::fs::create_dir_all(&python_paths.site_packages) - .expect("unable to create site-packages directory"); - - let mut extra_envs = prepare_hacked_distutils( - logger, - dist, - temp_dir.path(), - &[&python_paths.site_packages, &python_paths.main], - ) - .expect("unable to hack distutils"); + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); for (key, value) in rule.extra_env.iter() { extra_envs.insert(key.clone(), value.clone()); } - warn!(logger, "python setup.py installing to {}", target_dir_s); - let mut args = vec!["setup.py"]; for arg in &rule.extra_global_arguments { @@ -744,10 +645,19 @@ fn resolve_setup_py_install( args.push("--verbose"); } - args.extend(&["install", "--prefix", &target_dir_s, "--no-compile"]); + let prefix_dir_s = python_paths.prefix.display().to_string(); + + args.extend(&["install", "--prefix", &prefix_dir_s, "--no-compile"]); + + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); // TODO send stderr to stdout. - let mut cmd = std::process::Command::new(&dist.python_exe) + let mut cmd = std::process::Command::new(&python_paths.python_exe) .current_dir(cwd) .args(&args) .envs(&extra_envs) @@ -768,81 +678,16 @@ fn resolve_setup_py_install( panic!("error running setup.py"); } - let packages_path = python_paths.site_packages; - - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - resolve_built_extensions( - &PathBuf::from(extra_envs.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()), - &mut res, + process_resources( + &logger, + &python_paths.site_packages, &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), ) - .unwrap(); - - res } /// Resolves a Python packaging rule to resources to package. @@ -872,9 +717,9 @@ pub fn resolve_python_packaging( PythonPackaging::Stdlib(rule) => resolve_stdlib(logger, dist, &rule), - PythonPackaging::Virtualenv(rule) => resolve_virtualenv(dist, &rule), + PythonPackaging::Virtualenv(rule) => resolve_virtualenv(logger, dist, &rule), - PythonPackaging::PackageRoot(rule) => resolve_package_root(&rule), + PythonPackaging::PackageRoot(rule) => resolve_package_root(logger, &rule), PythonPackaging::PipInstallSimple(rule) => { resolve_pip_install_simple(logger, dist, &rule, verbose) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 88e482aad..9bb1c4a98 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -14,6 +14,7 @@ # ported to VS 2015 by Steve Dower import json +import pathlib import os import shutil import stat @@ -562,9 +563,6 @@ def extension_link_shared_object(self, package=None, ): - if 'PYOXIDIZER_DISTUTILS_STATE_DIR' not in os.environ: - raise Exception('PYOXIDIZER_DISTUTILS_STATE_DIR not defined') - # The extension is compiled as a built-in, so linking a shared library # won't work due to symbol visibility/export issues. The extension is # expecting all CPython symbols to be available in the current binary, @@ -580,7 +578,10 @@ def extension_link_shared_object(self, # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. - dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + dest_path = str(pathlib.Path(sys.prefix) / 'state' / 'pyoxidizer') + + if not os.path.exists(dest_path): + os.makedirs(dest_path) # We need to copy the object files because they may be in a temp # directory that doesn't outlive this process. @@ -604,6 +605,7 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/distutils/unixccompiler.py b/pyoxidizer/src/distutils/unixccompiler.py index 26f575fe0..55e615e12 100644 --- a/pyoxidizer/src/distutils/unixccompiler.py +++ b/pyoxidizer/src/distutils/unixccompiler.py @@ -14,6 +14,7 @@ """ import json, os, sys, re, shutil +import pathlib from distutils import sysconfig from distutils.dep_util import newer @@ -224,9 +225,6 @@ def extension_link_shared_object(self, package=None, ): - if 'PYOXIDIZER_DISTUTILS_STATE_DIR' not in os.environ: - raise Exception('PYOXIDIZER_DISTUTILS_STATE_DIR not defined') - self.link(CCompiler.SHARED_OBJECT, objects, output_filename, output_dir, libraries, library_dirs, runtime_library_dirs, @@ -236,7 +234,10 @@ def extension_link_shared_object(self, # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. - dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + dest_path = str(pathlib.Path(sys.prefix) / 'state' / 'pyoxidizer') + + if not os.path.exists(dest_path): + os.makedirs(dest_path) # We need to copy the object files because they may be in a temp # directory that doesn't outlive this process. @@ -260,6 +261,7 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/py_packaging/distribution.rs b/pyoxidizer/src/py_packaging/distribution.rs index 8a2f36a6e..2bdb04b4f 100644 --- a/pyoxidizer/src/py_packaging/distribution.rs +++ b/pyoxidizer/src/py_packaging/distribution.rs @@ -2,31 +2,45 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use copy_dir::copy_dir; use fs2::FileExt; use itertools::Itertools; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use slog::warn; -use std::collections::BTreeMap; +use slog::{info, warn}; +use std::collections::{BTreeMap, HashMap}; +use std::env; use std::fs; use std::fs::{create_dir_all, File}; -use std::io::{Cursor, Read}; +use std::io::{BufRead, BufReader, Cursor, Read}; use std::path::{Path, PathBuf}; use url::Url; use uuid::Uuid; +use which::which; +use super::distutils::prepare_hacked_distutils; use super::fsscan::{ find_python_resources, is_package_from_path, walk_tree_files, PythonFileResource, }; use super::resource::{ResourceData, SourceModule}; + use crate::licensing::NON_GPL_LICENSES; +#[cfg(windows)] +const PYTHON_EXE_BASENAME: &str = "python.exe"; + +#[cfg(unix)] +const PYTHON_EXE_BASENAME: &str = "python3"; + #[cfg(windows)] const PIP_EXE_BASENAME: &str = "pip3.exe"; #[cfg(unix)] const PIP_EXE_BASENAME: &str = "pip3"; +// This needs to be kept in sync with *compiler.py +const PYOXIDIZER_STATE_DIR: &str = "state/pyoxidizer"; + const STDLIB_TEST_PACKAGES: &[&str] = &[ "bsddb.test", "ctypes.test", @@ -311,6 +325,9 @@ pub struct ParsedPythonDistribution { /// Describes license info for things in this distribution. pub license_infos: BTreeMap>, + + /// Path to copy of hacked dist to use for packaging rules venvs + pub venv_base: PathBuf, } #[derive(Debug)] @@ -333,6 +350,102 @@ pub enum ExtensionModuleFilter { NoGPL, } +pub struct PythonPaths { + pub prefix: PathBuf, + pub bin_dir: PathBuf, + pub python_exe: PathBuf, + pub stdlib: PathBuf, + pub site_packages: PathBuf, + pub pyoxidizer_state_dir: PathBuf, +} + +/// Resolve the location of Python modules given a base install path. +pub fn resolve_python_paths(base: &Path, python_version: &str) -> PythonPaths { + let prefix = base.to_path_buf().canonicalize().unwrap(); + + let p = prefix.clone(); + + let bin_dir = if p.join("Scripts").exists() { + p.join("Scripts") + } else { + p.join("bin") + }; + + let python_exe = if bin_dir.join(PYTHON_EXE_BASENAME).exists() { + bin_dir.join(PYTHON_EXE_BASENAME) + } else { + p.join(PYTHON_EXE_BASENAME) + }; + + let mut pyoxidizer_state_dir = p.clone(); + pyoxidizer_state_dir.extend(PYOXIDIZER_STATE_DIR.split('/')); + + let unix_lib_dir = p.join("lib").join(format!("python{}", &python_version[0..3])); + + let stdlib = if unix_lib_dir.exists() { + unix_lib_dir.clone() + } else { + p.join("Lib") + }.canonicalize().unwrap(); + + let site_packages = stdlib.join("site-packages"); + + PythonPaths { + prefix, + bin_dir, + python_exe, + stdlib, + site_packages, + pyoxidizer_state_dir, + } +} + +pub fn invoke_python(python_paths: &PythonPaths, logger: &slog::Logger, args: &[&str]) { + let mut site_packages_s = python_paths + .site_packages + .canonicalize() + .unwrap() + .display() + .to_string(); + + if site_packages_s.starts_with("\\\\?\\") { + site_packages_s = site_packages_s[4..].to_string(); + } + + info!(logger, "setting PYTHONPATH {}", site_packages_s); + + let mut extra_envs = HashMap::new(); + extra_envs.insert("PYTHONPATH".to_string(), site_packages_s); + + info!( + logger, + "running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); + + let mut cmd = std::process::Command::new(&python_paths.python_exe.canonicalize().unwrap()) + .args(args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect( + format!( + "failed to run {} {}", + python_paths.python_exe.display(), + args.join(" ") + ) + .as_str(), + ); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); + } + } +} + impl ParsedPythonDistribution { pub fn from_path( logger: &slog::Logger, @@ -362,24 +475,126 @@ impl ParsedPythonDistribution { } /// Ensure pip is available to run in the distribution. - pub fn ensure_pip(&self) -> PathBuf { - let pip_path = self - .python_exe - .parent() - .expect("could not derive parent") - .to_path_buf() - .join(PIP_EXE_BASENAME); + /// This is only used by pip.rs + pub fn ensure_pip(&self, logger: &slog::Logger) -> PathBuf { + let dist_prefix = self.base_dir.join("python").join("install"); + let python_paths = resolve_python_paths(&dist_prefix, &self.version); + + let pip_path = python_paths.bin_dir.join(PIP_EXE_BASENAME); if !pip_path.exists() { - std::process::Command::new(&self.python_exe) - .args(&["-m", "ensurepip"]) - .status() - .expect("failed to run ensurepip"); + warn!(logger, "{} doesnt exist", pip_path.display().to_string()); + invoke_python(&python_paths, &logger, &["-m", "ensurepip"]); } pip_path } + /// Duplicate the python distribution, with distutils hacked + pub fn create_hacked_base(&self, logger: &slog::Logger) -> PythonPaths { + let venv_base = self.venv_base.clone(); + + let venv_dir_s = self.venv_base.display().to_string(); + + if !venv_base.exists() { + let dist_prefix = self.base_dir.join("python").join("install"); + + copy_dir(&dist_prefix, &venv_base).unwrap(); + + let dist_prefix_s = dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", dist_prefix_s, venv_dir_s + ); + } + + let python_paths = resolve_python_paths(&venv_base, &self.version); + + invoke_python(&python_paths, &logger, &["-m", "ensurepip"]); + + prepare_hacked_distutils(logger, &python_paths); + + python_paths + } + + /// Create a venv from the distribution at path. + pub fn create_venv(&self, logger: &slog::Logger, path: &PathBuf) -> PythonPaths { + let venv_dir_s = path.display().to_string(); + + // This will recreate it, if it was deleted + let python_paths = self.create_hacked_base(&logger); + + if path.exists() { + warn!(logger, "re-using {} {}", "venv", venv_dir_s); + } else { + warn!(logger, "creating {} {}", "venv", venv_dir_s); + invoke_python(&python_paths, &logger, &["-m", "venv", venv_dir_s.as_str()]); + } + + resolve_python_paths(&path, &self.version) + } + + /// Create or re-use an existing venv + pub fn prepare_venv( + &self, + logger: &slog::Logger, + venv_path: Option<&String>, + ) -> (PythonPaths, HashMap) { + let venv_dir_path = match venv_path { + Some(path_str) => PathBuf::from(path_str), + None => tempdir::TempDir::new("pyoxidizer-temp-venv") + .expect("could not create temp directory") + .path() + .join("venv"), + }; + + let python_paths = self.create_venv(logger, &venv_dir_path); + + let mut extra_envs = HashMap::new(); + + let prefix_s = python_paths + .prefix + .canonicalize() + .unwrap() + .display() + .to_string(); + + let venv_path_bin_s = python_paths + .bin_dir + .canonicalize() + .unwrap() + .display() + .to_string(); + + let path_separator = if cfg!(windows) { ";" } else { ":" }; + + let process_path_s = env::var("PATH").unwrap(); + + extra_envs.insert( + "PATH".to_string(), + format!("{}{}{}", venv_path_bin_s, path_separator, process_path_s), + ); + + let mut site_packages_s = python_paths + .site_packages + .canonicalize() + .unwrap() + .display() + .to_string(); + if site_packages_s.starts_with("\\\\?\\") { + site_packages_s = site_packages_s[4..].to_string(); + } + + extra_envs.insert("VIRTUAL_ENV".to_string(), prefix_s); + extra_envs.insert("PYTHONPATH".to_string(), site_packages_s); + + extra_envs.insert("PYOXIDIZER".to_string(), "1".to_string()); + + fs::create_dir_all(&python_paths.pyoxidizer_state_dir).unwrap(); + + (python_paths, extra_envs) + } + /// Obtain resolved `SourceModule` instances for this distribution. /// /// This effectively resolves the raw file content for .py files into @@ -689,6 +904,8 @@ pub fn analyze_python_distribution_data( }; } + let venv_base = dist_dir.parent().unwrap().join("hacked_base"); + Ok(ParsedPythonDistribution { flavor: pi.python_flavor.clone(), version: pi.python_version.clone(), @@ -715,6 +932,7 @@ pub fn analyze_python_distribution_data( py_modules, resources, license_infos, + venv_base, }) } diff --git a/pyoxidizer/src/py_packaging/distutils.rs b/pyoxidizer/src/py_packaging/distutils.rs index 11a932f38..2e6049f8f 100644 --- a/pyoxidizer/src/py_packaging/distutils.rs +++ b/pyoxidizer/src/py_packaging/distutils.rs @@ -5,11 +5,11 @@ use lazy_static::lazy_static; use serde::Deserialize; use slog::warn; -use std::collections::{BTreeMap, HashMap}; -use std::fs::{create_dir_all, read_dir, read_to_string}; +use std::collections::BTreeMap; +use std::fs::{read_dir, read_to_string}; use std::path::{Path, PathBuf}; -use super::distribution::ParsedPythonDistribution; +use super::distribution::PythonPaths; use super::resource::BuiltExtensionModule; lazy_static! { @@ -46,70 +46,21 @@ lazy_static! { /// modified distutils will survive multiple process invocations, unlike a /// monkeypatch. People do weird things in setup.py scripts and we want to /// support as many as possible. -pub fn prepare_hacked_distutils( - logger: &slog::Logger, - dist: &ParsedPythonDistribution, - dest_dir: &Path, - extra_python_paths: &[&Path], -) -> Result, String> { - let extra_sys_path = dest_dir.join("packages"); - +pub fn prepare_hacked_distutils(logger: &slog::Logger, target: &PythonPaths) { warn!( logger, "installing modified distutils to {}", - extra_sys_path.display() + target.stdlib.display() ); - let orig_distutils_path = dist.stdlib_path.join("distutils"); - let dest_distutils_path = extra_sys_path.join("distutils"); - - for entry in walkdir::WalkDir::new(&orig_distutils_path) { - match entry { - Ok(entry) => { - if entry.path().is_dir() { - continue; - } - - let source_path = entry.path(); - let rel_path = source_path - .strip_prefix(&orig_distutils_path) - .or_else(|_| Err("unable to strip prefix"))?; - let dest_path = dest_distutils_path.join(rel_path); - - let dest_dir = dest_path.parent().unwrap(); - std::fs::create_dir_all(&dest_dir).or_else(|e| Err(e.to_string()))?; - std::fs::copy(&source_path, &dest_path).or_else(|e| Err(e.to_string()))?; - } - Err(e) => return Err(e.to_string()), - } - } + let dest_distutils_path = target.stdlib.join("distutils"); for (path, data) in MODIFIED_DISTUTILS_FILES.iter() { let dest_path = dest_distutils_path.join(path); warn!(logger, "modifying distutils/{} for oxidation", path); - std::fs::write(dest_path, data).or_else(|e| Err(e.to_string()))?; + std::fs::write(dest_path, data).unwrap(); } - - let state_dir = dest_dir.join("pyoxidizer-build-state"); - create_dir_all(&state_dir).or_else(|e| Err(e.to_string()))?; - - let mut python_paths = vec![extra_sys_path.display().to_string()]; - python_paths.extend(extra_python_paths.iter().map(|p| p.display().to_string())); - - let path_separator = if cfg!(windows) { ";" } else { ":" }; - - let python_path = python_paths.join(path_separator); - - let mut res = HashMap::new(); - res.insert("PYTHONPATH".to_string(), python_path); - res.insert( - "PYOXIDIZER_DISTUTILS_STATE_DIR".to_string(), - state_dir.display().to_string(), - ); - res.insert("PYOXIDIZER".to_string(), "1".to_string()); - - Ok(res) } #[derive(Debug, Deserialize)] diff --git a/pyoxidizer/src/starlark/python_packaging.rs b/pyoxidizer/src/starlark/python_packaging.rs index 65516d8b3..d6710f2eb 100644 --- a/pyoxidizer/src/starlark/python_packaging.rs +++ b/pyoxidizer/src/starlark/python_packaging.rs @@ -16,8 +16,8 @@ use std::cmp::Ordering; use std::collections::HashMap; use super::env::{ - optional_dict_arg, optional_list_arg, required_bool_arg, required_list_arg, required_str_arg, - required_type_arg, + optional_dict_arg, optional_list_arg, optional_str_arg, required_bool_arg, required_list_arg, + required_str_arg, required_type_arg, }; use crate::app_packaging::config::{ resolve_install_location, PackagingFilterInclude, PackagingPackageRoot, @@ -519,6 +519,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] PipInstallSimple( package, + venv_path=None, extra_env=None, optimize_level=0, excludes=None, @@ -527,6 +528,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let package = required_str_arg("package", &package)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; optional_list_arg("excludes", "string", &excludes)?; @@ -565,6 +567,7 @@ starlark_module! { python_packaging_env => let rule = PackagingPipInstallSimple { package, + venv_path, extra_env, optimize_level, excludes, @@ -579,6 +582,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] PipRequirementsFile( requirements_path, + venv_path=None, extra_env=None, optimize_level=0, include_source=true, @@ -586,6 +590,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let requirements_path = required_str_arg("path", &requirements_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; let include_source = required_bool_arg("include_source", &include_source)?; @@ -618,6 +623,7 @@ starlark_module! { python_packaging_env => }; let rule = PackagingPipRequirementsFile { + venv_path, requirements_path, extra_env, optimize_level, @@ -632,6 +638,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] SetupPyInstall( package_path, + venv_path=None, extra_env=None, extra_global_arguments=None, optimize_level=0, @@ -640,6 +647,7 @@ starlark_module! { python_packaging_env => excludes=None ) { let package_path = required_str_arg("package_path", &package_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; optional_list_arg("extra_global_arguments", "string", &extra_global_arguments)?; required_type_arg("optimize_level", "int", &optimize_level)?; @@ -676,6 +684,7 @@ starlark_module! { python_packaging_env => let rule = PackagingSetupPyInstall { path: package_path, + venv_path, extra_env, extra_global_arguments, optimize_level: optimize_level.to_int().unwrap(), @@ -890,6 +899,7 @@ mod tests { let v = starlark_ok("PipInstallSimple('foo')"); let wanted = PackagingPipInstallSimple { package: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, excludes: Vec::new(), @@ -914,6 +924,7 @@ mod tests { let v = starlark_ok("PipRequirementsFile('path')"); let wanted = PackagingPipRequirementsFile { requirements_path: "path".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, include_source: true, @@ -928,6 +939,7 @@ mod tests { fn test_pip_requirements_file_extra_args() { let v = starlark_ok("PipRequirementsFile('path', extra_args=['foo'])"); let wanted = PackagingPipRequirementsFile { + venv_path: None, requirements_path: "path".to_string(), extra_env: HashMap::new(), optimize_level: 0, @@ -950,6 +962,7 @@ mod tests { let v = starlark_ok("SetupPyInstall('foo')"); let wanted = PackagingSetupPyInstall { path: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), extra_global_arguments: Vec::new(), optimize_level: 0, From aef54160186d9c0432f7bdf146068abb3f2f265c Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 25 Nov 2019 16:48:31 +0700 Subject: [PATCH 2/9] distutils.rs: Split and re-join Unsure why this is necessary; as it wasnt needed before --- pyoxidizer/src/py_packaging/distutils.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyoxidizer/src/py_packaging/distutils.rs b/pyoxidizer/src/py_packaging/distutils.rs index 2e6049f8f..d629e8c77 100644 --- a/pyoxidizer/src/py_packaging/distutils.rs +++ b/pyoxidizer/src/py_packaging/distutils.rs @@ -56,7 +56,8 @@ pub fn prepare_hacked_distutils(logger: &slog::Logger, target: &PythonPaths) { let dest_distutils_path = target.stdlib.join("distutils"); for (path, data) in MODIFIED_DISTUTILS_FILES.iter() { - let dest_path = dest_distutils_path.join(path); + let mut dest_path = dest_distutils_path.clone(); + dest_path.extend(path.split('/')); warn!(logger, "modifying distutils/{} for oxidation", path); std::fs::write(dest_path, data).unwrap(); From 03b4eebab511dcc15707d1f3bc5585459734b724 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 19 Nov 2019 08:55:21 +0700 Subject: [PATCH 3/9] Unix CI: Verify building cryptography --- ci/azure-pipelines-template.yml | 18 +++++++++++++++--- ci/pyapp.py | 23 ++++++++++++++++------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/ci/azure-pipelines-template.yml b/ci/azure-pipelines-template.yml index f3aa9d5f5..6926f2428 100644 --- a/ci/azure-pipelines-template.yml +++ b/ci/azure-pipelines-template.yml @@ -47,14 +47,26 @@ jobs: displayName: Clippy condition: ne( variables['rustup_toolchain'], 'nightly' ) - - ${{ if ne(parameters.name, 'Windows') }}: + - ${{ if eq(parameters.name, 'Linux') }}: + - script: | + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography ~/pyapp + cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp + displayName: Build Oxidized Application + + - ${{ if eq(parameters.name, 'macOS') }}: + - script: brew install openssl@1.1 + displayName: 'Install OpenSSL' + + - ${{ if eq(parameters.name, 'macOS') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 ~/pyapp + export CPPFLAGS="-I$(brew --prefix openssl@1.1)/include" + export LDFLAGS="-L$(brew --prefix openssl@1.1)/lib" + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 ~/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp displayName: Build Oxidized Application - ${{ if eq(parameters.name, 'Windows') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 %USERPROFILE%/pyapp + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 %USERPROFILE%/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run %USERPROFILE%/pyapp displayName: Build Oxidized Application (Windows) diff --git a/ci/pyapp.py b/ci/pyapp.py index ae9a27407..6556bbc66 100644 --- a/ci/pyapp.py +++ b/ci/pyapp.py @@ -1,11 +1,20 @@ import appdirs import _cffi_backend -# Allow AttributeError rather than ImportError -# as this indirectly triggers a missing __file__ -# https://github.com/dabeaz/ply/issues/216 -try: - import zero_buffer -except AttributeError: - pass print("hello, world") + +import cryptography.fernet + +m = b'does this work?' +s = 'original message:\n' + str(m) + '\n\n' + +k = cryptography.fernet.Fernet.generate_key() +fernet = cryptography.fernet.Fernet(k) +t = fernet.encrypt(m) +d = fernet.decrypt(t) + +s += 'key:\n' + str(k) + '\n\n' +s += 'token:\n' + str(t) + '\n\n' +s += 'decoded message:\n' + str(d) +print(s) +assert d == m From 65f29b8eea7ac4ecdf58431d8d63da7e11f66b12 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 24 Nov 2019 15:11:26 +0700 Subject: [PATCH 4/9] Mac: distutils: Exclude openssl libs These libs are already in the static python3 binary so shouldnt be linked in shared libraries used during build dependencies. --- pyoxidizer/src/distutils/unixccompiler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyoxidizer/src/distutils/unixccompiler.py b/pyoxidizer/src/distutils/unixccompiler.py index 55e615e12..ed25dc376 100644 --- a/pyoxidizer/src/distutils/unixccompiler.py +++ b/pyoxidizer/src/distutils/unixccompiler.py @@ -225,9 +225,15 @@ def extension_link_shared_object(self, package=None, ): + libs=libraries.copy() + if 'crypto' in libs: + libs.remove('crypto') + if 'ssl' in libs: + libs.remove('ssl') + self.link(CCompiler.SHARED_OBJECT, objects, output_filename, output_dir, - libraries, library_dirs, runtime_library_dirs, + libs, library_dirs, runtime_library_dirs, export_symbols, debug, extra_preargs, extra_postargs, build_temp, target_lang) From 1c9e24e956a22f5017e98ce9b2225ab13f16768e Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 19 Nov 2019 13:13:14 +0700 Subject: [PATCH 5/9] Win: distribution.rs: Use external python --- pyoxidizer/Cargo.toml | 1 + pyoxidizer/src/py_packaging/distribution.rs | 34 +++++++++++++++------ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index caff3c44d..319d675c0 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -54,6 +54,7 @@ url = "1.7" uuid = { version = "0.7", features = ["v4", "v5"] } version-compare = "0.0" walkdir = "2" +which = "3.1.0" xml-rs = "0.8" zip = "0.5" zstd = "0.4" diff --git a/pyoxidizer/src/py_packaging/distribution.rs b/pyoxidizer/src/py_packaging/distribution.rs index 2bdb04b4f..9328c3ebb 100644 --- a/pyoxidizer/src/py_packaging/distribution.rs +++ b/pyoxidizer/src/py_packaging/distribution.rs @@ -497,15 +497,31 @@ impl ParsedPythonDistribution { let venv_dir_s = self.venv_base.display().to_string(); if !venv_base.exists() { - let dist_prefix = self.base_dir.join("python").join("install"); - - copy_dir(&dist_prefix, &venv_base).unwrap(); - - let dist_prefix_s = dist_prefix.display().to_string(); - warn!( - logger, - "copied {} to create hacked base {}", dist_prefix_s, venv_dir_s - ); + if self.os == "windows" { + let external_python = which("python").unwrap().clone(); + warn!(logger, "python={}", external_python.display().to_string()); + // TODO: check python version + + let external_dist_prefix = external_python.parent().unwrap(); + + copy_dir(&external_dist_prefix, &venv_base).unwrap(); + + let external_dist_prefix_s = external_dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", external_dist_prefix_s, venv_dir_s + ); + } else { + let dist_prefix = self.base_dir.join("python").join("install"); + + copy_dir(&dist_prefix, &venv_base).unwrap(); + + let dist_prefix_s = dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", dist_prefix_s, venv_dir_s + ); + }; } let python_paths = resolve_python_paths(&venv_base, &self.version); From 56774fd82ea646c63d649bcd0c05fafd0f224104 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 19 Nov 2019 08:55:21 +0700 Subject: [PATCH 6/9] Win CI: Verify building cryptography The Windows standalone distribution doesnt include venv and presents other difficulties when running setup.py in the venv's due to various build dependencies fail. --- ci/azure-pipelines-template.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/azure-pipelines-template.yml b/ci/azure-pipelines-template.yml index 6926f2428..21cbdb0d7 100644 --- a/ci/azure-pipelines-template.yml +++ b/ci/azure-pipelines-template.yml @@ -65,8 +65,19 @@ jobs: cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp displayName: Build Oxidized Application + - ${{ if eq(parameters.name, 'Windows') }}: + - script: choco install openssl + displayName: 'Install OpenSSL' + + - ${{ if eq(parameters.name, 'Windows') }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + - ${{ if eq(parameters.name, 'Windows') }}: - script: | + set INCLUDE=%ProgramFiles%\OpenSSL-Win64\include;%INCLUDE% + set LIB=%ProgramFiles%\OpenSSL-Win64\lib\;%LIB% cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 %USERPROFILE%/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run %USERPROFILE%/pyapp displayName: Build Oxidized Application (Windows) From 0968207c8bc937b15f0e397f2eef4216da83b1f1 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 19 Nov 2019 08:55:21 +0700 Subject: [PATCH 7/9] Win: distutils: Build shared libs and static objs --- pyoxidizer/src/distutils/_msvccompiler.py | 42 +++++++++++++------ pyoxidizer/src/distutils/command/build_ext.py | 4 -- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 9bb1c4a98..6103a68f0 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -19,6 +19,7 @@ import shutil import stat import subprocess +import sys import winreg from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ @@ -427,6 +428,18 @@ def compile(self, sources, except DistutilsExecError as msg: raise CompileError(msg) + obj_index = args.index("/Fo" + obj) + args.remove("/Fo" + obj) + args.insert(obj_index, "/Fo" + obj + '.static') + # This is needed to activate specific symbol visibility / linking + # behavior on Windows. It isn't needed on UNIX but should be harmless. + args.append('/DPy_BUILD_CORE_BUILTIN=1') + + try: + self.spawn(args) + except DistutilsExecError as msg: + raise CompileError(msg) + return objects @@ -563,18 +576,6 @@ def extension_link_shared_object(self, package=None, ): - # The extension is compiled as a built-in, so linking a shared library - # won't work due to symbol visibility/export issues. The extension is - # expecting all CPython symbols to be available in the current binary, - # which they won't be for a shared library. PyOxidizer doesn't use the - # library anyway (at least not yet), so don't even bother with any - # linking. - #self.link(CCompiler.SHARED_OBJECT, objects, - # output_filename, output_dir, - # libraries, library_dirs, runtime_library_dirs, - # export_symbols, debug, - # extra_preargs, extra_postargs, build_temp, target_lang) - # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. @@ -583,10 +584,11 @@ def extension_link_shared_object(self, if not os.path.exists(dest_path): os.makedirs(dest_path) - # We need to copy the object files because they may be in a temp + # We need to copy the static object files because they may be in a temp # directory that doesn't outlive this process. object_paths = [] for i, o in enumerate(objects): + o = o + '.static' p = os.path.join(dest_path, '%s.%d.o' % (name, i)) shutil.copyfile(o, p) object_paths.append(p) @@ -607,6 +609,20 @@ def extension_link_shared_object(self, print('Wrote {}'.format(json_path), file=sys.stderr) + # This is the shared library, which would only be used if this is a + # build dependencies during the packaging rule processing, and even + # then it is possible that the package is usable as a build dependency + # without this extension. + try: + self.link(CCompiler.SHARED_OBJECT, objects, + output_filename, output_dir, + libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, + extra_preargs, extra_postargs, build_temp, target_lang) + except Exception as e: + print('Linking shared {} failed: {!r}'.format(name, e), + file=sys.stderr) + # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in # ccompiler.py. diff --git a/pyoxidizer/src/distutils/command/build_ext.py b/pyoxidizer/src/distutils/command/build_ext.py index 156ce30a9..9b0f854e2 100644 --- a/pyoxidizer/src/distutils/command/build_ext.py +++ b/pyoxidizer/src/distutils/command/build_ext.py @@ -525,10 +525,6 @@ def build_extension(self, ext): for undef in ext.undef_macros: macros.append((undef,)) - # This is needed to activate specific symbol visibility / linking - # behavior on Windows. It isn't needed on UNIX but should be harmless. - macros.append(('Py_BUILD_CORE_BUILTIN', '1')) - objects = self.compiler.compile(sources, output_dir=self.build_temp, macros=macros, From bc54fba9d356fa642e4960e3787f51c313ecc552 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 25 Nov 2019 07:29:44 +0700 Subject: [PATCH 8/9] distutils: Discard cffi's libffi on MSVC cffi includes a near replica of the CPython 3.7's libffi on MSVC, which needs to be discard to avoid duplicate syms. --- pyoxidizer/src/distutils/_msvccompiler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 6103a68f0..4c805031c 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -588,6 +588,11 @@ def extension_link_shared_object(self, # directory that doesn't outlive this process. object_paths = [] for i, o in enumerate(objects): + if 'libffi_msvc' in o: + print('Ignored static {}'.format(o)) + # https://github.com/indygreg/python-build-standalone/issues/23 + # cffi includes a near replica of CPython 3.7's custom libffi. + continue o = o + '.static' p = os.path.join(dest_path, '%s.%d.o' % (name, i)) shutil.copyfile(o, p) From 4b600a2f0aa52c0d5adcd815db79f7ffdc696408 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 26 Nov 2019 20:48:51 +0700 Subject: [PATCH 9/9] Fix pip.rs by removing built ext support It isnt being used at the moment, as it doesnt align with a venv based solution, and pip.rs doesnt yet support the setup.py rule which is critical for working around various problems. --- pyoxidizer/src/app_packaging/packaging_rule.rs | 1 + pyoxidizer/src/py_packaging/pip.rs | 17 ++++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyoxidizer/src/app_packaging/packaging_rule.rs b/pyoxidizer/src/app_packaging/packaging_rule.rs index 4c4557d2a..bf66e57ac 100644 --- a/pyoxidizer/src/app_packaging/packaging_rule.rs +++ b/pyoxidizer/src/app_packaging/packaging_rule.rs @@ -111,6 +111,7 @@ fn resource_full_name(resource: &PythonFileResource) -> &str { } } +// part of new approach from pip.rs; unused atm fn python_resource_full_name(resource: &PythonResource) -> String { match resource { PythonResource::ModuleSource { name, .. } => name.clone(), diff --git a/pyoxidizer/src/py_packaging/pip.rs b/pyoxidizer/src/py_packaging/pip.rs index d2044b17d..9482f935a 100644 --- a/pyoxidizer/src/py_packaging/pip.rs +++ b/pyoxidizer/src/py_packaging/pip.rs @@ -6,14 +6,18 @@ use slog::warn; use std::collections::HashMap; use std::convert::TryFrom; use std::io::{BufRead, BufReader}; -use std::path::PathBuf; -use super::distribution::ParsedPythonDistribution; +use super::distribution::{ParsedPythonDistribution}; use super::distutils::{prepare_hacked_distutils, read_built_extensions}; use super::fsscan::{find_python_resources, PythonFileResource}; use super::resource::PythonResource; /// Run `pip install` and return found resources. +/// This is unused at the moment as it is based around pip install --target +/// instead of `venv`s. +/// It doesnt use prepare_hacked_distutils as that isnt needed for its test +/// case, and a rewrite is needed to support core Python modules like +/// cffi, cryptography, Cython, PyNaCl, etc. pub fn pip_install( logger: &slog::Logger, dist: &ParsedPythonDistribution, @@ -24,9 +28,9 @@ pub fn pip_install( let temp_dir = tempdir::TempDir::new("pyoxidizer-pip-install") .or_else(|_| Err("could not create temp directory".to_string()))?; - dist.ensure_pip(); + dist.ensure_pip(&logger); - let mut env = prepare_hacked_distutils(logger, dist, temp_dir.path(), &[])?; + let mut env = HashMap::new(); for (key, value) in extra_envs.iter() { env.insert(key.clone(), value.clone()); @@ -98,10 +102,5 @@ pub fn pip_install( } } - let state_dir = PathBuf::from(env.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()); - for ext in read_built_extensions(&state_dir)? { - res.push(PythonResource::BuiltExtensionModule(ext)); - } - Ok(res) }