diff --git a/ci/azure-pipelines-template.yml b/ci/azure-pipelines-template.yml index f3aa9d5f5..21cbdb0d7 100644 --- a/ci/azure-pipelines-template.yml +++ b/ci/azure-pipelines-template.yml @@ -47,14 +47,37 @@ jobs: displayName: Clippy condition: ne( variables['rustup_toolchain'], 'nightly' ) - - ${{ if ne(parameters.name, 'Windows') }}: + - ${{ if eq(parameters.name, 'Linux') }}: + - script: | + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography ~/pyapp + cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp + displayName: Build Oxidized Application + + - ${{ if eq(parameters.name, 'macOS') }}: + - script: brew install openssl@1.1 + displayName: 'Install OpenSSL' + + - ${{ if eq(parameters.name, 'macOS') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 ~/pyapp + export CPPFLAGS="-I$(brew --prefix openssl@1.1)/include" + export LDFLAGS="-L$(brew --prefix openssl@1.1)/lib" + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 ~/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run ~/pyapp displayName: Build Oxidized Application + - ${{ if eq(parameters.name, 'Windows') }}: + - script: choco install openssl + displayName: 'Install OpenSSL' + + - ${{ if eq(parameters.name, 'Windows') }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + - ${{ if eq(parameters.name, 'Windows') }}: - script: | - cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1 %USERPROFILE%/pyapp + set INCLUDE=%ProgramFiles%\OpenSSL-Win64\include;%INCLUDE% + set LIB=%ProgramFiles%\OpenSSL-Win64\lib\;%LIB% + cargo run --bin pyoxidizer -- init --pip-install appdirs==1.4.3 --pip-install cryptography==2.8 %USERPROFILE%/pyapp cat ci/pyapp.py | cargo run --bin pyoxidizer -- run %USERPROFILE%/pyapp displayName: Build Oxidized Application (Windows) diff --git a/ci/pyapp.py b/ci/pyapp.py index ae9a27407..6556bbc66 100644 --- a/ci/pyapp.py +++ b/ci/pyapp.py @@ -1,11 +1,20 @@ import appdirs import _cffi_backend -# Allow AttributeError rather than ImportError -# as this indirectly triggers a missing __file__ -# https://github.com/dabeaz/ply/issues/216 -try: - import zero_buffer -except AttributeError: - pass print("hello, world") + +import cryptography.fernet + +m = b'does this work?' +s = 'original message:\n' + str(m) + '\n\n' + +k = cryptography.fernet.Fernet.generate_key() +fernet = cryptography.fernet.Fernet(k) +t = fernet.encrypt(m) +d = fernet.decrypt(t) + +s += 'key:\n' + str(k) + '\n\n' +s += 'token:\n' + str(t) + '\n\n' +s += 'decoded message:\n' + str(d) +print(s) +assert d == m diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index ddfd7772c..319d675c0 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -27,6 +27,7 @@ cc = "1.0" clap = "2.32" codemap = "0.1" codemap-diagnostic = "0.1" +copy_dir = "0.1.2" encoding_rs = "0.8" fs2 = "0.4" git2 = "0.9" @@ -53,6 +54,7 @@ url = "1.7" uuid = { version = "0.7", features = ["v4", "v5"] } version-compare = "0.0" walkdir = "2" +which = "3.1.0" xml-rs = "0.8" zip = "0.5" zstd = "0.4" diff --git a/pyoxidizer/src/app_packaging/config.rs b/pyoxidizer/src/app_packaging/config.rs index 80e0ce2cd..a18089914 100644 --- a/pyoxidizer/src/app_packaging/config.rs +++ b/pyoxidizer/src/app_packaging/config.rs @@ -26,6 +26,7 @@ pub enum InstallLocation { #[derive(Clone, Debug, PartialEq)] pub struct PackagingSetupPyInstall { pub path: String, + pub venv_path: Option, pub extra_env: HashMap, pub extra_global_arguments: Vec, pub optimize_level: i64, @@ -87,6 +88,7 @@ pub struct PackagingPackageRoot { #[derive(Clone, Debug, PartialEq)] pub struct PackagingPipInstallSimple { pub package: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub excludes: Vec, @@ -99,6 +101,7 @@ pub struct PackagingPipInstallSimple { pub struct PackagingPipRequirementsFile { // TODO resolve to a PathBuf. pub requirements_path: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub include_source: bool, diff --git a/pyoxidizer/src/app_packaging/packaging_rule.rs b/pyoxidizer/src/app_packaging/packaging_rule.rs index 032066e1d..bf66e57ac 100644 --- a/pyoxidizer/src/app_packaging/packaging_rule.rs +++ b/pyoxidizer/src/app_packaging/packaging_rule.rs @@ -15,12 +15,13 @@ use super::config::{ PackagingStdlibExtensionsPolicy, PackagingVirtualenv, PythonPackaging, }; use super::state::BuildContext; -use crate::py_packaging::distribution::{is_stdlib_test_package, ParsedPythonDistribution}; -use crate::py_packaging::distutils::{prepare_hacked_distutils, read_built_extensions}; +use crate::py_packaging::distribution::{ + is_stdlib_test_package, resolve_python_paths, ParsedPythonDistribution, +}; +use crate::py_packaging::distutils::read_built_extensions; use crate::py_packaging::fsscan::{ find_python_resources, is_package_from_path, PythonFileResource, }; -use crate::py_packaging::pip::pip_install; use crate::py_packaging::resource::{AppRelativeResources, PythonResource}; #[derive(Debug)] @@ -110,6 +111,7 @@ fn resource_full_name(resource: &PythonFileResource) -> &str { } } +// part of new approach from pip.rs; unused atm fn python_resource_full_name(resource: &PythonResource) -> String { match resource { PythonResource::ModuleSource { name, .. } => name.clone(), @@ -119,30 +121,6 @@ fn python_resource_full_name(resource: &PythonResource) -> String { } } -struct PythonPaths { - main: PathBuf, - site_packages: PathBuf, -} - -/// Resolve the location of Python modules given a base install path. -fn resolve_python_paths(base: &Path, python_version: &str, is_windows: bool) -> PythonPaths { - let mut p = base.to_path_buf(); - - if is_windows { - p.push("Lib"); - } else { - p.push("lib"); - p.push(format!("python{}", &python_version[0..3])); - } - - let site_packages = p.join("site-packages"); - - PythonPaths { - main: p, - site_packages, - } -} - fn resolve_built_extensions( state_dir: &Path, res: &mut Vec, @@ -159,6 +137,122 @@ fn resolve_built_extensions( Ok(()) } +/// Processes resources in a path +/// Args includes and excludes are ignored if None or an empty Vec. +fn process_resources( + logger: &slog::Logger, + path: &PathBuf, + location: &ResourceLocation, + state_dir: Option<&PathBuf>, + include_source: bool, + optimize_level: i64, + includes: Option<&Vec>, + excludes: Option<&Vec>, +) -> Vec { + let mut res = Vec::new(); + + let path_s = path.display().to_string(); + warn!(logger, "processing resources from {}", path_s); + + for resource in find_python_resources(path) { + let full_name = resource_full_name(&resource); + + let excluded = match includes { + Some(values) => values.iter().any(|v| { + let prefix = v.clone() + "."; + full_name != v && !full_name.starts_with(&prefix) + }), + None => false, + }; + + if excluded { + info!( + logger, + "whitelist skipping {}", full_name + ); + continue; + } + + let excluded = match excludes { + Some(values) => match values.is_empty() { + true => false, + false => values.iter().all(|v| { + let prefix = v.clone() + "."; + full_name == v || full_name.starts_with(&prefix) + }), + }, + None => false, + }; + + if excluded { + info!( + logger, + "blacklist skipping {}", full_name + ); + continue; + } + + match resource { + PythonFileResource::Source { + full_name, path, .. + } => { + let is_package = is_package_from_path(&path); + let source = fs::read(path).expect("error reading source file"); + + if include_source { + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleSource { + name: full_name.clone(), + source: source.clone(), + is_package, + }, + }); + } + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleBytecodeRequest { + name: full_name.clone(), + source, + optimize_level: optimize_level as i32, + is_package, + }, + }); + } + + PythonFileResource::Resource(resource) => { + let data = fs::read(resource.path).expect("error reading resource file"); + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::Resource { + package: resource.package.clone(), + name: resource.stem.clone(), + data, + }, + }); + } + + _ => {} + } + } + + match state_dir { + Some(dir) => { + if dir.exists() { + resolve_built_extensions(&dir, &mut res, &location).unwrap(); + } + } + None => {} + }; + + res +} + fn resolve_stdlib_extensions_policy( logger: &slog::Logger, dist: &ParsedPythonDistribution, @@ -343,165 +437,44 @@ fn resolve_stdlib( } fn resolve_virtualenv( + logger: &slog::Logger, dist: &ParsedPythonDistribution, rule: &PackagingVirtualenv, ) -> Vec { - let mut res = Vec::new(); - let location = ResourceLocation::new(&rule.install_location); let python_paths = - resolve_python_paths(&Path::new(&rule.path), &dist.version, dist.os == "windows"); - let packages_path = python_paths.site_packages; + resolve_python_paths(&Path::new(&rule.path), &dist.version); - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), + ) } -fn resolve_package_root(rule: &PackagingPackageRoot) -> Vec { - let mut res = Vec::new(); - +fn resolve_package_root( + logger: &slog::Logger, + rule: &PackagingPackageRoot, +) -> Vec { let location = ResourceLocation::new(&rule.install_location); let path = PathBuf::from(&rule.path); - for resource in find_python_resources(&path) { - let mut relevant = false; - let full_name = resource_full_name(&resource); - - for package in &rule.packages { - let prefix = package.clone() + "."; - - if full_name == package || full_name.starts_with(&prefix) { - relevant = true; - } - } - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &path, + &location, + None, + rule.include_source, + rule.optimize_level, + Some(&rule.packages), + None, + ) } fn resolve_pip_install_simple( @@ -510,97 +483,61 @@ fn resolve_pip_install_simple( rule: &PackagingPipInstallSimple, verbose: bool, ) -> Vec { - let mut install_args = vec![ - "--no-binary".to_string(), - ":all:".to_string(), - rule.package.clone(), - ]; - - if let Some(ref args) = rule.extra_args { - install_args.extend(args.clone()); - } - - let resources = pip_install(logger, dist, verbose, &install_args, &rule.extra_env).unwrap(); - - let mut res = Vec::new(); let location = ResourceLocation::new(&rule.install_location); - for resource in resources { - let mut relevant = true; - - let full_name = python_resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); - if &full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } + let mut pip_args: Vec = vec![ + "-m".to_string(), + "pip".to_string(), + "--disable-pip-version-check".to_string(), + ]; - if !relevant { - continue; - } + if verbose { + pip_args.push("--verbose".to_string()); + } - match resource { - PythonResource::ModuleSource { - name, - is_package, - source, - } => { - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: name.clone(), - source: source.clone(), - is_package, - }, - }); - } + pip_args.extend(vec![ + "install".to_string(), + "--no-binary".to_string(), + ":all:".to_string(), + rule.package.clone(), + ]); - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name, - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } + if let Some(ref args) = rule.extra_args { + pip_args.extend(args.clone()); + } - PythonResource::Resource { - package, - name, - data, - } => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package, - name, - data, - }, - }); - } + for (key, value) in rule.extra_env.iter() { + extra_envs.insert(key.clone(), value.clone()); + } - PythonResource::BuiltExtensionModule(em) => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::BuiltExtensionModule(em), - }); - } + // TODO send stderr to stdout. + let mut cmd = std::process::Command::new(&python_paths.python_exe) + .args(&pip_args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("error running pip"); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); - _ => {} + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); } } - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), + ) } fn resolve_pip_requirements_file( @@ -609,83 +546,69 @@ fn resolve_pip_requirements_file( rule: &PackagingPipRequirementsFile, verbose: bool, ) -> Vec { - let mut install_args = vec![ + let location = ResourceLocation::new(&rule.install_location); + + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); + + let mut pip_args: Vec = vec![ + "-m".to_string(), + "pip".to_string(), + "--disable-pip-version-check".to_string(), + ]; + + if verbose { + pip_args.push("--verbose".to_string()); + } + + pip_args.extend(vec![ + "install".to_string(), "--no-binary".to_string(), ":all:".to_string(), "--requirement".to_string(), rule.requirements_path.to_string(), - ]; + ]); if let Some(ref args) = rule.extra_args { - install_args.extend(args.clone()); + pip_args.extend(args.clone()); } - let resources = pip_install(logger, dist, verbose, &install_args, &rule.extra_env).unwrap(); - - let location = ResourceLocation::new(&rule.install_location); - - let mut res = Vec::new(); - - for resource in resources { - match resource { - PythonResource::ModuleSource { - name, - is_package, - source, - } => { - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name, - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } + for (key, value) in rule.extra_env.iter() { + extra_envs.insert(key.clone(), value.clone()); + } - PythonResource::Resource { - package, - name, - data, - } => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package, - name, - data, - }, - }); - } + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + pip_args.join(" ") + ); - PythonResource::BuiltExtensionModule(em) => { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::BuiltExtensionModule(em), - }); - } + // TODO send stderr to stdout. + let mut cmd = std::process::Command::new(&python_paths.python_exe) + .args(&pip_args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("error running pip"); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); - _ => {} + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); } } - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + None, + ) } fn resolve_setup_py_install( @@ -695,8 +618,6 @@ fn resolve_setup_py_install( rule: &PackagingSetupPyInstall, verbose: bool, ) -> Vec { - let mut res = Vec::new(); - // Execution directory is resolved relative to the active configuration // file unless it is absolute. let rule_path = PathBuf::from(&rule.path); @@ -709,31 +630,12 @@ fn resolve_setup_py_install( let location = ResourceLocation::new(&rule.install_location); - let temp_dir = tempdir::TempDir::new("pyoxidizer-setup-py-install") - .expect("could not create temp directory"); - - let target_dir_path = temp_dir.path().join("install"); - let target_dir_s = target_dir_path.display().to_string(); - - let python_paths = resolve_python_paths(&target_dir_path, &dist.version, dist.os == "windows"); - - std::fs::create_dir_all(&python_paths.site_packages) - .expect("unable to create site-packages directory"); - - let mut extra_envs = prepare_hacked_distutils( - logger, - dist, - temp_dir.path(), - &[&python_paths.site_packages, &python_paths.main], - ) - .expect("unable to hack distutils"); + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); for (key, value) in rule.extra_env.iter() { extra_envs.insert(key.clone(), value.clone()); } - warn!(logger, "python setup.py installing to {}", target_dir_s); - let mut args = vec!["setup.py"]; for arg in &rule.extra_global_arguments { @@ -744,10 +646,19 @@ fn resolve_setup_py_install( args.push("--verbose"); } - args.extend(&["install", "--prefix", &target_dir_s, "--no-compile"]); + let prefix_dir_s = python_paths.prefix.display().to_string(); + + args.extend(&["install", "--prefix", &prefix_dir_s, "--no-compile"]); + + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); // TODO send stderr to stdout. - let mut cmd = std::process::Command::new(&dist.python_exe) + let mut cmd = std::process::Command::new(&python_paths.python_exe) .current_dir(cwd) .args(&args) .envs(&extra_envs) @@ -768,81 +679,16 @@ fn resolve_setup_py_install( panic!("error running setup.py"); } - let packages_path = python_paths.site_packages; - - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecodeRequest { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - resolve_built_extensions( - &PathBuf::from(extra_envs.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()), - &mut res, + process_resources( + &logger, + &python_paths.site_packages, &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), ) - .unwrap(); - - res } /// Resolves a Python packaging rule to resources to package. @@ -872,9 +718,9 @@ pub fn resolve_python_packaging( PythonPackaging::Stdlib(rule) => resolve_stdlib(logger, dist, &rule), - PythonPackaging::Virtualenv(rule) => resolve_virtualenv(dist, &rule), + PythonPackaging::Virtualenv(rule) => resolve_virtualenv(logger, dist, &rule), - PythonPackaging::PackageRoot(rule) => resolve_package_root(&rule), + PythonPackaging::PackageRoot(rule) => resolve_package_root(logger, &rule), PythonPackaging::PipInstallSimple(rule) => { resolve_pip_install_simple(logger, dist, &rule, verbose) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 88e482aad..4c805031c 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -14,10 +14,12 @@ # ported to VS 2015 by Steve Dower import json +import pathlib import os import shutil import stat import subprocess +import sys import winreg from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ @@ -426,6 +428,18 @@ def compile(self, sources, except DistutilsExecError as msg: raise CompileError(msg) + obj_index = args.index("/Fo" + obj) + args.remove("/Fo" + obj) + args.insert(obj_index, "/Fo" + obj + '.static') + # This is needed to activate specific symbol visibility / linking + # behavior on Windows. It isn't needed on UNIX but should be harmless. + args.append('/DPy_BUILD_CORE_BUILTIN=1') + + try: + self.spawn(args) + except DistutilsExecError as msg: + raise CompileError(msg) + return objects @@ -562,30 +576,24 @@ def extension_link_shared_object(self, package=None, ): - if 'PYOXIDIZER_DISTUTILS_STATE_DIR' not in os.environ: - raise Exception('PYOXIDIZER_DISTUTILS_STATE_DIR not defined') - - # The extension is compiled as a built-in, so linking a shared library - # won't work due to symbol visibility/export issues. The extension is - # expecting all CPython symbols to be available in the current binary, - # which they won't be for a shared library. PyOxidizer doesn't use the - # library anyway (at least not yet), so don't even bother with any - # linking. - #self.link(CCompiler.SHARED_OBJECT, objects, - # output_filename, output_dir, - # libraries, library_dirs, runtime_library_dirs, - # export_symbols, debug, - # extra_preargs, extra_postargs, build_temp, target_lang) - # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. - dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + dest_path = str(pathlib.Path(sys.prefix) / 'state' / 'pyoxidizer') + + if not os.path.exists(dest_path): + os.makedirs(dest_path) - # We need to copy the object files because they may be in a temp + # We need to copy the static object files because they may be in a temp # directory that doesn't outlive this process. object_paths = [] for i, o in enumerate(objects): + if 'libffi_msvc' in o: + print('Ignored static {}'.format(o)) + # https://github.com/indygreg/python-build-standalone/issues/23 + # cffi includes a near replica of CPython 3.7's custom libffi. + continue + o = o + '.static' p = os.path.join(dest_path, '%s.%d.o' % (name, i)) shutil.copyfile(o, p) object_paths.append(p) @@ -604,6 +612,21 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) + + # This is the shared library, which would only be used if this is a + # build dependencies during the packaging rule processing, and even + # then it is possible that the package is usable as a build dependency + # without this extension. + try: + self.link(CCompiler.SHARED_OBJECT, objects, + output_filename, output_dir, + libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, + extra_preargs, extra_postargs, build_temp, target_lang) + except Exception as e: + print('Linking shared {} failed: {!r}'.format(name, e), + file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/distutils/command/build_ext.py b/pyoxidizer/src/distutils/command/build_ext.py index 156ce30a9..9b0f854e2 100644 --- a/pyoxidizer/src/distutils/command/build_ext.py +++ b/pyoxidizer/src/distutils/command/build_ext.py @@ -525,10 +525,6 @@ def build_extension(self, ext): for undef in ext.undef_macros: macros.append((undef,)) - # This is needed to activate specific symbol visibility / linking - # behavior on Windows. It isn't needed on UNIX but should be harmless. - macros.append(('Py_BUILD_CORE_BUILTIN', '1')) - objects = self.compiler.compile(sources, output_dir=self.build_temp, macros=macros, diff --git a/pyoxidizer/src/distutils/unixccompiler.py b/pyoxidizer/src/distutils/unixccompiler.py index 26f575fe0..ed25dc376 100644 --- a/pyoxidizer/src/distutils/unixccompiler.py +++ b/pyoxidizer/src/distutils/unixccompiler.py @@ -14,6 +14,7 @@ """ import json, os, sys, re, shutil +import pathlib from distutils import sysconfig from distutils.dep_util import newer @@ -224,19 +225,25 @@ def extension_link_shared_object(self, package=None, ): - if 'PYOXIDIZER_DISTUTILS_STATE_DIR' not in os.environ: - raise Exception('PYOXIDIZER_DISTUTILS_STATE_DIR not defined') + libs=libraries.copy() + if 'crypto' in libs: + libs.remove('crypto') + if 'ssl' in libs: + libs.remove('ssl') self.link(CCompiler.SHARED_OBJECT, objects, output_filename, output_dir, - libraries, library_dirs, runtime_library_dirs, + libs, library_dirs, runtime_library_dirs, export_symbols, debug, extra_preargs, extra_postargs, build_temp, target_lang) # In addition to performing the requested link, we also write out # files that PyOxidizer can use to embed the extension in a larger # binary. - dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + dest_path = str(pathlib.Path(sys.prefix) / 'state' / 'pyoxidizer') + + if not os.path.exists(dest_path): + os.makedirs(dest_path) # We need to copy the object files because they may be in a temp # directory that doesn't outlive this process. @@ -260,6 +267,7 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/py_packaging/distribution.rs b/pyoxidizer/src/py_packaging/distribution.rs index 8a2f36a6e..9328c3ebb 100644 --- a/pyoxidizer/src/py_packaging/distribution.rs +++ b/pyoxidizer/src/py_packaging/distribution.rs @@ -2,31 +2,45 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use copy_dir::copy_dir; use fs2::FileExt; use itertools::Itertools; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use slog::warn; -use std::collections::BTreeMap; +use slog::{info, warn}; +use std::collections::{BTreeMap, HashMap}; +use std::env; use std::fs; use std::fs::{create_dir_all, File}; -use std::io::{Cursor, Read}; +use std::io::{BufRead, BufReader, Cursor, Read}; use std::path::{Path, PathBuf}; use url::Url; use uuid::Uuid; +use which::which; +use super::distutils::prepare_hacked_distutils; use super::fsscan::{ find_python_resources, is_package_from_path, walk_tree_files, PythonFileResource, }; use super::resource::{ResourceData, SourceModule}; + use crate::licensing::NON_GPL_LICENSES; +#[cfg(windows)] +const PYTHON_EXE_BASENAME: &str = "python.exe"; + +#[cfg(unix)] +const PYTHON_EXE_BASENAME: &str = "python3"; + #[cfg(windows)] const PIP_EXE_BASENAME: &str = "pip3.exe"; #[cfg(unix)] const PIP_EXE_BASENAME: &str = "pip3"; +// This needs to be kept in sync with *compiler.py +const PYOXIDIZER_STATE_DIR: &str = "state/pyoxidizer"; + const STDLIB_TEST_PACKAGES: &[&str] = &[ "bsddb.test", "ctypes.test", @@ -311,6 +325,9 @@ pub struct ParsedPythonDistribution { /// Describes license info for things in this distribution. pub license_infos: BTreeMap>, + + /// Path to copy of hacked dist to use for packaging rules venvs + pub venv_base: PathBuf, } #[derive(Debug)] @@ -333,6 +350,102 @@ pub enum ExtensionModuleFilter { NoGPL, } +pub struct PythonPaths { + pub prefix: PathBuf, + pub bin_dir: PathBuf, + pub python_exe: PathBuf, + pub stdlib: PathBuf, + pub site_packages: PathBuf, + pub pyoxidizer_state_dir: PathBuf, +} + +/// Resolve the location of Python modules given a base install path. +pub fn resolve_python_paths(base: &Path, python_version: &str) -> PythonPaths { + let prefix = base.to_path_buf().canonicalize().unwrap(); + + let p = prefix.clone(); + + let bin_dir = if p.join("Scripts").exists() { + p.join("Scripts") + } else { + p.join("bin") + }; + + let python_exe = if bin_dir.join(PYTHON_EXE_BASENAME).exists() { + bin_dir.join(PYTHON_EXE_BASENAME) + } else { + p.join(PYTHON_EXE_BASENAME) + }; + + let mut pyoxidizer_state_dir = p.clone(); + pyoxidizer_state_dir.extend(PYOXIDIZER_STATE_DIR.split('/')); + + let unix_lib_dir = p.join("lib").join(format!("python{}", &python_version[0..3])); + + let stdlib = if unix_lib_dir.exists() { + unix_lib_dir.clone() + } else { + p.join("Lib") + }.canonicalize().unwrap(); + + let site_packages = stdlib.join("site-packages"); + + PythonPaths { + prefix, + bin_dir, + python_exe, + stdlib, + site_packages, + pyoxidizer_state_dir, + } +} + +pub fn invoke_python(python_paths: &PythonPaths, logger: &slog::Logger, args: &[&str]) { + let mut site_packages_s = python_paths + .site_packages + .canonicalize() + .unwrap() + .display() + .to_string(); + + if site_packages_s.starts_with("\\\\?\\") { + site_packages_s = site_packages_s[4..].to_string(); + } + + info!(logger, "setting PYTHONPATH {}", site_packages_s); + + let mut extra_envs = HashMap::new(); + extra_envs.insert("PYTHONPATH".to_string(), site_packages_s); + + info!( + logger, + "running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); + + let mut cmd = std::process::Command::new(&python_paths.python_exe.canonicalize().unwrap()) + .args(args) + .envs(&extra_envs) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect( + format!( + "failed to run {} {}", + python_paths.python_exe.display(), + args.join(" ") + ) + .as_str(), + ); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); + } + } +} + impl ParsedPythonDistribution { pub fn from_path( logger: &slog::Logger, @@ -362,24 +475,142 @@ impl ParsedPythonDistribution { } /// Ensure pip is available to run in the distribution. - pub fn ensure_pip(&self) -> PathBuf { - let pip_path = self - .python_exe - .parent() - .expect("could not derive parent") - .to_path_buf() - .join(PIP_EXE_BASENAME); + /// This is only used by pip.rs + pub fn ensure_pip(&self, logger: &slog::Logger) -> PathBuf { + let dist_prefix = self.base_dir.join("python").join("install"); + let python_paths = resolve_python_paths(&dist_prefix, &self.version); + + let pip_path = python_paths.bin_dir.join(PIP_EXE_BASENAME); if !pip_path.exists() { - std::process::Command::new(&self.python_exe) - .args(&["-m", "ensurepip"]) - .status() - .expect("failed to run ensurepip"); + warn!(logger, "{} doesnt exist", pip_path.display().to_string()); + invoke_python(&python_paths, &logger, &["-m", "ensurepip"]); } pip_path } + /// Duplicate the python distribution, with distutils hacked + pub fn create_hacked_base(&self, logger: &slog::Logger) -> PythonPaths { + let venv_base = self.venv_base.clone(); + + let venv_dir_s = self.venv_base.display().to_string(); + + if !venv_base.exists() { + if self.os == "windows" { + let external_python = which("python").unwrap().clone(); + warn!(logger, "python={}", external_python.display().to_string()); + // TODO: check python version + + let external_dist_prefix = external_python.parent().unwrap(); + + copy_dir(&external_dist_prefix, &venv_base).unwrap(); + + let external_dist_prefix_s = external_dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", external_dist_prefix_s, venv_dir_s + ); + } else { + let dist_prefix = self.base_dir.join("python").join("install"); + + copy_dir(&dist_prefix, &venv_base).unwrap(); + + let dist_prefix_s = dist_prefix.display().to_string(); + warn!( + logger, + "copied {} to create hacked base {}", dist_prefix_s, venv_dir_s + ); + }; + } + + let python_paths = resolve_python_paths(&venv_base, &self.version); + + invoke_python(&python_paths, &logger, &["-m", "ensurepip"]); + + prepare_hacked_distutils(logger, &python_paths); + + python_paths + } + + /// Create a venv from the distribution at path. + pub fn create_venv(&self, logger: &slog::Logger, path: &PathBuf) -> PythonPaths { + let venv_dir_s = path.display().to_string(); + + // This will recreate it, if it was deleted + let python_paths = self.create_hacked_base(&logger); + + if path.exists() { + warn!(logger, "re-using {} {}", "venv", venv_dir_s); + } else { + warn!(logger, "creating {} {}", "venv", venv_dir_s); + invoke_python(&python_paths, &logger, &["-m", "venv", venv_dir_s.as_str()]); + } + + resolve_python_paths(&path, &self.version) + } + + /// Create or re-use an existing venv + pub fn prepare_venv( + &self, + logger: &slog::Logger, + venv_path: Option<&String>, + ) -> (PythonPaths, HashMap) { + let venv_dir_path = match venv_path { + Some(path_str) => PathBuf::from(path_str), + None => tempdir::TempDir::new("pyoxidizer-temp-venv") + .expect("could not create temp directory") + .path() + .join("venv"), + }; + + let python_paths = self.create_venv(logger, &venv_dir_path); + + let mut extra_envs = HashMap::new(); + + let prefix_s = python_paths + .prefix + .canonicalize() + .unwrap() + .display() + .to_string(); + + let venv_path_bin_s = python_paths + .bin_dir + .canonicalize() + .unwrap() + .display() + .to_string(); + + let path_separator = if cfg!(windows) { ";" } else { ":" }; + + let process_path_s = env::var("PATH").unwrap(); + + extra_envs.insert( + "PATH".to_string(), + format!("{}{}{}", venv_path_bin_s, path_separator, process_path_s), + ); + + let mut site_packages_s = python_paths + .site_packages + .canonicalize() + .unwrap() + .display() + .to_string(); + if site_packages_s.starts_with("\\\\?\\") { + site_packages_s = site_packages_s[4..].to_string(); + } + + extra_envs.insert("VIRTUAL_ENV".to_string(), prefix_s); + extra_envs.insert("PYTHONPATH".to_string(), site_packages_s); + + extra_envs.insert("PYOXIDIZER".to_string(), "1".to_string()); + + fs::create_dir_all(&python_paths.pyoxidizer_state_dir).unwrap(); + + (python_paths, extra_envs) + } + /// Obtain resolved `SourceModule` instances for this distribution. /// /// This effectively resolves the raw file content for .py files into @@ -689,6 +920,8 @@ pub fn analyze_python_distribution_data( }; } + let venv_base = dist_dir.parent().unwrap().join("hacked_base"); + Ok(ParsedPythonDistribution { flavor: pi.python_flavor.clone(), version: pi.python_version.clone(), @@ -715,6 +948,7 @@ pub fn analyze_python_distribution_data( py_modules, resources, license_infos, + venv_base, }) } diff --git a/pyoxidizer/src/py_packaging/distutils.rs b/pyoxidizer/src/py_packaging/distutils.rs index 11a932f38..d629e8c77 100644 --- a/pyoxidizer/src/py_packaging/distutils.rs +++ b/pyoxidizer/src/py_packaging/distutils.rs @@ -5,11 +5,11 @@ use lazy_static::lazy_static; use serde::Deserialize; use slog::warn; -use std::collections::{BTreeMap, HashMap}; -use std::fs::{create_dir_all, read_dir, read_to_string}; +use std::collections::BTreeMap; +use std::fs::{read_dir, read_to_string}; use std::path::{Path, PathBuf}; -use super::distribution::ParsedPythonDistribution; +use super::distribution::PythonPaths; use super::resource::BuiltExtensionModule; lazy_static! { @@ -46,70 +46,22 @@ lazy_static! { /// modified distutils will survive multiple process invocations, unlike a /// monkeypatch. People do weird things in setup.py scripts and we want to /// support as many as possible. -pub fn prepare_hacked_distutils( - logger: &slog::Logger, - dist: &ParsedPythonDistribution, - dest_dir: &Path, - extra_python_paths: &[&Path], -) -> Result, String> { - let extra_sys_path = dest_dir.join("packages"); - +pub fn prepare_hacked_distutils(logger: &slog::Logger, target: &PythonPaths) { warn!( logger, "installing modified distutils to {}", - extra_sys_path.display() + target.stdlib.display() ); - let orig_distutils_path = dist.stdlib_path.join("distutils"); - let dest_distutils_path = extra_sys_path.join("distutils"); - - for entry in walkdir::WalkDir::new(&orig_distutils_path) { - match entry { - Ok(entry) => { - if entry.path().is_dir() { - continue; - } - - let source_path = entry.path(); - let rel_path = source_path - .strip_prefix(&orig_distutils_path) - .or_else(|_| Err("unable to strip prefix"))?; - let dest_path = dest_distutils_path.join(rel_path); - - let dest_dir = dest_path.parent().unwrap(); - std::fs::create_dir_all(&dest_dir).or_else(|e| Err(e.to_string()))?; - std::fs::copy(&source_path, &dest_path).or_else(|e| Err(e.to_string()))?; - } - Err(e) => return Err(e.to_string()), - } - } + let dest_distutils_path = target.stdlib.join("distutils"); for (path, data) in MODIFIED_DISTUTILS_FILES.iter() { - let dest_path = dest_distutils_path.join(path); + let mut dest_path = dest_distutils_path.clone(); + dest_path.extend(path.split('/')); warn!(logger, "modifying distutils/{} for oxidation", path); - std::fs::write(dest_path, data).or_else(|e| Err(e.to_string()))?; + std::fs::write(dest_path, data).unwrap(); } - - let state_dir = dest_dir.join("pyoxidizer-build-state"); - create_dir_all(&state_dir).or_else(|e| Err(e.to_string()))?; - - let mut python_paths = vec![extra_sys_path.display().to_string()]; - python_paths.extend(extra_python_paths.iter().map(|p| p.display().to_string())); - - let path_separator = if cfg!(windows) { ";" } else { ":" }; - - let python_path = python_paths.join(path_separator); - - let mut res = HashMap::new(); - res.insert("PYTHONPATH".to_string(), python_path); - res.insert( - "PYOXIDIZER_DISTUTILS_STATE_DIR".to_string(), - state_dir.display().to_string(), - ); - res.insert("PYOXIDIZER".to_string(), "1".to_string()); - - Ok(res) } #[derive(Debug, Deserialize)] diff --git a/pyoxidizer/src/py_packaging/pip.rs b/pyoxidizer/src/py_packaging/pip.rs index d2044b17d..9482f935a 100644 --- a/pyoxidizer/src/py_packaging/pip.rs +++ b/pyoxidizer/src/py_packaging/pip.rs @@ -6,14 +6,18 @@ use slog::warn; use std::collections::HashMap; use std::convert::TryFrom; use std::io::{BufRead, BufReader}; -use std::path::PathBuf; -use super::distribution::ParsedPythonDistribution; +use super::distribution::{ParsedPythonDistribution}; use super::distutils::{prepare_hacked_distutils, read_built_extensions}; use super::fsscan::{find_python_resources, PythonFileResource}; use super::resource::PythonResource; /// Run `pip install` and return found resources. +/// This is unused at the moment as it is based around pip install --target +/// instead of `venv`s. +/// It doesnt use prepare_hacked_distutils as that isnt needed for its test +/// case, and a rewrite is needed to support core Python modules like +/// cffi, cryptography, Cython, PyNaCl, etc. pub fn pip_install( logger: &slog::Logger, dist: &ParsedPythonDistribution, @@ -24,9 +28,9 @@ pub fn pip_install( let temp_dir = tempdir::TempDir::new("pyoxidizer-pip-install") .or_else(|_| Err("could not create temp directory".to_string()))?; - dist.ensure_pip(); + dist.ensure_pip(&logger); - let mut env = prepare_hacked_distutils(logger, dist, temp_dir.path(), &[])?; + let mut env = HashMap::new(); for (key, value) in extra_envs.iter() { env.insert(key.clone(), value.clone()); @@ -98,10 +102,5 @@ pub fn pip_install( } } - let state_dir = PathBuf::from(env.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()); - for ext in read_built_extensions(&state_dir)? { - res.push(PythonResource::BuiltExtensionModule(ext)); - } - Ok(res) } diff --git a/pyoxidizer/src/starlark/python_packaging.rs b/pyoxidizer/src/starlark/python_packaging.rs index 65516d8b3..d6710f2eb 100644 --- a/pyoxidizer/src/starlark/python_packaging.rs +++ b/pyoxidizer/src/starlark/python_packaging.rs @@ -16,8 +16,8 @@ use std::cmp::Ordering; use std::collections::HashMap; use super::env::{ - optional_dict_arg, optional_list_arg, required_bool_arg, required_list_arg, required_str_arg, - required_type_arg, + optional_dict_arg, optional_list_arg, optional_str_arg, required_bool_arg, required_list_arg, + required_str_arg, required_type_arg, }; use crate::app_packaging::config::{ resolve_install_location, PackagingFilterInclude, PackagingPackageRoot, @@ -519,6 +519,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] PipInstallSimple( package, + venv_path=None, extra_env=None, optimize_level=0, excludes=None, @@ -527,6 +528,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let package = required_str_arg("package", &package)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; optional_list_arg("excludes", "string", &excludes)?; @@ -565,6 +567,7 @@ starlark_module! { python_packaging_env => let rule = PackagingPipInstallSimple { package, + venv_path, extra_env, optimize_level, excludes, @@ -579,6 +582,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] PipRequirementsFile( requirements_path, + venv_path=None, extra_env=None, optimize_level=0, include_source=true, @@ -586,6 +590,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let requirements_path = required_str_arg("path", &requirements_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; let include_source = required_bool_arg("include_source", &include_source)?; @@ -618,6 +623,7 @@ starlark_module! { python_packaging_env => }; let rule = PackagingPipRequirementsFile { + venv_path, requirements_path, extra_env, optimize_level, @@ -632,6 +638,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case, clippy::ptr_arg)] SetupPyInstall( package_path, + venv_path=None, extra_env=None, extra_global_arguments=None, optimize_level=0, @@ -640,6 +647,7 @@ starlark_module! { python_packaging_env => excludes=None ) { let package_path = required_str_arg("package_path", &package_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; optional_list_arg("extra_global_arguments", "string", &extra_global_arguments)?; required_type_arg("optimize_level", "int", &optimize_level)?; @@ -676,6 +684,7 @@ starlark_module! { python_packaging_env => let rule = PackagingSetupPyInstall { path: package_path, + venv_path, extra_env, extra_global_arguments, optimize_level: optimize_level.to_int().unwrap(), @@ -890,6 +899,7 @@ mod tests { let v = starlark_ok("PipInstallSimple('foo')"); let wanted = PackagingPipInstallSimple { package: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, excludes: Vec::new(), @@ -914,6 +924,7 @@ mod tests { let v = starlark_ok("PipRequirementsFile('path')"); let wanted = PackagingPipRequirementsFile { requirements_path: "path".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, include_source: true, @@ -928,6 +939,7 @@ mod tests { fn test_pip_requirements_file_extra_args() { let v = starlark_ok("PipRequirementsFile('path', extra_args=['foo'])"); let wanted = PackagingPipRequirementsFile { + venv_path: None, requirements_path: "path".to_string(), extra_env: HashMap::new(), optimize_level: 0, @@ -950,6 +962,7 @@ mod tests { let v = starlark_ok("SetupPyInstall('foo')"); let wanted = PackagingSetupPyInstall { path: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), extra_global_arguments: Vec::new(), optimize_level: 0,