From 89deed638a2fa86cf465bfc2e572c78ec84baf31 Mon Sep 17 00:00:00 2001 From: Thomas Debrunner Date: Tue, 9 Apr 2019 18:20:55 +0200 Subject: [PATCH 1/8] added resolution for @ paths --- .gitignore | 1 + macpack/dependency.py | 51 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index d751ad7..2599150 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ build/ macpack.egg-info/ MANIFEST +.idea diff --git a/macpack/dependency.py b/macpack/dependency.py index 81a73b2..e416af6 100644 --- a/macpack/dependency.py +++ b/macpack/dependency.py @@ -3,12 +3,16 @@ import re import pathlib import subprocess +import re +import os +import sys class Dependency: def __init__(self, path): self.path = pathlib.PosixPath(path).resolve(strict=True) self.symlinks = [] self.dependencies = [] + self.rpaths = [] if self.path != path: self.symlinks.append(str(path)) @@ -42,13 +46,16 @@ def merge(self, dependency): self.dependencies.append(d) async def find_dependencies(self): + # find all rpaths associated with this item + self.rpaths = await self.find_rpaths() + process = await asyncio.create_subprocess_exec('otool', '-L', str(self.path), stdout = subprocess.PIPE, stderr = subprocess.PIPE) (out, err) = await process.communicate() - paths = Dependency.extract_paths_from_output(out.decode('utf-8')) + paths = self.extract_paths_from_output(out.decode('utf-8')) (deps, failed_paths) = Dependency.deps_from_paths(paths) self.dependencies = deps @@ -71,23 +78,57 @@ def get_dependencies(self, is_sys = False): def get_direct_dependencies(self, is_sys = False): return [d for d in self.dependencies if is_sys or not d.is_sys()] - def extract_dep(line): - return line[1:line.find(' (compatibility version ')] + + async def find_rpaths(self): + process = await asyncio.create_subprocess_exec('otool', '-l', str(self.path), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = await process.communicate() + out = out.decode('utf-8') + return re.findall('LC_RPATH\n.*\n.*path ([a-zA-Z0-9/ ]+) \(', out, re.MULTILINE) + + def find_in_rpath(self, library_name): + for rpath in self.rpaths: + if os.path.exists(rpath + library_name): + return rpath + library_name + + return None + + def extract_dep(self, line): + path = line[1:line.find(' (compatibility version ')] + + # if path is relative to loader, we substitute it with the path of the requester + if path.startswith('@loader_path'): + path = path.replace('@loader_path', str(self.path.parent)) + + if path.startswith('@rpath'): + path = self.find_in_rpath(path.replace('@rpath', '')) + + if path is None: + print('Could not resolve %s in rpath' % line, file=sys.stderr) + + return path + def is_dep_line(line): return len(line) > 0 and line[0] == '\t' - def extract_paths_from_output(s): - return [Dependency.extract_dep(l) for l in s.split('\n') if Dependency.is_dep_line(l)] + def extract_paths_from_output(self, s): + return [self.extract_dep(l) for l in s.split('\n') if Dependency.is_dep_line(l)] def deps_from_paths(paths): dependencies = [] failed_paths = [] for path in paths: + if path is None: + continue + try: dependencies.append(Dependency(path)) except FileNotFoundError: failed_paths.append(path) return (dependencies, failed_paths) + + From 8032c15536474e32ff38804332affc0e0444fa62 Mon Sep 17 00:00:00 2001 From: Thomas Debrunner Date: Wed, 10 Apr 2019 10:35:20 +0200 Subject: [PATCH 2/8] changed logic to a pure referral -> file system --- macpack/dependency.py | 79 +++++++++++++++++++++---------------------- macpack/patcher.py | 13 ++++--- 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/macpack/dependency.py b/macpack/dependency.py index e416af6..2e669e5 100644 --- a/macpack/dependency.py +++ b/macpack/dependency.py @@ -7,19 +7,17 @@ import os import sys + class Dependency: - def __init__(self, path): - self.path = pathlib.PosixPath(path).resolve(strict=True) - self.symlinks = [] + def __init__(self, reference, file_path): + self.path = file_path + self.referred_as = {reference} self.dependencies = [] self.rpaths = [] - if self.path != path: - self.symlinks.append(str(path)) - def __repr__(self): return ('Dependency(\'' + str(self.path) + '\', ' - 'symlinks=' + str(len(self.symlinks)) + '\', ' + 'referred_as=' + str(len(self.referred_as)) + '\', ' 'dependencies=' + str(len(self.dependencies)) + ')') def __eq__(self, b): @@ -34,13 +32,9 @@ def is_sys(self): return is_sys - def add_symlink(self, path): - if path not in self.symlinks: - self.symlinks.append(path) - def merge(self, dependency): - for s in dependency.symlinks: - self.add_symlink(s) + for reference in dependency.referred_as: + self.referred_as.add(reference) for d in dependency.dependencies: self.dependencies.append(d) @@ -55,12 +49,12 @@ async def find_dependencies(self): (out, err) = await process.communicate() - paths = self.extract_paths_from_output(out.decode('utf-8')) - (deps, failed_paths) = Dependency.deps_from_paths(paths) + references = self.extract_references_from_output(out.decode('utf-8')) + (deps, failed_references) = self.deps_from_references(references) self.dependencies = deps - return (deps, failed_paths) + return (deps, failed_references) def get_dependencies(self, is_sys = False): stack = [self] @@ -87,48 +81,53 @@ async def find_rpaths(self): out = out.decode('utf-8') return re.findall('LC_RPATH\n.*\n.*path ([a-zA-Z0-9/ ]+) \(', out, re.MULTILINE) - def find_in_rpath(self, library_name): + def resolve_in_rpath(self, library_name): for rpath in self.rpaths: if os.path.exists(rpath + library_name): - return rpath + library_name - + return os.path.realpath(rpath + library_name) return None - def extract_dep(self, line): + def extract_referral(self, line): path = line[1:line.find(' (compatibility version ')] + return path - # if path is relative to loader, we substitute it with the path of the requester - if path.startswith('@loader_path'): - path = path.replace('@loader_path', str(self.path.parent)) - if path.startswith('@rpath'): - path = self.find_in_rpath(path.replace('@rpath', '')) + def is_dep_line(line): + return len(line) > 0 and line[0] == '\t' - if path is None: - print('Could not resolve %s in rpath' % line, file=sys.stderr) + def extract_references_from_output(self, s): + return [self.extract_referral(l) for l in s.split('\n') if Dependency.is_dep_line(l)] - return path + def file_path_from_reference(self, reference): + # if path is relative to loader, we substitute it with the path of the requester + result = reference + if reference.startswith('@loader_path'): + result = reference.replace('@loader_path', str(self.path.parent)) - def is_dep_line(line): - return len(line) > 0 and line[0] == '\t' + if reference.startswith('@rpath'): + result = self.resolve_in_rpath(reference.replace('@rpath', '')) - def extract_paths_from_output(self, s): - return [self.extract_dep(l) for l in s.split('\n') if Dependency.is_dep_line(l)] + if result is None: + print('Could not resolve %s in rpath' % reference, file=sys.stderr) + else: + result = pathlib.PosixPath(result).resolve(strict=True) - def deps_from_paths(paths): + return result + + + def deps_from_references(self, references): dependencies = [] - failed_paths = [] + failed_references = [] - for path in paths: - if path is None: + for reference in references: + if reference is None: continue try: - dependencies.append(Dependency(path)) + dependencies.append(Dependency(reference, self.file_path_from_reference(reference))) except FileNotFoundError: - failed_paths.append(path) - - return (dependencies, failed_paths) + failed_references.append(reference) + return dependencies, failed_references diff --git a/macpack/patcher.py b/macpack/patcher.py index 0a15e50..b40ca4e 100755 --- a/macpack/patcher.py +++ b/macpack/patcher.py @@ -75,14 +75,13 @@ async def patch(root_dep, dest_path, root_loader_path): pargs += ['-id', str(loader_path / dep.path.name)] for dep_dep in dep.get_direct_dependencies(): - pargs += ['-change', str(dep_dep.path), str(loader_path / dep_dep.path.name)] - for symlink in dep_dep.symlinks: - pargs += ['-change', symlink, str(loader_path / dep_dep.path.name)] + for referral in dep_dep.referred_as: + pargs += ['-change', referral, str(loader_path / dep_dep.path.name)] process_coros.append(asyncio.create_subprocess_exec(*pargs, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE - )) + stdout = subprocess.PIPE, + stderr = subprocess.PIPE + )) processes = await asyncio.gather(*process_coros) results = await asyncio.gather(*[p.communicate() for p in processes]) @@ -137,7 +136,7 @@ def get_dest_and_loader_path(root_dep_path, dest_path): def main(): try: - d = dependency.Dependency(args.file) + d = dependency.Dependency(args.file, pathlib.PosixPath(args.file).resolve(strict=True)) except FileNotFoundError: print('{} does not exist!'.format(str(args.file)), file=sys.stderr) sys.exit(1) From fb602e886e43369491b868dc3d69f0868dcca50d Mon Sep 17 00:00:00 2001 From: Thomas Debrunner Date: Thu, 11 Apr 2019 11:37:58 +0200 Subject: [PATCH 3/8] fixed rpath regex to allow all filenames --- macpack/dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macpack/dependency.py b/macpack/dependency.py index 2e669e5..f761b93 100644 --- a/macpack/dependency.py +++ b/macpack/dependency.py @@ -79,7 +79,7 @@ async def find_rpaths(self): stderr=subprocess.PIPE) out, err = await process.communicate() out = out.decode('utf-8') - return re.findall('LC_RPATH\n.*\n.*path ([a-zA-Z0-9/ ]+) \(', out, re.MULTILINE) + return re.findall('LC_RPATH\n.*\n.*path ([a-zA-Z0-9/ .-]+) \(', out, re.MULTILINE) def resolve_in_rpath(self, library_name): for rpath in self.rpaths: From c139bffdc09718c82278f0028bf89c8e4807a690 Mon Sep 17 00:00:00 2001 From: Thomas Debrunner Date: Tue, 7 May 2019 12:48:58 +0200 Subject: [PATCH 4/8] added underscore to rpath resolution, refactored code a bit --- macpack/dependency.py | 232 ++++++++++++++++++------------------- macpack/patcher.py | 260 +++++++++++++++++++++++------------------- 2 files changed, 255 insertions(+), 237 deletions(-) diff --git a/macpack/dependency.py b/macpack/dependency.py index f761b93..43c9e18 100644 --- a/macpack/dependency.py +++ b/macpack/dependency.py @@ -9,125 +9,119 @@ class Dependency: - def __init__(self, reference, file_path): - self.path = file_path - self.referred_as = {reference} - self.dependencies = [] - self.rpaths = [] + def __init__(self, reference, file_path): + self.path = file_path + self.referred_as = {reference} + self.dependencies = [] + self.rpaths = [] + + def __repr__(self): + return ('Dependency(\'' + str(self.path) + '\', ' + 'referred_as=' + str(len(self.referred_as)) + '\', ' + 'dependencies=' + str( + len(self.dependencies)) + ')') + + def __eq__(self, b): + return self.path == b.path + + def is_sys(self): + is_sys = True + try: + self.path.relative_to('/usr/lib') + except ValueError: + is_sys = any((p for p in self.path.parts if re.search('.framework$', p))) + + return is_sys + + def merge(self, dependency): + for reference in dependency.referred_as: + self.referred_as.add(reference) + + for d in dependency.dependencies: + self.dependencies.append(d) + + async def find_dependencies(self): + # find all rpaths associated with this item + self.rpaths = await self.find_rpaths() + + process = await asyncio.create_subprocess_exec('otool', '-L', str(self.path), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + out, err = await process.communicate() + + references = self.extract_references_from_output(out.decode('utf-8')) + deps, failed_references = self.deps_from_references(references) + self.dependencies = deps + return deps, failed_references + + def get_dependencies(self, is_sys=False): + stack = [self] + ret = [] + + while len(stack) > 0: + dep = stack.pop() + for d in dep.get_direct_dependencies(is_sys): + if d not in ret: + ret.append(d) + stack.append(d) + + return ret + + def get_direct_dependencies(self, is_sys=False): + return [d for d in self.dependencies if is_sys or not d.is_sys()] + + async def find_rpaths(self): + process = await asyncio.create_subprocess_exec('otool', '-l', str(self.path), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = await process.communicate() + out = out.decode('utf-8') + return re.findall('LC_RPATH\n.*\n.*path ([a-zA-Z0-9_/ .-]+) \(', out, re.MULTILINE) + + def resolve_in_rpath(self, library_name): + for rpath in self.rpaths: + if os.path.exists(rpath + library_name): + return os.path.realpath(rpath + library_name) + return None + + def extract_referral(self, line): + path = line[1:line.find(' (compatibility version ')] + return path + + def is_dep_line(line): + return len(line) > 0 and line[0] == '\t' + + def extract_references_from_output(self, s): + return [self.extract_referral(l) for l in s.split('\n') if Dependency.is_dep_line(l)] + + def file_path_from_reference(self, reference): + # if path is relative to loader, we substitute it with the path of the requester + result = reference + if reference.startswith('@loader_path'): + result = reference.replace('@loader_path', str(self.path.parent)) + + if reference.startswith('@rpath'): + result = self.resolve_in_rpath(reference.replace('@rpath', '')) + + if result is None: + print('Could not resolve %s in rpath' % reference, file=sys.stderr) + else: + result = pathlib.PosixPath(result).resolve(strict=True) + + return result + + def deps_from_references(self, references): + dependencies = [] + failed_references = [] - def __repr__(self): - return ('Dependency(\'' + str(self.path) + '\', ' - 'referred_as=' + str(len(self.referred_as)) + '\', ' - 'dependencies=' + str(len(self.dependencies)) + ')') + for reference in references: + if reference is None: + continue - def __eq__(self, b): - return self.path == b.path - - def is_sys(self): - is_sys = True - try: - self.path.relative_to('/usr/lib') - except ValueError: - is_sys = any((p for p in self.path.parts if re.search('.framework$', p))) - - return is_sys - - def merge(self, dependency): - for reference in dependency.referred_as: - self.referred_as.add(reference) - - for d in dependency.dependencies: - self.dependencies.append(d) - - async def find_dependencies(self): - # find all rpaths associated with this item - self.rpaths = await self.find_rpaths() - - process = await asyncio.create_subprocess_exec('otool', '-L', str(self.path), - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - - (out, err) = await process.communicate() - - references = self.extract_references_from_output(out.decode('utf-8')) - (deps, failed_references) = self.deps_from_references(references) - - self.dependencies = deps - - return (deps, failed_references) - - def get_dependencies(self, is_sys = False): - stack = [self] - ret = [] - - while len(stack) > 0: - dep = stack.pop() - for d in dep.get_direct_dependencies(is_sys): - if d not in ret: - ret.append(d) - stack.append(d) - - return ret - - def get_direct_dependencies(self, is_sys = False): - return [d for d in self.dependencies if is_sys or not d.is_sys()] - - - async def find_rpaths(self): - process = await asyncio.create_subprocess_exec('otool', '-l', str(self.path), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = await process.communicate() - out = out.decode('utf-8') - return re.findall('LC_RPATH\n.*\n.*path ([a-zA-Z0-9/ .-]+) \(', out, re.MULTILINE) - - def resolve_in_rpath(self, library_name): - for rpath in self.rpaths: - if os.path.exists(rpath + library_name): - return os.path.realpath(rpath + library_name) - return None - - def extract_referral(self, line): - path = line[1:line.find(' (compatibility version ')] - return path - - - def is_dep_line(line): - return len(line) > 0 and line[0] == '\t' - - def extract_references_from_output(self, s): - return [self.extract_referral(l) for l in s.split('\n') if Dependency.is_dep_line(l)] - - - def file_path_from_reference(self, reference): - # if path is relative to loader, we substitute it with the path of the requester - result = reference - if reference.startswith('@loader_path'): - result = reference.replace('@loader_path', str(self.path.parent)) - - if reference.startswith('@rpath'): - result = self.resolve_in_rpath(reference.replace('@rpath', '')) - - if result is None: - print('Could not resolve %s in rpath' % reference, file=sys.stderr) - else: - result = pathlib.PosixPath(result).resolve(strict=True) - - return result - - - def deps_from_references(self, references): - dependencies = [] - failed_references = [] - - for reference in references: - if reference is None: - continue - - try: - dependencies.append(Dependency(reference, self.file_path_from_reference(reference))) - except FileNotFoundError: - failed_references.append(reference) - - return dependencies, failed_references + try: + dependencies.append(Dependency(reference, self.file_path_from_reference(reference))) + except FileNotFoundError: + failed_references.append(reference) + return dependencies, failed_references diff --git a/macpack/patcher.py b/macpack/patcher.py index b40ca4e..b024044 100755 --- a/macpack/patcher.py +++ b/macpack/patcher.py @@ -10,165 +10,189 @@ from macpack import dependency + class PatchError(Exception): - pass + pass + async def collect(root_dep): - failed_paths = [] - all_resolved = [] - stack = [[root_dep]] - - while len(stack) > 0: - resolving_deps = stack.pop() - results = await asyncio.gather(*[d.find_dependencies() for d in resolving_deps]) - all_resolved += resolving_deps - - to_resolve = [] - for i, deps_and_failed in enumerate(results): - failed_paths += deps_and_failed[1] - - for j, dep in enumerate(deps_and_failed[0]): - if not dep.is_sys(): - if dep in all_resolved: - existing_dep = all_resolved[all_resolved.index(dep)] - elif dep in to_resolve: - existing_dep = to_resolve[to_resolve.index(dep)] - else: - existing_dep = None - - if existing_dep: - existing_dep.merge(dep) - resolving_deps[i].dependencies[j] = existing_dep - else: - to_resolve.append(dep) - - if len(to_resolve) > 0: stack.append(to_resolve) - - if len(failed_paths): - print('Some of the paths in the dependency tree could not be resolved', file=sys.stderr) - print('Maybe you already bundled {}?'.format(root_dep.path.name), file=sys.stderr) - if (args.verbose): - for path in failed_paths: - print('Could not resolve {}'.format(path), file=sys.stderr) - else: - print('Run with -v to see failed paths', file=sys.stderr) + failed_paths = [] + all_resolved = [] + stack = [[root_dep]] + + while len(stack) > 0: + # pop one item from stack, and collect all its dependencies + current_items = stack.pop() + all_resolved += current_items + current_deps_and_fails = await asyncio.gather(*[d.find_dependencies() for d in current_items]) + + for _, fails in current_deps_and_fails: + failed_paths += fails + current_items_deps = [deps_and_fail[0] for deps_and_fail in current_deps_and_fails] + + to_resolve = [] + for item, item_deps in zip(current_items, current_items_deps): + + # for every dep of the current item, try to see if we already resolved it before. If not, we + # still have to resolve it -> push it onto the stack + for dep in item_deps: + if not dep.is_sys(): + # check if we have seen that dependency before + if dep in all_resolved: + existing_dep = all_resolved[all_resolved.index(dep)] + elif dep in to_resolve: + existing_dep = to_resolve[to_resolve.index(dep)] + else: + existing_dep = None + + if existing_dep: + # if we have already seen it before, merge the references into + # the existing dependency, and assign that one as the items dependency + existing_dep.merge(dep) + item.dependencies[item.dependencies.index(dep)] = existing_dep + else: + to_resolve.append(dep) + + if len(to_resolve) > 0: + stack.append(to_resolve) + + if len(failed_paths) > 0: + print('Some of the paths in the dependency tree could not be resolved', file=sys.stderr) + print('Maybe you already bundled {}?'.format(root_dep.path.name), file=sys.stderr) + if args.verbose: + for path in failed_paths: + print('Could not resolve {}'.format(path), file=sys.stderr) + else: + print('Run with -v to see failed paths', file=sys.stderr) + def ensure_dir(path): - if not os.path.exists(str(path)): - os.makedirs(str(path)) + if not os.path.exists(str(path)): + os.makedirs(str(path)) + async def patch(root_dep, dest_path, root_loader_path): - process_coros = [] - patch_deps = [root_dep] + root_dep.get_dependencies() + process_coros = [] + patch_deps = [root_dep] + root_dep.get_dependencies() - ensure_dir(dest_path) + ensure_dir(dest_path) - for dep in patch_deps: - if dep == root_dep: - pargs = ['install_name_tool', str(root_dep.path)] - loader_path = root_loader_path - else: - shutil.copyfile(str(dep.path), str(dest_path / dep.path.name)) - pargs = ['install_name_tool', str(dest_path / dep.path.name)] - loader_path = pathlib.PurePath('@loader_path') + for dep in patch_deps: + if dep == root_dep: + pargs = ['install_name_tool', str(root_dep.path)] + loader_path = root_loader_path + else: + shutil.copyfile(str(dep.path), str(dest_path / dep.path.name)) + pargs = ['install_name_tool', str(dest_path / dep.path.name)] + loader_path = pathlib.PurePath('@loader_path') + + pargs += ['-id', str(loader_path / dep.path.name)] - pargs += ['-id', str(loader_path / dep.path.name)] + for dep_dep in dep.get_direct_dependencies(): + for referral in dep_dep.referred_as: + pargs += ['-change', referral, str(loader_path / dep_dep.path.name)] - for dep_dep in dep.get_direct_dependencies(): - for referral in dep_dep.referred_as: - pargs += ['-change', referral, str(loader_path / dep_dep.path.name)] + process_coros.append(asyncio.create_subprocess_exec(*pargs, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + )) - process_coros.append(asyncio.create_subprocess_exec(*pargs, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE - )) + processes = await asyncio.gather(*process_coros) + results = await asyncio.gather(*[p.communicate() for p in processes]) - processes = await asyncio.gather(*process_coros) - results = await asyncio.gather(*[p.communicate() for p in processes]) + did_error = False + for process, (out, err), dep in zip(processes, results, patch_deps): + if process.returncode: + did_error = True + print('Error patching {}'.format(str(dep.path.name)), file=sys.stderr) + if args.verbose: + print(err.decode('utf-8')) - did_error = False - for process, (out, err), dep in zip(processes, results, patch_deps): - if process.returncode: - did_error = True - print('Error patching {}'.format(str(dep.path.name)), file=sys.stderr) - if args.verbose: - print(err.decode('utf-8')) + if did_error: + raise PatchError('One or more dependencies could not be patched') - if did_error: raise PatchError('One or more dependencies could not be patched') def print_deps_minimal(d): - deps = d.get_dependencies() + deps = d.get_dependencies() - print(str(len(deps)) + ' total non-system dependenc{}'.format('y' if len(deps) == 1 else 'ies')) + print(str(len(deps)) + ' total non-system dependenc{}'.format('y' if len(deps) == 1 else 'ies')) + + for i, dep in enumerate(deps): + dep_slots = [str(deps.index(d) + 1) for d in dep.get_direct_dependencies()] + s = ', '.join(dep_slots) if len(dep_slots) > 0 else 'No dependencies' + print(str(i + 1) + '\t' + dep.path.name + ' -> ' + s) - for i, dep in enumerate(deps): - dep_slots = [str(deps.index(d) + 1) for d in dep.get_direct_dependencies()] - s = ', '.join(dep_slots) if len(dep_slots) > 0 else 'No dependencies' - print(str(i+1) + '\t' + dep.path.name + ' -> ' + s) def print_deps(d): - deps = d.get_dependencies() + deps = d.get_dependencies() + + print(str(len(deps)) + ' total non-system dependenc{}'.format('y' if len(deps) == 1 else 'ies')) - print(str(len(deps)) + ' total non-system dependenc{}'.format('y' if len(deps) == 1 else 'ies')) + for dep in deps: + print(dep.path.name) + for dep_dep in dep.get_dependencies(): + print('-> ' + dep_dep.path.name) - for dep in deps: - print(dep.path.name) - for dep_dep in dep.get_dependencies(): - print('-> ' + dep_dep.path.name) def prepatch_output(d): - print("Patching {}".format(str(args.file))) + print("Patching {}".format(str(args.file))) + + if args.verbose: + print_deps(d) + else: + print_deps_minimal(d) - if args.verbose: - print_deps(d) - else: - print_deps_minimal(d) def get_dest_and_loader_path(root_dep_path, dest_path): - if dest_path.is_absolute(): - loader_path = dest_path - else: - dest_path = root_dep_path.parent / dest_path - rel_to_binary = os.path.relpath(str(dest_path), str(root_dep_path.parent)) - loader_path = pathlib.PurePath('@loader_path', rel_to_binary) + if dest_path.is_absolute(): + loader_path = dest_path + else: + dest_path = root_dep_path.parent / dest_path + rel_to_binary = os.path.relpath(str(dest_path), str(root_dep_path.parent)) + loader_path = pathlib.PurePath('@loader_path', rel_to_binary) + + return (dest_path, loader_path) - return (dest_path, loader_path) def main(): - try: - d = dependency.Dependency(args.file, pathlib.PosixPath(args.file).resolve(strict=True)) - except FileNotFoundError: - print('{} does not exist!'.format(str(args.file)), file=sys.stderr) - sys.exit(1) + try: + d = dependency.Dependency(args.file, pathlib.PosixPath(args.file).resolve(strict=True)) + except FileNotFoundError: + print('{} does not exist!'.format(str(args.file)), file=sys.stderr) + sys.exit(1) - loop = asyncio.get_event_loop() + loop = asyncio.get_event_loop() - loop.run_until_complete(collect(d)) + loop.run_until_complete(collect(d)) - dest_path, root_loader_path = get_dest_and_loader_path(d.path, args.destination) + dest_path, root_loader_path = get_dest_and_loader_path(d.path, args.destination) - prepatch_output(d) + prepatch_output(d) - if not args.dry_run: - try: - loop.run_until_complete(patch(d, dest_path, root_loader_path)) - except PatchError: # the error should have been already printed here - if not args.verbose: print('Run with -v for more information', file=sys.stderr) - sys.exit(1) + if not args.dry_run: + try: + loop.run_until_complete(patch(d, dest_path, root_loader_path)) + except PatchError: # the error should have been already printed here + if not args.verbose: print('Run with -v for more information', file=sys.stderr) + sys.exit(1) - n_deps = len(d.get_dependencies()) - print() - print('{} + {} dependenc{} successfully patched'.format(args.file.name, n_deps, 'y' if n_deps == 1 else 'ies')) + n_deps = len(d.get_dependencies()) + print() + print('{} + {} dependenc{} successfully patched'.format(args.file.name, n_deps, 'y' if n_deps == 1 else 'ies')) - loop.close() + loop.close() -parser = argparse.ArgumentParser(description='Copies non-system libraries used by your executable and patches them to work as a standalone bundle') + +parser = argparse.ArgumentParser( + description='Copies non-system libraries used by your executable and patches them to work as a standalone bundle') parser.add_argument('file', help='file to patch (the root, main binary)', type=pathlib.PurePath) -parser.add_argument('-v', '--verbose', help='displays more library information and output of install_name_tool', action='store_true') -parser.add_argument('-n', '--dry-run', help='just show the dependency tree but do not do any patching', action='store_true') -parser.add_argument('-d', '--destination', help='destination directory where the binaries will be placed and loaded', type=pathlib.Path, default='../libs') +parser.add_argument('-v', '--verbose', help='displays more library information and output of install_name_tool', + action='store_true') +parser.add_argument('-n', '--dry-run', help='just show the dependency tree but do not do any patching', + action='store_true') +parser.add_argument('-d', '--destination', help='destination directory where the binaries will be placed and loaded', + type=pathlib.Path, default='../libs') args = parser.parse_args() # Allow execution of this file directly (it's executed differently when installed) if __name__ == '__main__': main() - From 6c059c7df75293551ffeb8f8de1b42d02d77f9f4 Mon Sep 17 00:00:00 2001 From: Thomas Debrunner Date: Tue, 7 May 2019 14:53:37 +0200 Subject: [PATCH 5/8] always add loader path into possible rpaths as well --- macpack/dependency.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/macpack/dependency.py b/macpack/dependency.py index 43c9e18..0e9ff89 100644 --- a/macpack/dependency.py +++ b/macpack/dependency.py @@ -77,7 +77,10 @@ async def find_rpaths(self): stderr=subprocess.PIPE) out, err = await process.communicate() out = out.decode('utf-8') - return re.findall('LC_RPATH\n.*\n.*path ([a-zA-Z0-9_/ .-]+) \(', out, re.MULTILINE) + rpaths = re.findall('LC_RPATH\n.*\n.*path ([a-zA-Z0-9_/ .-]+) \(', out, re.MULTILINE) + # always add current path to be a valid rpath + rpaths.append(str(self.path.parent)) + return rpaths def resolve_in_rpath(self, library_name): for rpath in self.rpaths: From b926ace920e0dfab57733adcc538f85c4438a528 Mon Sep 17 00:00:00 2001 From: Thomas Debrunner Date: Tue, 7 May 2019 15:07:36 +0200 Subject: [PATCH 6/8] fixed bug that caused self rpath not to patch --- macpack/dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macpack/dependency.py b/macpack/dependency.py index 0e9ff89..73e3862 100644 --- a/macpack/dependency.py +++ b/macpack/dependency.py @@ -11,7 +11,7 @@ class Dependency: def __init__(self, reference, file_path): self.path = file_path - self.referred_as = {reference} + self.referred_as = {str(reference)} self.dependencies = [] self.rpaths = [] From 33dab83f688ce8623ed53f1c5cb92bc66f3b930e Mon Sep 17 00:00:00 2001 From: Thomas Debrunner Date: Tue, 7 May 2019 16:13:39 +0200 Subject: [PATCH 7/8] added functionality to properly resolve self references --- macpack/patcher.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/macpack/patcher.py b/macpack/patcher.py index b024044..2cd1800 100755 --- a/macpack/patcher.py +++ b/macpack/patcher.py @@ -71,26 +71,28 @@ def ensure_dir(path): os.makedirs(str(path)) -async def patch(root_dep, dest_path, root_loader_path): +async def patch(root_item, dest_path, root_loader_path): process_coros = [] - patch_deps = [root_dep] + root_dep.get_dependencies() + items = [root_item] + root_item.get_dependencies() ensure_dir(dest_path) - for dep in patch_deps: - if dep == root_dep: - pargs = ['install_name_tool', str(root_dep.path)] - loader_path = root_loader_path + for item in items: + loader_path = pathlib.PurePath('@loader_path') + + if item == root_item: + pargs = ['install_name_tool', str(root_item.path)] else: - shutil.copyfile(str(dep.path), str(dest_path / dep.path.name)) - pargs = ['install_name_tool', str(dest_path / dep.path.name)] - loader_path = pathlib.PurePath('@loader_path') + shutil.copyfile(str(item.path), str(dest_path / item.path.name)) + pargs = ['install_name_tool', str(dest_path / item.path.name)] - pargs += ['-id', str(loader_path / dep.path.name)] + pargs += ['-id', str(loader_path / item.path.name)] - for dep_dep in dep.get_direct_dependencies(): - for referral in dep_dep.referred_as: - pargs += ['-change', referral, str(loader_path / dep_dep.path.name)] + for dep in item.get_direct_dependencies(): + for reference in dep.referred_as: + new_path = loader_path if item != root_item else root_loader_path + new_path = new_path if dep != root_item else loader_path + pargs += ['-change', reference, str(new_path / dep.path.name)] process_coros.append(asyncio.create_subprocess_exec(*pargs, stdout=subprocess.PIPE, @@ -101,7 +103,7 @@ async def patch(root_dep, dest_path, root_loader_path): results = await asyncio.gather(*[p.communicate() for p in processes]) did_error = False - for process, (out, err), dep in zip(processes, results, patch_deps): + for process, (out, err), dep in zip(processes, results, items): if process.returncode: did_error = True print('Error patching {}'.format(str(dep.path.name)), file=sys.stderr) From 20b518e9bc0f4e58d47c5416a686a4b246a3764d Mon Sep 17 00:00:00 2001 From: Thomas Debrunner Date: Wed, 5 Feb 2020 17:42:21 +0100 Subject: [PATCH 8/8] serialize the patching of libraries --- macpack/patcher.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/macpack/patcher.py b/macpack/patcher.py index 2cd1800..47fba42 100755 --- a/macpack/patcher.py +++ b/macpack/patcher.py @@ -72,11 +72,12 @@ def ensure_dir(path): async def patch(root_item, dest_path, root_loader_path): - process_coros = [] items = [root_item] + root_item.get_dependencies() ensure_dir(dest_path) + process_results = [] + for item in items: loader_path = pathlib.PurePath('@loader_path') @@ -94,17 +95,16 @@ async def patch(root_item, dest_path, root_loader_path): new_path = new_path if dep != root_item else loader_path pargs += ['-change', reference, str(new_path / dep.path.name)] - process_coros.append(asyncio.create_subprocess_exec(*pargs, + process = await asyncio.create_subprocess_exec(*pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE - )) - - processes = await asyncio.gather(*process_coros) - results = await asyncio.gather(*[p.communicate() for p in processes]) + ) + out, err = await process.communicate() + process_results.append((process.returncode, out, err)) did_error = False - for process, (out, err), dep in zip(processes, results, items): - if process.returncode: + for (returncode, out, err), dep in zip(process_results, items): + if returncode: did_error = True print('Error patching {}'.format(str(dep.path.name)), file=sys.stderr) if args.verbose: