diff --git a/.gitignore b/.gitignore index d751ad7..2599150 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ build/ macpack.egg-info/ MANIFEST +.idea diff --git a/macpack/dependency.py b/macpack/dependency.py index 81a73b2..73e3862 100644 --- a/macpack/dependency.py +++ b/macpack/dependency.py @@ -3,91 +3,128 @@ import re import pathlib import subprocess +import re +import os +import sys -class Dependency: - def __init__(self, path): - self.path = pathlib.PosixPath(path).resolve(strict=True) - self.symlinks = [] - self.dependencies = [] - - if self.path != path: - self.symlinks.append(str(path)) - - def __repr__(self): - return ('Dependency(\'' + str(self.path) + '\', ' - 'symlinks=' + str(len(self.symlinks)) + '\', ' - 'dependencies=' + str(len(self.dependencies)) + ')') - - def __eq__(self, b): - return self.path == b.path - - def is_sys(self): - is_sys = True - try: - self.path.relative_to('/usr/lib') - except ValueError: - is_sys = any((p for p in self.path.parts if re.search('.framework$', p))) - - return is_sys - - def add_symlink(self, path): - if path not in self.symlinks: - self.symlinks.append(path) - - def merge(self, dependency): - for s in dependency.symlinks: - self.add_symlink(s) - - for d in dependency.dependencies: - self.dependencies.append(d) - - async def find_dependencies(self): - process = await asyncio.create_subprocess_exec('otool', '-L', str(self.path), - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - - (out, err) = await process.communicate() - - paths = Dependency.extract_paths_from_output(out.decode('utf-8')) - (deps, failed_paths) = Dependency.deps_from_paths(paths) - - self.dependencies = deps - - return (deps, failed_paths) - - def get_dependencies(self, is_sys = False): - stack = [self] - ret = [] - - while len(stack) > 0: - dep = stack.pop() - for d in dep.get_direct_dependencies(is_sys): - if d not in ret: - ret.append(d) - stack.append(d) - - return ret - - def get_direct_dependencies(self, is_sys = False): - return [d for d in self.dependencies if is_sys or not d.is_sys()] - - def extract_dep(line): - return line[1:line.find(' (compatibility version ')] - - def is_dep_line(line): - return len(line) > 0 and line[0] == '\t' - - def extract_paths_from_output(s): - return [Dependency.extract_dep(l) for l in s.split('\n') if Dependency.is_dep_line(l)] - - def deps_from_paths(paths): - dependencies = [] - failed_paths = [] - - for path in paths: - try: - dependencies.append(Dependency(path)) - except FileNotFoundError: - failed_paths.append(path) - return (dependencies, failed_paths) +class Dependency: + def __init__(self, reference, file_path): + self.path = file_path + self.referred_as = {str(reference)} + self.dependencies = [] + self.rpaths = [] + + def __repr__(self): + return ('Dependency(\'' + str(self.path) + '\', ' + 'referred_as=' + str(len(self.referred_as)) + '\', ' + 'dependencies=' + str( + len(self.dependencies)) + ')') + + def __eq__(self, b): + return self.path == b.path + + def is_sys(self): + is_sys = True + try: + self.path.relative_to('/usr/lib') + except ValueError: + is_sys = any((p for p in self.path.parts if re.search('.framework$', p))) + + return is_sys + + def merge(self, dependency): + for reference in dependency.referred_as: + self.referred_as.add(reference) + + for d in dependency.dependencies: + self.dependencies.append(d) + + async def find_dependencies(self): + # find all rpaths associated with this item + self.rpaths = await self.find_rpaths() + + process = await asyncio.create_subprocess_exec('otool', '-L', str(self.path), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + out, err = await process.communicate() + + references = self.extract_references_from_output(out.decode('utf-8')) + deps, failed_references = self.deps_from_references(references) + self.dependencies = deps + return deps, failed_references + + def get_dependencies(self, is_sys=False): + stack = [self] + ret = [] + + while len(stack) > 0: + dep = stack.pop() + for d in dep.get_direct_dependencies(is_sys): + if d not in ret: + ret.append(d) + stack.append(d) + + return ret + + def get_direct_dependencies(self, is_sys=False): + return [d for d in self.dependencies if is_sys or not d.is_sys()] + + async def find_rpaths(self): + process = await asyncio.create_subprocess_exec('otool', '-l', str(self.path), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = await process.communicate() + out = out.decode('utf-8') + rpaths = re.findall('LC_RPATH\n.*\n.*path ([a-zA-Z0-9_/ .-]+) \(', out, re.MULTILINE) + # always add current path to be a valid rpath + rpaths.append(str(self.path.parent)) + return rpaths + + def resolve_in_rpath(self, library_name): + for rpath in self.rpaths: + if os.path.exists(rpath + library_name): + return os.path.realpath(rpath + library_name) + return None + + def extract_referral(self, line): + path = line[1:line.find(' (compatibility version ')] + return path + + def is_dep_line(line): + return len(line) > 0 and line[0] == '\t' + + def extract_references_from_output(self, s): + return [self.extract_referral(l) for l in s.split('\n') if Dependency.is_dep_line(l)] + + def file_path_from_reference(self, reference): + # if path is relative to loader, we substitute it with the path of the requester + result = reference + if reference.startswith('@loader_path'): + result = reference.replace('@loader_path', str(self.path.parent)) + + if reference.startswith('@rpath'): + result = self.resolve_in_rpath(reference.replace('@rpath', '')) + + if result is None: + print('Could not resolve %s in rpath' % reference, file=sys.stderr) + else: + result = pathlib.PosixPath(result).resolve(strict=True) + + return result + + def deps_from_references(self, references): + dependencies = [] + failed_references = [] + + for reference in references: + if reference is None: + continue + + try: + dependencies.append(Dependency(reference, self.file_path_from_reference(reference))) + except FileNotFoundError: + failed_references.append(reference) + + return dependencies, failed_references diff --git a/macpack/patcher.py b/macpack/patcher.py index 0a15e50..47fba42 100755 --- a/macpack/patcher.py +++ b/macpack/patcher.py @@ -10,166 +10,191 @@ from macpack import dependency + class PatchError(Exception): - pass + pass + async def collect(root_dep): - failed_paths = [] - all_resolved = [] - stack = [[root_dep]] - - while len(stack) > 0: - resolving_deps = stack.pop() - results = await asyncio.gather(*[d.find_dependencies() for d in resolving_deps]) - all_resolved += resolving_deps - - to_resolve = [] - for i, deps_and_failed in enumerate(results): - failed_paths += deps_and_failed[1] - - for j, dep in enumerate(deps_and_failed[0]): - if not dep.is_sys(): - if dep in all_resolved: - existing_dep = all_resolved[all_resolved.index(dep)] - elif dep in to_resolve: - existing_dep = to_resolve[to_resolve.index(dep)] - else: - existing_dep = None - - if existing_dep: - existing_dep.merge(dep) - resolving_deps[i].dependencies[j] = existing_dep - else: - to_resolve.append(dep) - - if len(to_resolve) > 0: stack.append(to_resolve) - - if len(failed_paths): - print('Some of the paths in the dependency tree could not be resolved', file=sys.stderr) - print('Maybe you already bundled {}?'.format(root_dep.path.name), file=sys.stderr) - if (args.verbose): - for path in failed_paths: - print('Could not resolve {}'.format(path), file=sys.stderr) - else: - print('Run with -v to see failed paths', file=sys.stderr) + failed_paths = [] + all_resolved = [] + stack = [[root_dep]] + + while len(stack) > 0: + # pop one item from stack, and collect all its dependencies + current_items = stack.pop() + all_resolved += current_items + current_deps_and_fails = await asyncio.gather(*[d.find_dependencies() for d in current_items]) + + for _, fails in current_deps_and_fails: + failed_paths += fails + current_items_deps = [deps_and_fail[0] for deps_and_fail in current_deps_and_fails] + + to_resolve = [] + for item, item_deps in zip(current_items, current_items_deps): + + # for every dep of the current item, try to see if we already resolved it before. If not, we + # still have to resolve it -> push it onto the stack + for dep in item_deps: + if not dep.is_sys(): + # check if we have seen that dependency before + if dep in all_resolved: + existing_dep = all_resolved[all_resolved.index(dep)] + elif dep in to_resolve: + existing_dep = to_resolve[to_resolve.index(dep)] + else: + existing_dep = None + + if existing_dep: + # if we have already seen it before, merge the references into + # the existing dependency, and assign that one as the items dependency + existing_dep.merge(dep) + item.dependencies[item.dependencies.index(dep)] = existing_dep + else: + to_resolve.append(dep) + + if len(to_resolve) > 0: + stack.append(to_resolve) + + if len(failed_paths) > 0: + print('Some of the paths in the dependency tree could not be resolved', file=sys.stderr) + print('Maybe you already bundled {}?'.format(root_dep.path.name), file=sys.stderr) + if args.verbose: + for path in failed_paths: + print('Could not resolve {}'.format(path), file=sys.stderr) + else: + print('Run with -v to see failed paths', file=sys.stderr) + def ensure_dir(path): - if not os.path.exists(str(path)): - os.makedirs(str(path)) + if not os.path.exists(str(path)): + os.makedirs(str(path)) -async def patch(root_dep, dest_path, root_loader_path): - process_coros = [] - patch_deps = [root_dep] + root_dep.get_dependencies() - ensure_dir(dest_path) +async def patch(root_item, dest_path, root_loader_path): + items = [root_item] + root_item.get_dependencies() - for dep in patch_deps: - if dep == root_dep: - pargs = ['install_name_tool', str(root_dep.path)] - loader_path = root_loader_path - else: - shutil.copyfile(str(dep.path), str(dest_path / dep.path.name)) - pargs = ['install_name_tool', str(dest_path / dep.path.name)] - loader_path = pathlib.PurePath('@loader_path') + ensure_dir(dest_path) - pargs += ['-id', str(loader_path / dep.path.name)] + process_results = [] - for dep_dep in dep.get_direct_dependencies(): - pargs += ['-change', str(dep_dep.path), str(loader_path / dep_dep.path.name)] - for symlink in dep_dep.symlinks: - pargs += ['-change', symlink, str(loader_path / dep_dep.path.name)] + for item in items: + loader_path = pathlib.PurePath('@loader_path') - process_coros.append(asyncio.create_subprocess_exec(*pargs, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE - )) + if item == root_item: + pargs = ['install_name_tool', str(root_item.path)] + else: + shutil.copyfile(str(item.path), str(dest_path / item.path.name)) + pargs = ['install_name_tool', str(dest_path / item.path.name)] - processes = await asyncio.gather(*process_coros) - results = await asyncio.gather(*[p.communicate() for p in processes]) + pargs += ['-id', str(loader_path / item.path.name)] - did_error = False - for process, (out, err), dep in zip(processes, results, patch_deps): - if process.returncode: - did_error = True - print('Error patching {}'.format(str(dep.path.name)), file=sys.stderr) - if args.verbose: - print(err.decode('utf-8')) + for dep in item.get_direct_dependencies(): + for reference in dep.referred_as: + new_path = loader_path if item != root_item else root_loader_path + new_path = new_path if dep != root_item else loader_path + pargs += ['-change', reference, str(new_path / dep.path.name)] + + process = await asyncio.create_subprocess_exec(*pargs, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + out, err = await process.communicate() + process_results.append((process.returncode, out, err)) + + did_error = False + for (returncode, out, err), dep in zip(process_results, items): + if returncode: + did_error = True + print('Error patching {}'.format(str(dep.path.name)), file=sys.stderr) + if args.verbose: + print(err.decode('utf-8')) + + if did_error: + raise PatchError('One or more dependencies could not be patched') - if did_error: raise PatchError('One or more dependencies could not be patched') def print_deps_minimal(d): - deps = d.get_dependencies() + deps = d.get_dependencies() + + print(str(len(deps)) + ' total non-system dependenc{}'.format('y' if len(deps) == 1 else 'ies')) - print(str(len(deps)) + ' total non-system dependenc{}'.format('y' if len(deps) == 1 else 'ies')) + for i, dep in enumerate(deps): + dep_slots = [str(deps.index(d) + 1) for d in dep.get_direct_dependencies()] + s = ', '.join(dep_slots) if len(dep_slots) > 0 else 'No dependencies' + print(str(i + 1) + '\t' + dep.path.name + ' -> ' + s) - for i, dep in enumerate(deps): - dep_slots = [str(deps.index(d) + 1) for d in dep.get_direct_dependencies()] - s = ', '.join(dep_slots) if len(dep_slots) > 0 else 'No dependencies' - print(str(i+1) + '\t' + dep.path.name + ' -> ' + s) def print_deps(d): - deps = d.get_dependencies() + deps = d.get_dependencies() - print(str(len(deps)) + ' total non-system dependenc{}'.format('y' if len(deps) == 1 else 'ies')) + print(str(len(deps)) + ' total non-system dependenc{}'.format('y' if len(deps) == 1 else 'ies')) + + for dep in deps: + print(dep.path.name) + for dep_dep in dep.get_dependencies(): + print('-> ' + dep_dep.path.name) - for dep in deps: - print(dep.path.name) - for dep_dep in dep.get_dependencies(): - print('-> ' + dep_dep.path.name) def prepatch_output(d): - print("Patching {}".format(str(args.file))) + print("Patching {}".format(str(args.file))) + + if args.verbose: + print_deps(d) + else: + print_deps_minimal(d) - if args.verbose: - print_deps(d) - else: - print_deps_minimal(d) def get_dest_and_loader_path(root_dep_path, dest_path): - if dest_path.is_absolute(): - loader_path = dest_path - else: - dest_path = root_dep_path.parent / dest_path - rel_to_binary = os.path.relpath(str(dest_path), str(root_dep_path.parent)) - loader_path = pathlib.PurePath('@loader_path', rel_to_binary) + if dest_path.is_absolute(): + loader_path = dest_path + else: + dest_path = root_dep_path.parent / dest_path + rel_to_binary = os.path.relpath(str(dest_path), str(root_dep_path.parent)) + loader_path = pathlib.PurePath('@loader_path', rel_to_binary) + + return (dest_path, loader_path) - return (dest_path, loader_path) def main(): - try: - d = dependency.Dependency(args.file) - except FileNotFoundError: - print('{} does not exist!'.format(str(args.file)), file=sys.stderr) - sys.exit(1) + try: + d = dependency.Dependency(args.file, pathlib.PosixPath(args.file).resolve(strict=True)) + except FileNotFoundError: + print('{} does not exist!'.format(str(args.file)), file=sys.stderr) + sys.exit(1) - loop = asyncio.get_event_loop() + loop = asyncio.get_event_loop() - loop.run_until_complete(collect(d)) + loop.run_until_complete(collect(d)) - dest_path, root_loader_path = get_dest_and_loader_path(d.path, args.destination) + dest_path, root_loader_path = get_dest_and_loader_path(d.path, args.destination) - prepatch_output(d) + prepatch_output(d) - if not args.dry_run: - try: - loop.run_until_complete(patch(d, dest_path, root_loader_path)) - except PatchError: # the error should have been already printed here - if not args.verbose: print('Run with -v for more information', file=sys.stderr) - sys.exit(1) + if not args.dry_run: + try: + loop.run_until_complete(patch(d, dest_path, root_loader_path)) + except PatchError: # the error should have been already printed here + if not args.verbose: print('Run with -v for more information', file=sys.stderr) + sys.exit(1) - n_deps = len(d.get_dependencies()) - print() - print('{} + {} dependenc{} successfully patched'.format(args.file.name, n_deps, 'y' if n_deps == 1 else 'ies')) + n_deps = len(d.get_dependencies()) + print() + print('{} + {} dependenc{} successfully patched'.format(args.file.name, n_deps, 'y' if n_deps == 1 else 'ies')) - loop.close() + loop.close() -parser = argparse.ArgumentParser(description='Copies non-system libraries used by your executable and patches them to work as a standalone bundle') + +parser = argparse.ArgumentParser( + description='Copies non-system libraries used by your executable and patches them to work as a standalone bundle') parser.add_argument('file', help='file to patch (the root, main binary)', type=pathlib.PurePath) -parser.add_argument('-v', '--verbose', help='displays more library information and output of install_name_tool', action='store_true') -parser.add_argument('-n', '--dry-run', help='just show the dependency tree but do not do any patching', action='store_true') -parser.add_argument('-d', '--destination', help='destination directory where the binaries will be placed and loaded', type=pathlib.Path, default='../libs') +parser.add_argument('-v', '--verbose', help='displays more library information and output of install_name_tool', + action='store_true') +parser.add_argument('-n', '--dry-run', help='just show the dependency tree but do not do any patching', + action='store_true') +parser.add_argument('-d', '--destination', help='destination directory where the binaries will be placed and loaded', + type=pathlib.Path, default='../libs') args = parser.parse_args() # Allow execution of this file directly (it's executed differently when installed) if __name__ == '__main__': main() -