From 6b6283525f7da9d65a173265eb6a0452ad3fa596 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 20 Oct 2025 14:15:39 -0500 Subject: [PATCH 01/47] reckless: report version without loading config --- tests/test_reckless.py | 12 ++++++++++++ tools/reckless | 3 +++ 2 files changed, 15 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index d294b79a50ef..3d2d66a7758a 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -170,6 +170,18 @@ def test_basic_help(): assert r.search_stdout("options:") or r.search_stdout("optional arguments:") +def test_version(): + '''Version should be reported without loading config and should advance + with lightningd''' + r = reckless(["-V", "-v", "--json"]) + assert r.returncode == 0 + import json + json_out = ''.join(r.stdout) + with open('.version', 'r') as f: + version = f.readlines()[0].strip() + assert json.loads(json_out)['result'][0] == version + + def test_contextual_help(node_factory): n = get_reckless_node(node_factory) for subcmd in ['install', 'uninstall', 'search', diff --git a/tools/reckless b/tools/reckless index c3f89bbd9810..76ced2b8c948 100755 --- a/tools/reckless +++ b/tools/reckless @@ -2082,6 +2082,9 @@ if __name__ == '__main__': 'signet', 'testnet', 'testnet4'] if args.version: report_version() + if log.capture: + log.reply_json() + sys.exit(0) elif args.cmd1 is None: parser.print_help(sys.stdout) sys.exit(1) From 03785202c65137cc45c696d0fe00959e26dfc79e Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 20 Oct 2025 14:39:18 -0500 Subject: [PATCH 02/47] reckless: redirect help alias output when --json option is used This doesn't change the argparse behavior with --help/-h, but it does correct the output in this one case where we must manually call it. --- tools/reckless | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/reckless b/tools/reckless index 76ced2b8c948..28e1bc1677c1 100755 --- a/tools/reckless +++ b/tools/reckless @@ -5,6 +5,7 @@ import argparse import copy import datetime from enum import Enum +import io import json import logging import os @@ -1124,9 +1125,16 @@ INSTALLERS = [pythonuv, pythonuvlegacy, python3venv, poetryvenv, def help_alias(targets: list): if len(targets) == 0: - parser.print_help(sys.stdout) + if log.capture: + help_output = io.StringIO() + parser.print_help(help_output) + log.add_result(help_output.getvalue()) + else: + parser.print_help(sys.stdout) else: log.info('try "reckless {} -h"'.format(' '.join(targets))) + if log.capture: + log.reply_json() sys.exit(1) @@ -2124,7 +2132,9 @@ if __name__ == '__main__': if 'targets' in args: # and len(args.targets) > 0: if args.func.__name__ == 'help_alias': - args.func(args.targets) + log.add_result(args.func(args.targets)) + if log.capture: + log.reply_json() sys.exit(0) # Catch a missing argument so that we can overload functions. if len(args.targets) == 0: From dca598297b17c1d9ba7bb96cb064ee3585420af4 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 21 Oct 2025 14:13:52 -0500 Subject: [PATCH 03/47] reckless: handle .git in source locations --- tools/reckless | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/reckless b/tools/reckless index 28e1bc1677c1..41605eaa43de 100755 --- a/tools/reckless +++ b/tools/reckless @@ -277,14 +277,14 @@ class InstInfo: pass # If unable to search deeper, resort to matching directory name elif recursion < 1: - if sub.name.lower() == self.name.lower(): + if sub.name.lower().removesuffix('.git') == self.name.lower(): # Partial success (can't check for entrypoint) self.name = sub.name return sub return None sub.populate() - if sub.name.lower() == self.name.lower(): + if sub.name.lower().removesuffix('.git') == self.name.lower(): # Directory matches the name we're trying to install, so check # for entrypoint and dependencies. for inst in INSTALLERS: @@ -302,7 +302,7 @@ class InstInfo: self.entry = found_entry.name self.deps = found_dep.name return sub - log.debug(f"missing dependency for {self}") + log.debug(f"{inst.name} installer: missing dependency for {self}") found_entry = None for file in sub.contents: if isinstance(file, SourceDir): @@ -404,7 +404,7 @@ class Source(Enum): trailing = Path(source.lower().partition('github.com/')[2]).parts if len(trailing) < 2: return None, None - return trailing[0], trailing[1] + return trailing[0], trailing[1].removesuffix('.git') class SourceDir(): @@ -451,7 +451,7 @@ class SourceDir(): for c in self.contents: if ftype and not isinstance(c, ftype): continue - if c.name.lower() == name.lower(): + if c.name.lower().removesuffix('.git') == name.lower(): return c return None @@ -627,7 +627,7 @@ def populate_github_repo(url: str) -> list: while '' in repo: repo.remove('') repo_name = None - parsed_url = urlparse(url) + parsed_url = urlparse(url.removesuffix('.git')) if 'github.com' not in parsed_url.netloc: return None if len(parsed_url.path.split('/')) < 2: @@ -1212,7 +1212,7 @@ def _git_update(github_source: InstInfo, local_copy: PosixPath): if git.returncode != 0: return False default_branch = git.stdout.splitlines()[0] - if default_branch != 'origin/master': + if default_branch not in ['origin/master', 'origin/main']: log.debug(f'UNUSUAL: fetched default branch {default_branch} for ' f'{github_source.source_loc}') @@ -1589,7 +1589,7 @@ def search(plugin_name: str) -> Union[InstInfo, None]: for src in RECKLESS_SOURCES: # Search repos named after the plugin before collections if Source.get_type(src) == Source.GITHUB_REPO: - if src.split('/')[-1].lower() == plugin_name.lower(): + if src.split('/')[-1].lower().removesuffix('.git') == plugin_name.lower(): ordered_sources.remove(src) ordered_sources.insert(0, src) # Check locally before reaching out to remote repositories From 9dd8f033af2c03422dfc20f94d8c911980086a94 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 23 Oct 2025 12:24:48 -0500 Subject: [PATCH 04/47] reckless: cleanup failed installation attempts This is needed when installation is managed by an application that may not have access to the filesystem to clean up manually. --- tests/test_reckless.py | 12 ++++++++++++ tools/reckless | 29 +++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 3d2d66a7758a..6dd479d45bbf 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -276,6 +276,18 @@ def test_install(node_factory): assert os.path.exists(plugin_path) +def test_install_cleanup(node_factory): + """test failed installation and post install cleanup""" + n = get_reckless_node(node_factory) + n.start() + r = reckless([f"--network={NETWORK}", "-v", "install", "testplugfail"], dir=n.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('testplugfail failed to start') + r.check_stderr() + plugin_path = Path(n.lightning_dir) / 'reckless/testplugfail' + assert not os.path.exists(plugin_path) + + @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") def test_poetry_install(node_factory): """test search, git clone, and installation to folder.""" diff --git a/tools/reckless b/tools/reckless index 41605eaa43de..8c69bda97795 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1484,10 +1484,22 @@ def _enable_installed(installed: InstInfo, plugin_name: str) -> Union[str, None] if enable(installed.name): return f"{installed.source_loc}" - log.error(('dynamic activation failed: ' - f'{installed.name} not found in reckless directory')) + log.error('dynamic activation failed') return None + +def cleanup_plugin_installation(plugin_name): + """Remove traces of an installation attempt.""" + inst_path = Path(RECKLESS_CONFIG.reckless_dir) / plugin_name + if not inst_path.exists(): + log.warning(f'asked to clean up {inst_path}, but nothing is present.') + return + + log.info(f'Cleaning up partial installation of {plugin_name} at {inst_path}') + shutil.rmtree(inst_path) + return + + def install(plugin_name: str) -> Union[str, None]: """Downloads plugin from source repos, installs and activates plugin. Returns the location of the installed plugin or "None" in the case of @@ -1504,7 +1516,7 @@ def install(plugin_name: str) -> Union[str, None]: direct_location, name = location_from_name(name) src = None if direct_location: - logging.debug(f"install of {name} requested from {direct_location}") + log.debug(f"install of {name} requested from {direct_location}") src = InstInfo(name, direct_location, name) # Treating a local git repo as a directory allows testing # uncommitted changes. @@ -1529,8 +1541,17 @@ def install(plugin_name: str) -> Union[str, None]: except FileExistsError as err: log.error(f'File exists: {err.filename}') return None - return _enable_installed(installed, plugin_name) + except InstallationFailure as err: + cleanup_plugin_installation(plugin_name) + if log.capture: + log.warning(err) + return None + raise err + result = _enable_installed(installed, plugin_name) + if not result: + cleanup_plugin_installation(plugin_name) + return result def uninstall(plugin_name: str) -> str: From abc265e0c64941440b260b9cb2cb0b8054def1fa Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 23 Oct 2025 16:40:14 -0500 Subject: [PATCH 05/47] reckless: add listinstalled command to list reckless managed plugins 'Reckless listinstalled' will now list all plugins installed and managed by reckless. Changelog-Added: reckless: `listinstalled` command lists plugins installed by reckless. --- tools/reckless | 92 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index 8c69bda97795..403f22962879 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1958,6 +1958,93 @@ def update_plugins(plugin_name: str): return update_results +MD_FORMAT = {'installation date': "None", + 'installation time': "None", + 'original source': "None", + 'requested commit': "None", + 'installed commit': "None", + } + + +def extract_metadata(plugin_name: str) -> dict: + metadata_file = Path(RECKLESS_CONFIG.reckless_dir) / plugin_name / '.metadata' + if not metadata_file.exists(): + return None + + with open(metadata_file, 'r') as md: + lines = md.readlines() + metadata = MD_FORMAT.copy() + current_key = None + + for line in lines: + if line.strip() in metadata: + current_key = line.strip() + continue + + if current_key: + metadata.update({current_key: line.strip()}) + current_key = None + + return metadata + + +def listinstalled(): + """list all plugins currently managed by reckless""" + dir_contents = os.listdir(RECKLESS_CONFIG.reckless_dir) + plugins = {} + for plugin in dir_contents: + if (Path(RECKLESS_CONFIG.reckless_dir) / plugin).is_dir(): + # skip hidden dirs such as reckless' .remote_sources + if plugin[0] == '.': + continue + plugins.update({plugin: None}) + + # Format output in a simple table + name_len = 0 + inst_len = 0 + for plugin in plugins.keys(): + md = extract_metadata(plugin) + name_len = max(name_len, len(plugin) + 1) + if md: + inst_len = max(inst_len, len(md['installed commit']) + 1) + else: + inst_len = max(inst_len, 5) + for plugin in plugins.keys(): + md = extract_metadata(plugin) + # Older installed plugins may be missing a .metadata file + if not md: + md = MD_FORMAT.copy() + try: + installed = InferInstall(plugin) + except: + log.debug(f'no plugin detected in directory {plugin}') + continue + + status = "unmanaged" + for line in RECKLESS_CONFIG.content: + if installed.entry in line.strip() : + if line.strip()[:7] == 'plugin=': + status = "enabled" + elif line.strip()[:15] == 'disable-plugin=': + status = "disabled" + else: + print(f'cant handle {line}') + log.info(f"{plugin:<{name_len}} {md['installed commit']:<{inst_len}} " + f"{md['installation date']:<11} {status}") + # This doesn't originate from the metadata, but we want to provide enabled status for json output + md['enabled'] = status == "enabled" + md['entrypoint'] = installed.entry + # Format for json output + for key in md: + if md[key] == 'None': + md[key] = None + if key == 'installation time' and md[key]: + md[key] = int(md[key]) + plugins[plugin] = {k.replace(' ', '_'): v for k, v in md.items()} + + return plugins + + def report_version() -> str: """return reckless version""" log.info(__VERSION__) @@ -2057,6 +2144,9 @@ if __name__ == '__main__': update.add_argument('targets', type=str, nargs='*') update.set_defaults(func=update_plugins) + list_cmd = cmd1.add_parser('listinstalled', help='list reckless-installed plugins') + list_cmd.set_defaults(func=listinstalled) + help_cmd = cmd1.add_parser('help', help='for contextual help, use ' '"reckless -h"') help_cmd.add_argument('targets', type=str, nargs='*') @@ -2067,7 +2157,7 @@ if __name__ == '__main__': all_parsers = [parser, install_cmd, uninstall_cmd, search_cmd, enable_cmd, disable_cmd, list_parse, source_add, source_rem, help_cmd, - update] + update, list_cmd] for p in all_parsers: # This default depends on the .lightning directory p.add_argument('-d', '--reckless-dir', action=StoreIdempotent, From bd38c6d95e96ff6b41c11440d44b3839ea8b8e74 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 28 Oct 2025 12:26:25 -0500 Subject: [PATCH 06/47] reckless: helper function for accessing local clone of remote repo --- tools/reckless | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tools/reckless b/tools/reckless index 403f22962879..d4a04e04a269 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1138,6 +1138,16 @@ def help_alias(targets: list): sys.exit(1) +def _get_local_clone(source: str) -> Union[Path, None]: + """Returns the path of a local repository clone of a github source. If one + already exists, prefer searching that to accessing the github API.""" + user, repo = Source.get_github_user_repo(source) + local_clone_location = RECKLESS_DIR / '.remote_sources' / user / repo + if local_clone_location.exists(): + return local_clone_location + return None + + def _source_search(name: str, src: str) -> Union[InstInfo, None]: """Identify source type, retrieve contents, and populate InstInfo if the relevant contents are found.""" @@ -1147,18 +1157,11 @@ def _source_search(name: str, src: str) -> Union[InstInfo, None]: # If a local clone of a github source already exists, prefer searching # that instead of accessing the github API. if source.srctype == Source.GITHUB_REPO: - # Do we have a local copy already? Use that. - user, repo = Source.get_github_user_repo(src) - assert user - assert repo - local_clone_location = RECKLESS_DIR / '.remote_sources' / user / repo - if local_clone_location.exists(): - # Make sure it's the correct remote source and fetch any updates. - if _git_update(source, local_clone_location): - log.debug(f"Using local clone of {src}: " - f"{local_clone_location}") - source.source_loc = str(local_clone_location) - source.srctype = Source.GIT_LOCAL_CLONE + local_clone = _get_local_clone(source) + if local_clone and _git_update(source, local_clone): + log.debug(f"Using local clone of {src}: {local_clone}") + source.source_loc = str(local_clone) + source.srctype = Source.GIT_LOCAL_CLONE if source.get_inst_details(): return source From 007345bd1023a02c0eb9be98ab80a0d2e0af2b9b Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 30 Oct 2025 16:19:40 -0500 Subject: [PATCH 07/47] reckless: only fetch cloned repositorys once Keep state on the clone and subdirectories so that subsequent access doesn't try fetching again. --- tools/reckless | 398 ++++++++++++++++++++++++++++--------------------- 1 file changed, 227 insertions(+), 171 deletions(-) diff --git a/tools/reckless b/tools/reckless index d4a04e04a269..81c5392eff7c 100755 --- a/tools/reckless +++ b/tools/reckless @@ -206,146 +206,6 @@ class Installer: return copy.deepcopy(self) -class InstInfo: - def __init__(self, name: str, location: str, git_url: str): - self.name = name - self.source_loc = str(location) # Used for 'git clone' - self.git_url: str = git_url # API access for github repos - self.srctype: Source = Source.get_type(location) - self.entry: SourceFile = None # relative to source_loc or subdir - self.deps: str = None - self.subdir: str = None - self.commit: str = None - - def __repr__(self): - return (f'InstInfo({self.name}, {self.source_loc}, {self.git_url}, ' - f'{self.entry}, {self.deps}, {self.subdir})') - - def get_repo_commit(self) -> Union[str, None]: - """The latest commit from a remote repo or the HEAD of a local repo.""" - if self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: - git = run(['git', 'rev-parse', 'HEAD'], cwd=str(self.source_loc), - stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=10) - if git.returncode != 0: - return None - return git.stdout.splitlines()[0] - - if self.srctype == Source.GITHUB_REPO: - parsed_url = urlparse(self.source_loc) - if 'github.com' not in parsed_url.netloc: - return None - if len(parsed_url.path.split('/')) < 2: - return None - start = 1 - # Maybe we were passed an api.github.com/repo/ url - if 'api' in parsed_url.netloc: - start += 1 - repo_user = parsed_url.path.split('/')[start] - repo_name = parsed_url.path.split('/')[start + 1] - api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/commits?ref=HEAD' - r = urlopen(api_url, timeout=5) - if r.status != 200: - return None - try: - return json.loads(r.read().decode())['0']['sha'] - except: - return None - - def get_inst_details(self) -> bool: - """Search the source_loc for plugin install details. - This may be necessary if a contents api is unavailable. - Extracts entrypoint and dependencies if searchable, otherwise - matches a directory to the plugin name and stops.""" - if self.srctype == Source.DIRECTORY: - assert Path(self.source_loc).exists() - assert os.path.isdir(self.source_loc) - target = SourceDir(self.source_loc, srctype=self.srctype) - # Set recursion for how many directories deep we should search - depth = 0 - if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, - Source.GIT_LOCAL_CLONE]: - depth = 5 - elif self.srctype == Source.GITHUB_REPO: - depth = 1 - - def search_dir(self, sub: SourceDir, subdir: bool, - recursion: int) -> Union[SourceDir, None]: - assert isinstance(recursion, int) - # carveout for archived plugins in lightningd/plugins. Other repos - # are only searched by API at the top level. - if recursion == 0 and 'archive' in sub.name.lower(): - pass - # If unable to search deeper, resort to matching directory name - elif recursion < 1: - if sub.name.lower().removesuffix('.git') == self.name.lower(): - # Partial success (can't check for entrypoint) - self.name = sub.name - return sub - return None - sub.populate() - - if sub.name.lower().removesuffix('.git') == self.name.lower(): - # Directory matches the name we're trying to install, so check - # for entrypoint and dependencies. - for inst in INSTALLERS: - for g in inst.get_entrypoints(self.name): - found_entry = sub.find(g, ftype=SourceFile) - if found_entry: - break - # FIXME: handle a list of dependencies - found_dep = sub.find(inst.dependency_file, - ftype=SourceFile) - if found_entry: - # Success! - if found_dep: - self.name = sub.name - self.entry = found_entry.name - self.deps = found_dep.name - return sub - log.debug(f"{inst.name} installer: missing dependency for {self}") - found_entry = None - for file in sub.contents: - if isinstance(file, SourceDir): - assert file.relative - success = search_dir(self, file, True, recursion - 1) - if success: - return success - return None - - try: - result = search_dir(self, target, False, depth) - # Using the rest API of github.com may result in a - # "Error 403: rate limit exceeded" or other access issues. - # Fall back to cloning and searching the local copy instead. - except HTTPError: - result = None - if self.srctype == Source.GITHUB_REPO: - # clone source to reckless dir - target = copy_remote_git_source(self) - if not target: - log.warning(f"could not clone github source {self}") - return False - log.debug(f"falling back to cloning remote repo {self}") - # Update to reflect use of a local clone - self.source_loc = str(target.location) - self.srctype = target.srctype - result = search_dir(self, target, False, 5) - - if not result: - return False - - if result: - if result != target: - if result.relative: - self.subdir = result.relative - else: - # populate() should always assign a relative path - # if not in the top-level source directory - assert self.subdir == result.name - return True - return False - - def create_dir(directory: PosixPath) -> bool: try: Path(directory).mkdir(parents=False, exist_ok=True) @@ -407,10 +267,41 @@ class Source(Enum): return trailing[0], trailing[1].removesuffix('.git') +class SubmoduleSource: + """Allows us to only fetch submodules once.""" + def __init__(self, location: str): + self.location = str(location) + self.local_clone = None + self.clone_fetched = False + + def __repr__(self): + return f'' + + +class LoadedSource: + """Allows loading all sources only once per call of reckless. Initialized + with a single line of the reckless .sources file. Keeping state also allows + minimizing API calls and refetching repositories.""" + def __init__(self, source: str): + self.original_source = source + self.type = Source.get_type(source) + self.content = SourceDir(source, self.type) + self.local_clone = None + self.local_clone_fetched = False + if self.type == Source.GITHUB_REPO: + local = _get_local_clone(source) + if local: + self.local_clone = SourceDir(local, Source.GIT_LOCAL_CLONE) + self.local_clone.parent_source = self + + def __repr__(self): + return f'' + + class SourceDir(): """Structure to search source contents.""" def __init__(self, location: str, srctype: Source = None, name: str = None, - relative: str = None): + relative: str = None, parent_source: LoadedSource = None): self.location = str(location) if name: self.name = name @@ -420,6 +311,7 @@ class SourceDir(): self.srctype = srctype self.prepopulated = False self.relative = relative # location relative to source + self.parent_source = parent_source def populate(self): """populates contents of the directory at least one level""" @@ -430,7 +322,7 @@ class SourceDir(): if self.srctype == Source.DIRECTORY: self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: - self.contents = populate_local_repo(self.location) + self.contents = populate_local_repo(self.location, parent_source=self.parent_source) elif self.srctype == Source.GITHUB_REPO: self.contents = populate_github_repo(self.location) else: @@ -484,6 +376,153 @@ class SourceFile(): return False +class InstInfo: + def __init__(self, name: str, location: str, git_url: str, source_dir: SourceDir=None): + self.name = name + self.source_loc = str(location) # Used for 'git clone' + self.source_dir = source_dir # Use this insead of source_loc to only fetch once. + self.git_url: str = git_url # API access for github repos + self.srctype: Source = Source.get_type(location) + self.entry: SourceFile = None # relative to source_loc or subdir + self.deps: str = None + self.subdir: str = None + self.commit: str = None + + def __repr__(self): + return (f'InstInfo({self.name}, {self.source_loc}, {self.git_url}, ' + f'{self.entry}, {self.deps}, {self.subdir})') + + def get_repo_commit(self) -> Union[str, None]: + """The latest commit from a remote repo or the HEAD of a local repo.""" + if self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: + git = run(['git', 'rev-parse', 'HEAD'], cwd=str(self.source_loc), + stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=10) + if git.returncode != 0: + return None + return git.stdout.splitlines()[0] + + if self.srctype == Source.GITHUB_REPO: + parsed_url = urlparse(self.source_loc) + if 'github.com' not in parsed_url.netloc: + return None + if len(parsed_url.path.split('/')) < 2: + return None + start = 1 + # Maybe we were passed an api.github.com/repo/ url + if 'api' in parsed_url.netloc: + start += 1 + repo_user = parsed_url.path.split('/')[start] + repo_name = parsed_url.path.split('/')[start + 1] + api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/commits?ref=HEAD' + r = urlopen(api_url, timeout=5) + if r.status != 200: + return None + try: + return json.loads(r.read().decode())['0']['sha'] + except: + return None + + def get_inst_details(self, permissive: bool=False) -> bool: + """Search the source_loc for plugin install details. + This may be necessary if a contents api is unavailable. + Extracts entrypoint and dependencies if searchable, otherwise + matches a directory to the plugin name and stops. + permissive: allows search to sometimes match directory name only for + faster searching of remote repositorys.""" + if self.srctype == Source.DIRECTORY: + assert Path(self.source_loc).exists() + assert os.path.isdir(self.source_loc) + target = self.source_dir + if not target: + target = SourceDir(self.source_loc, srctype=self.srctype) + # Set recursion for how many directories deep we should search + depth = 0 + if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, + Source.GIT_LOCAL_CLONE]: + depth = 5 + elif self.srctype == Source.GITHUB_REPO: + depth = 1 + + def search_dir(self, sub: SourceDir, subdir: bool, + recursion: int) -> Union[SourceDir, None]: + assert isinstance(recursion, int) + # carveout for archived plugins in lightningd/plugins. Other repos + # are only searched by API at the top level. + if recursion == 0 and 'archive' in sub.name.lower(): + pass + # If unable to search deeper, resort to matching directory name + elif recursion < 1 and permissive: + if sub.name.lower().removesuffix('.git') == self.name.lower(): + # Partial success (can't check for entrypoint) + self.name = sub.name + return sub + return None + if not sub.contents and not sub.prepopulated: + sub.populate() + + if sub.name.lower().removesuffix('.git') == self.name.lower(): + # Directory matches the name we're trying to install, so check + # for entrypoint and dependencies. + for inst in INSTALLERS: + for g in inst.get_entrypoints(self.name): + found_entry = sub.find(g, ftype=SourceFile) + if found_entry: + break + # FIXME: handle a list of dependencies + found_dep = sub.find(inst.dependency_file, + ftype=SourceFile) + if found_entry: + # Success! + if found_dep: + self.name = sub.name + self.entry = found_entry.name + self.deps = found_dep.name + return sub + if permissive is True: + log.debug(f"{inst.name} installer: missing dependency for {self}") + found_entry = None + for file in sub.contents: + if isinstance(file, SourceDir): + assert file.relative + success = search_dir(self, file, True, recursion - 1) + if success: + return success + return None + + try: + result = search_dir(self, target, False, depth) + # Using the rest API of github.com may result in a + # "Error 403: rate limit exceeded" or other access issues. + # Fall back to cloning and searching the local copy instead. + except HTTPError: + result = None + if self.srctype == Source.GITHUB_REPO: + # clone source to reckless dir + target = copy_remote_git_source(self) + if not target: + log.warning(f"could not clone github source {self}") + return False + log.debug(f"falling back to cloning remote repo {self}") + # Update to reflect use of a local clone + self.source_loc = str(target.location) + self.srctype = target.srctype + result = search_dir(self, target, False, 5) + + if not result: + return False + + if result: + if result != target: + if result.relative: + self.subdir = result.relative + else: + # populate() should always assign a relative path + # if not in the top-level source directory + assert self.subdir == result.name + return True + return False + + def populate_local_dir(path: str) -> list: assert Path(os.path.realpath(path)).exists() contents = [] @@ -497,7 +536,7 @@ def populate_local_dir(path: str) -> list: return contents -def populate_local_repo(path: str, parent=None) -> list: +def populate_local_repo(path: str, parent=None, parent_source=None) -> list: assert Path(os.path.realpath(path)).exists() if parent is None: basedir = SourceDir('base') @@ -571,10 +610,13 @@ def populate_local_repo(path: str, parent=None) -> list: relative_path = str(Path(basedir.relative) / filepath) assert relative_path submodule_dir = SourceDir(filepath, srctype=Source.LOCAL_REPO, - relative=relative_path) - populate_local_repo(Path(path) / filepath, parent=submodule_dir) + relative=relative_path, + parent_source=parent_source) + populate_local_repo(Path(path) / filepath, parent=submodule_dir, + parent_source=parent_source) submodule_dir.prepopulated = True basedir.contents.append(submodule_dir) + # parent_source.submodules.append(submodule_dir) else: populate_source_path(basedir, Path(filepath)) return basedir.contents @@ -681,7 +723,8 @@ def copy_remote_git_source(github_source: InstInfo): local_path = local_path / repo if local_path.exists(): # Fetch the latest - assert _git_update(github_source, local_path) + # FIXME: pass LoadedSource and check fetch status + assert _git_update(github_source.source_loc, local_path) else: _git_clone(github_source, local_path) return SourceDir(local_path, srctype=Source.GIT_LOCAL_CLONE) @@ -1148,22 +1191,26 @@ def _get_local_clone(source: str) -> Union[Path, None]: return None -def _source_search(name: str, src: str) -> Union[InstInfo, None]: +def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: """Identify source type, retrieve contents, and populate InstInfo if the relevant contents are found.""" - root_dir = SourceDir(src) + root_dir = src.content source = InstInfo(name, root_dir.location, None) # If a local clone of a github source already exists, prefer searching # that instead of accessing the github API. - if source.srctype == Source.GITHUB_REPO: - local_clone = _get_local_clone(source) - if local_clone and _git_update(source, local_clone): - log.debug(f"Using local clone of {src}: {local_clone}") - source.source_loc = str(local_clone) + if src.type == Source.GITHUB_REPO: + if src.local_clone: + if not src.local_clone_fetched: + # FIXME: Pass the LoadedSource here? + if _git_update(src.original_source, src.local_clone.location): + src.local_clone_fetched = True + log.debug(f'fetching local clone of {src.original_source}') + log.debug(f"Using local clone of {src}: {src.local_clone.location}") + source.source_loc = str(src.local_clone.location) source.srctype = Source.GIT_LOCAL_CLONE - if source.get_inst_details(): + if source.get_inst_details(permissive=True): return source return None @@ -1190,9 +1237,9 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str]) -> bool: return True -def _git_update(github_source: InstInfo, local_copy: PosixPath): +def _git_update(github_source: str, local_copy: PosixPath): # Ensure this is the correct source - git = run(['git', 'remote', 'set-url', 'origin', github_source.source_loc], + git = run(['git', 'remote', 'set-url', 'origin', github_source], cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) assert git.returncode == 0 @@ -1217,7 +1264,7 @@ def _git_update(github_source: InstInfo, local_copy: PosixPath): default_branch = git.stdout.splitlines()[0] if default_branch not in ['origin/master', 'origin/main']: log.debug(f'UNUSUAL: fetched default branch {default_branch} for ' - f'{github_source.source_loc}') + f'{github_source}') # Checkout default branch git = run(['git', 'checkout', default_branch], @@ -1337,7 +1384,11 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # FIXME: Validate path was cloned successfully. # Depending on how we accessed the original source, there may be install # details missing. Searching the cloned repo makes sure we have it. - cloned_src = _source_search(src.name, str(clone_path)) + clone = LoadedSource(plugin_path) + clone.content.populate() + # Make sure we don't try to fetch again! + assert clone.type in [Source.DIRECTORY, Source.LOCAL_REPO] + cloned_src = _source_search(src.name, clone) log.debug(f'cloned_src: {cloned_src}') if not cloned_src: log.warning('failed to find plugin after cloning repo.') @@ -1525,7 +1576,7 @@ def install(plugin_name: str) -> Union[str, None]: # uncommitted changes. if src and src.srctype == Source.LOCAL_REPO: src.srctype = Source.DIRECTORY - if not src.get_inst_details(): + if not src.get_inst_details(permissive=True): src = None if not direct_location or not src: log.debug(f"Searching for {name}") @@ -1596,7 +1647,7 @@ def _get_all_plugins_from_source(src: str) -> list: return plugins plugins.append((root.name, src)) - + for item in root.contents: if isinstance(item, SourceDir): # Skip archive directories @@ -1612,20 +1663,20 @@ def search(plugin_name: str) -> Union[InstInfo, None]: for src in RECKLESS_SOURCES: # Search repos named after the plugin before collections - if Source.get_type(src) == Source.GITHUB_REPO: - if src.split('/')[-1].lower().removesuffix('.git') == plugin_name.lower(): + if src.type == Source.GITHUB_REPO: + if src.original_source.split('/')[-1].lower().removesuffix('.git') == plugin_name.lower(): ordered_sources.remove(src) ordered_sources.insert(0, src) # Check locally before reaching out to remote repositories for src in RECKLESS_SOURCES: - if Source.get_type(src) in [Source.DIRECTORY, Source.LOCAL_REPO]: + if src.type in [Source.DIRECTORY, Source.LOCAL_REPO]: ordered_sources.remove(src) ordered_sources.insert(0, src) # First, collect all partial matches to display to user partial_matches = [] for source in ordered_sources: - for plugin_name_found, src_url in _get_all_plugins_from_source(source): + for plugin_name_found, src_url in _get_all_plugins_from_source(source.original_source): if plugin_name.lower() in plugin_name_found.lower(): partial_matches.append((plugin_name_found, src_url)) @@ -1638,12 +1689,11 @@ def search(plugin_name: str) -> Union[InstInfo, None]: # Now try exact match for installation purposes exact_match = None for source in ordered_sources: - srctype = Source.get_type(source) - if srctype == Source.UNKNOWN: - log.debug(f'cannot search {srctype} {source}') + if source.type == Source.UNKNOWN: + log.debug(f'cannot search {source.type} {source.original_source}') continue - if srctype in [Source.DIRECTORY, Source.LOCAL_REPO, - Source.GITHUB_REPO, Source.OTHER_URL]: + if source.type in [Source.DIRECTORY, Source.LOCAL_REPO, + Source.GITHUB_REPO, Source.OTHER_URL]: found = _source_search(plugin_name, source) if found: log.debug(f"{found}, {found.srctype}") @@ -1835,8 +1885,14 @@ def load_sources() -> list: log.debug('Warning: Reckless requires write access') Config(path=str(sources_file), default_text='https://github.com/lightningd/plugins') - return ['https://github.com/lightningd/plugins'] - return sources_from_file() + sources = ['https://github.com/lightningd/plugins'] + else: + sources = sources_from_file() + + all_sources = [] + for src in sources: + all_sources.append(LoadedSource(src)) + return all_sources def add_source(src: str): From f1d8d565c6f4b1a11e574ab16bc002f1db26e55d Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 13 Nov 2025 14:16:00 -0600 Subject: [PATCH 08/47] recklessrpc: allow single term commands Allows listconfig, listinstalled, and listavailable to be called via rpc. Also allow processing non-array result in listconfig output. --- contrib/msggen/msggen/schema.json | 19 ++++++++---- doc/reckless.7.md | 5 ++-- doc/schemas/reckless.json | 19 ++++++++---- plugins/recklessrpc.c | 49 +++++++++++++++++++++++-------- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json index ca068e2fb5e7..8e5862952203 100644 --- a/contrib/msggen/msggen/schema.json +++ b/contrib/msggen/msggen/schema.json @@ -30038,12 +30038,19 @@ "additionalProperties": false, "properties": { "result": { - "type": "array", - "items": { - "type": "string" - }, - "description": [ - "Output of the requested reckless command." + "oneOf": [ + { + "type": "array", + "description": [ + "Output of the requested reckless command." + ] + }, + { + "type": "object", + "description": [ + "Output of the requested reckless command." + ] + } ] }, "log": { diff --git a/doc/reckless.7.md b/doc/reckless.7.md index 49918023d56b..acf5f4cd40d9 100644 --- a/doc/reckless.7.md +++ b/doc/reckless.7.md @@ -28,8 +28,9 @@ RETURN VALUE On success, an object is returned, containing: -- **result** (array of strings): Output of the requested reckless command.: - - (string, optional) +- **result** (one of): + - (array): Output of the requested reckless command. + - (object): Output of the requested reckless command.: - **log** (array of strings): Verbose log entries of the requested reckless command.: - (string, optional) diff --git a/doc/schemas/reckless.json b/doc/schemas/reckless.json index 294fdf8f9690..372eb772dd3e 100644 --- a/doc/schemas/reckless.json +++ b/doc/schemas/reckless.json @@ -67,12 +67,19 @@ "additionalProperties": false, "properties": { "result": { - "type": "array", - "items": { - "type": "string" - }, - "description": [ - "Output of the requested reckless command." + "oneOf": [ + { + "type": "array", + "description": [ + "Output of the requested reckless command." + ] + }, + { + "type": "object", + "description": [ + "Output of the requested reckless command." + ] + } ] }, "log": { diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 510a3e55ddcd..794f985c94b3 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -68,7 +68,7 @@ static struct command_result *reckless_result(struct io_conn *conn, reckless->process_failed); return command_finished(reckless->cmd, response); } - const jsmntok_t *results, *result, *logs, *log; + const jsmntok_t *results, *result, *logs, *log, *conf; size_t i; jsmn_parser parser; jsmntok_t *toks; @@ -97,15 +97,26 @@ static struct command_result *reckless_result(struct io_conn *conn, } response = jsonrpc_stream_success(reckless->cmd); - json_array_start(response, "result"); results = json_get_member(reckless->stdoutbuf, toks, "result"); - json_for_each_arr(i, result, results) { - json_add_string(response, - NULL, - json_strdup(reckless, reckless->stdoutbuf, - result)); + conf = json_get_member(reckless->stdoutbuf, results, "requested_lightning_conf"); + if (conf) { + plugin_log(plugin, LOG_DBG, "dealing with listconfigs output"); + json_object_start(response, "result"); + json_for_each_obj(i, result, results) { + json_add_tok(response, json_strdup(tmpctx, reckless->stdoutbuf, result), result+1, reckless->stdoutbuf); + } + json_object_end(response); + + } else { + json_array_start(response, "result"); + json_for_each_arr(i, result, results) { + json_add_string(response, + NULL, + json_strdup(reckless, reckless->stdoutbuf, + result)); + } + json_array_end(response); } - json_array_end(response); json_array_start(response, "log"); logs = json_get_member(reckless->stdoutbuf, toks, "log"); json_for_each_arr(i, log, logs) { @@ -151,6 +162,7 @@ static void reckless_conn_finish(struct io_conn *conn, reckless_result(conn, reckless); /* Don't try to process json if python raised an error. */ } else { + plugin_log(plugin, LOG_DBG, "%s", reckless->stderrbuf); plugin_log(plugin, LOG_DBG, "Reckless process has crashed (%i).", WEXITSTATUS(status)); @@ -211,13 +223,25 @@ static struct io_plan *stderr_conn_init(struct io_conn *conn, return stderr_read_more(conn, reckless); } +static bool is_single_arg_cmd(const char *command) { + if (strcmp(command, "listconfig")) + return true; + if (strcmp(command, "listavailable")) + return true; + if (strcmp(command, "listinstalled")) + return true; + return false; +} + static struct command_result *reckless_call(struct command *cmd, const char *subcommand, const char *target, const char *target2) { - if (!subcommand || !target) - return command_fail(cmd, PLUGIN_ERROR, "invalid reckless call"); + if (!is_single_arg_cmd(subcommand)) { + if (!subcommand || !target) + return command_fail(cmd, PLUGIN_ERROR, "invalid reckless call"); + } char **my_call; my_call = tal_arrz(tmpctx, char *, 0); tal_arr_expand(&my_call, "reckless"); @@ -232,7 +256,8 @@ static struct command_result *reckless_call(struct command *cmd, tal_arr_expand(&my_call, lconfig.config); } tal_arr_expand(&my_call, (char *) subcommand); - tal_arr_expand(&my_call, (char *) target); + if (target) + tal_arr_expand(&my_call, (char *) target); if (target2) tal_arr_expand(&my_call, (char *) target2); tal_arr_expand(&my_call, NULL); @@ -273,7 +298,7 @@ static struct command_result *json_reckless(struct command *cmd, /* Allow check command to evaluate. */ if (!param(cmd, buf, params, p_req("command", param_string, &command), - p_req("target/subcommand", param_string, &target), + p_opt("target/subcommand", param_string, &target), p_opt("target", param_string, &target2), NULL)) return command_param_failed(); From 1845ba4c00cb2c8dd8022729889470ceaa4db12a Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 28 Oct 2025 12:44:03 -0500 Subject: [PATCH 09/47] reckless: Add `listavailable` command to list plugins available to install reckless listavailable sorts through the available sources to find plugins for which we have installers. Changelog-Added: reckless gained the 'listavailable' command to list available plugins from reckless' sources. --- tools/reckless | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index 81c5392eff7c..3bddd20de521 100755 --- a/tools/reckless +++ b/tools/reckless @@ -2104,6 +2104,54 @@ def listinstalled(): return plugins +def find_plugin_candidates(source: SourceDir, depth=2) -> list: + """Filter through a source and return any candidates that appear to be + installable plugins with the registered installers.""" + candidates = [] + assert isinstance(source, SourceDir) + if not source.contents and not source.prepopulated: + source.populate() + + guess = InstInfo(source.name, source.location, None, source_dir=source) + if guess.get_inst_details(): + candidates.append(source.name) + if depth <= 1: + return candidates + + for c in source.contents: + if not isinstance(c, SourceDir): + continue + candidates.extend(find_plugin_candidates(c, depth=depth-1)) + + return candidates + + +def available_plugins() -> list: + """List installable plugins available from the sources list""" + candidates = [] + # FIXME: update for LoadedSource object + for source in RECKLESS_SOURCES: + if source.type == Source.UNKNOWN: + log.debug(f'confusing source: {source.type}') + continue + # It takes too many API calls to query for installable plugins accurately. + if source.type == Source.GITHUB_REPO and not source.local_clone: + # FIXME: ignoring non-cloned repos for now. + log.debug(f'unable to search {source.original_source} without a local clone of the repository.') + continue + + if source.local_clone: + candidates.extend(find_plugin_candidates(source.local_clone)) + else: + candidates.extend(find_plugin_candidates(source.content)) + + # Order and deduplicate results + candidates = list(set(candidates)) + candidates.sort() + log.info(' '.join(candidates)) + return candidates + + def report_version() -> str: """return reckless version""" log.info(__VERSION__) @@ -2177,6 +2225,10 @@ if __name__ == '__main__': search_cmd.add_argument('targets', type=str, nargs='*') search_cmd.set_defaults(func=search) + available_cmd = cmd1.add_parser('listavailable', help='list plugins available ' + 'from the sources list') + available_cmd.set_defaults(func=available_plugins) + enable_cmd = cmd1.add_parser('enable', help='dynamically enable a plugin ' 'and update config') enable_cmd.add_argument('targets', type=str, nargs='*') @@ -2216,7 +2268,7 @@ if __name__ == '__main__': all_parsers = [parser, install_cmd, uninstall_cmd, search_cmd, enable_cmd, disable_cmd, list_parse, source_add, source_rem, help_cmd, - update, list_cmd] + update, list_cmd, available_cmd] for p in all_parsers: # This default depends on the .lightning directory p.add_argument('-d', '--reckless-dir', action=StoreIdempotent, From a2892a3c8112d42ddd92bc3b049c89ba2a14f773 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 30 Oct 2025 18:15:21 -0500 Subject: [PATCH 10/47] reckless: handle lack of cloned source in listavailable cmd --- tools/reckless | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tools/reckless b/tools/reckless index 3bddd20de521..941002e42d6f 100755 --- a/tools/reckless +++ b/tools/reckless @@ -707,7 +707,7 @@ def populate_github_repo(url: str) -> list: return contents -def copy_remote_git_source(github_source: InstInfo): +def copy_remote_git_source(github_source: InstInfo, verbose: bool=True): """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: @@ -726,7 +726,7 @@ def copy_remote_git_source(github_source: InstInfo): # FIXME: pass LoadedSource and check fetch status assert _git_update(github_source.source_loc, local_path) else: - _git_clone(github_source, local_path) + _git_clone(github_source, local_path, verbose) return SourceDir(local_path, srctype=Source.GIT_LOCAL_CLONE) @@ -1215,8 +1215,11 @@ def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: return None -def _git_clone(src: InstInfo, dest: Union[PosixPath, str]) -> bool: - log.info(f'cloning {src.srctype} {src}') +def _git_clone(src: InstInfo, dest: Union[PosixPath, str], verbose: bool=True) -> bool: + if verbose: + log.info(f'cloning {src.srctype} {src}') + else: + log.debug(f'cloning {src.srctype} {src}') if src.srctype == Source.GITHUB_REPO: assert 'github.com' in src.source_loc source = f"{GITHUB_COM}" + src.source_loc.split("github.com")[-1] @@ -2137,8 +2140,17 @@ def available_plugins() -> list: # It takes too many API calls to query for installable plugins accurately. if source.type == Source.GITHUB_REPO and not source.local_clone: # FIXME: ignoring non-cloned repos for now. - log.debug(f'unable to search {source.original_source} without a local clone of the repository.') - continue + log.debug(f'cloning {source.original_source} in order to search') + clone = copy_remote_git_source(InstInfo(None, + source.original_source, + source.original_source, + source_dir=source.content), + verbose=False) + if not clone: + log.warning(f"could not clone github source {source.original_source}") + continue + source.local_clone = clone + source.local_clone.parent_source = source if source.local_clone: candidates.extend(find_plugin_candidates(source.local_clone)) From 294bad45ea87bc0e6c283f3210e02f753efbe4f8 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 30 Oct 2025 18:27:51 -0500 Subject: [PATCH 11/47] pytest: add reckless listavailable test --- tests/test_reckless.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 6dd479d45bbf..7d613267235d 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -418,3 +418,15 @@ def test_reckless_uv_install(node_factory): assert r.search_stdout('using installer pythonuv') r.check_stderr() + + +def test_reckless_available(node_factory): + """list available plugins""" + n = get_reckless_node(node_factory) + r = reckless([f"--network={NETWORK}", "listavailable", "-v", "--json"], dir=n.lightning_dir) + assert r.returncode == 0 + # All plugins in the default repo should be found and identified as installable. + assert r.search_stdout('testplugfail') + assert r.search_stdout('testplugpass') + assert r.search_stdout('testplugpyproj') + assert r.search_stdout('testpluguv') From 2a41f64183948fed86684c4e741e1854564a6971 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 5 Nov 2025 17:07:16 -0600 Subject: [PATCH 12/47] reckless: add listconfig command Requested by @ShahanaFarooqui while accessing reckless via rpc in order to find out where the plugins are installed and enabled. --- tests/test_reckless.py | 23 +++++++++++++++++---- tools/reckless | 45 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 7d613267235d..43212de3b606 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -3,6 +3,7 @@ from pathlib import PosixPath, Path import socket from pyln.testing.utils import VALGRIND +import json import pytest import os import re @@ -170,16 +171,30 @@ def test_basic_help(): assert r.search_stdout("options:") or r.search_stdout("optional arguments:") -def test_version(): +def test_reckless_version(node_factory): '''Version should be reported without loading config and should advance - with lightningd''' - r = reckless(["-V", "-v", "--json"]) + with lightningd.''' + node = get_reckless_node(node_factory) + r = reckless(["-V", "-v", "--json"], dir=node.lightning_dir) assert r.returncode == 0 - import json json_out = ''.join(r.stdout) with open('.version', 'r') as f: version = f.readlines()[0].strip() assert json.loads(json_out)['result'][0] == version + assert not r.search_stdout('config file not found') + + # reckless listconfig should report the reckless version as well. + NETWORK = os.environ.get('TEST_NETWORK') + if not NETWORK: + NETWORK = 'regtest' + r = reckless(['listconfig', f'--network={NETWORK}', '--json'], + dir=node.lightning_dir) + assert r.returncode == 0 + result = json.loads(''.join(r.stdout))['result'] + assert result['network'] == NETWORK + assert result['reckless_dir'] == str(node.lightning_dir / 'reckless') + assert result['lightning_conf'] == str(node.lightning_dir / NETWORK / 'config') + assert result['version'] == version def test_contextual_help(node_factory): diff --git a/tools/reckless b/tools/reckless index 941002e42d6f..46ba97b69a85 100755 --- a/tools/reckless +++ b/tools/reckless @@ -99,10 +99,13 @@ class Logger: def reply_json(self): """json output to stdout with accumulated result.""" - if len(log.json_output["result"]) == 1 and \ - isinstance(log.json_output["result"][0], list): - # unpack sources output - log.json_output["result"] = log.json_output["result"][0] + if len(log.json_output["result"]) == 1: + if isinstance(log.json_output["result"][0], list): + # unpack sources output + log.json_output["result"] = log.json_output["result"][0] + elif isinstance(log.json_output['result'][0], dict): + # If result is only a single dict, unpack it from the result list + log.json_output['result'] = log.json_output['result'][0] output = json.dumps(log.json_output, indent=3) + '\n' ratelimit_output(output) @@ -846,6 +849,8 @@ class RecklessConfig(Config): ) Config.__init__(self, path=str(path), default_text=default_text) self.reckless_dir = Path(path).parent + # Which lightning config needs to inherit the reckless config? + self.lightning_conf = None class LightningBitcoinConfig(Config): @@ -1863,6 +1868,7 @@ def load_config(reckless_dir: Union[str, None] = None, reckless_abort('Error: could not load or create the network specific lightningd' ' config (default .lightning/bitcoin)') net_conf.editConfigFile(f'include {reckless_conf.conf_fp}', None) + reckless_conf.lightning_conf = network_path return reckless_conf @@ -2164,6 +2170,31 @@ def available_plugins() -> list: return candidates +def listconfig() -> dict: + """Useful for checking options passed through the reckless-rpc.""" + config = {} + + log.info(f'requested lightning config: {LIGHTNING_CONFIG}') + config.update({'requested_lightning_conf': LIGHTNING_CONFIG}) + + log.info(f'lightning config in use: {RECKLESS_CONFIG.lightning_conf}') + config.update({'lightning_conf': str(RECKLESS_CONFIG.lightning_conf)}) + + log.info(f'lightning directory: {LIGHTNING_DIR}') + config.update({'lightning_dir': str(LIGHTNING_DIR)}) + + log.info(f'reckless directory: {RECKLESS_CONFIG.reckless_dir}') + config.update({'reckless_dir': str(RECKLESS_CONFIG.reckless_dir)}) + + log.info(f'network: {NETWORK}') + config.update({'network': NETWORK}) + + log.info(f'reckless version: {__VERSION__}') + config.update({'version': __VERSION__}) + + return config + + def report_version() -> str: """return reckless version""" log.info(__VERSION__) @@ -2178,7 +2209,7 @@ def unpack_json_arg(json_target: str) -> list: return None if isinstance(targets, list): return targets - log.warning(f'input {target_list} is not a json array') + log.warning(f'input {json_target} is not a json array') return None @@ -2277,10 +2308,12 @@ if __name__ == '__main__': parser.add_argument('-V', '--version', action=StoreTrueIdempotent, const=None, help='print version and exit') + listconfig_cmd = cmd1.add_parser('listconfig', help='list options passed to reckless') + listconfig_cmd.set_defaults(func=listconfig) all_parsers = [parser, install_cmd, uninstall_cmd, search_cmd, enable_cmd, disable_cmd, list_parse, source_add, source_rem, help_cmd, - update, list_cmd, available_cmd] + update, list_cmd, available_cmd, listconfig_cmd] for p in all_parsers: # This default depends on the .lightning directory p.add_argument('-d', '--reckless-dir', action=StoreIdempotent, From 0deb2f691d39e5027e0d1279b36d061011d50379 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 13 Nov 2025 18:14:38 -0600 Subject: [PATCH 13/47] pytest: test reckless listconfig via rpc --- tests/test_reckless.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 43212de3b606..4d971825b886 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -171,7 +171,7 @@ def test_basic_help(): assert r.search_stdout("options:") or r.search_stdout("optional arguments:") -def test_reckless_version(node_factory): +def test_reckless_version_listconfig(node_factory): '''Version should be reported without loading config and should advance with lightningd.''' node = get_reckless_node(node_factory) @@ -196,6 +196,16 @@ def test_reckless_version(node_factory): assert result['lightning_conf'] == str(node.lightning_dir / NETWORK / 'config') assert result['version'] == version + # Now test via reckless-rpc plugin + node.start() + # FIXME: the plugin finds the installed reckless utility rather than the build directory reckless + listconfig = node.rpc.reckless('listconfig') + print(listconfig) + assert listconfig['result']['lightning_dir'] == str(node.lightning_dir) + assert listconfig['result']['lightning_conf'] == str(node.lightning_dir / NETWORK / 'config') + assert listconfig['result']['network'] == NETWORK + assert listconfig['result']['version'] == version + def test_contextual_help(node_factory): n = get_reckless_node(node_factory) From f1e5d90dc10de851bf3b7bea6f931f8daeb3c39a Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 20 Nov 2025 18:41:45 -0600 Subject: [PATCH 14/47] reckless: check for manifest.json in plugin directories The manifest.json provides a short and long description of the plugin, dependencies, and specifies the entrypoint in case it's not named the same as the plugin. changelog-changed: Reckless uses a manifest in the plugin directory to gain additional details about plugin and installation. --- .../lightningd/testplugfail/manifest.json | 7 + .../lightningd/testplugpass/manifest.json | 7 + .../lightningd/testplugpyproj/manifest.json | 7 + .../lightningd/testpluguv/manifest.json | 7 + tools/reckless | 134 +++++++++++++++--- 5 files changed, 141 insertions(+), 21 deletions(-) create mode 100644 tests/data/recklessrepo/lightningd/testplugfail/manifest.json create mode 100644 tests/data/recklessrepo/lightningd/testplugpass/manifest.json create mode 100644 tests/data/recklessrepo/lightningd/testplugpyproj/manifest.json create mode 100644 tests/data/recklessrepo/lightningd/testpluguv/manifest.json diff --git a/tests/data/recklessrepo/lightningd/testplugfail/manifest.json b/tests/data/recklessrepo/lightningd/testplugfail/manifest.json new file mode 100644 index 000000000000..8c6857aaa70e --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugfail/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testplugfail", + "short_description": "a plugin to test reckless installation where the plugin fails to start", + "long_description": "This plugin is one of several used in the reckless blackbox tests.", + "entrypoint": "testplugfail.py", + "requirements": ["python3"] +} diff --git a/tests/data/recklessrepo/lightningd/testplugpass/manifest.json b/tests/data/recklessrepo/lightningd/testplugpass/manifest.json new file mode 100644 index 000000000000..9df31c6d19f0 --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugpass/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testplugpass", + "short_description": "a plugin to test reckless installation", + "long_description": "This plugin is one of several used in the reckless blackbox tests. This one should success in dependenciy installation, and start up when activated in Core Lightning.", + "entrypoint": "testplugpass.py", + "requirements": ["python3"] +} diff --git a/tests/data/recklessrepo/lightningd/testplugpyproj/manifest.json b/tests/data/recklessrepo/lightningd/testplugpyproj/manifest.json new file mode 100644 index 000000000000..0215ca2fae31 --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugpyproj/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testplugpyproj", + "short_description": "a plugin to test reckless installation", + "long_description": "This plugin is one of several used in the reckless blackbox tests. This one should succeed while specifying dependencies in pyproject.toml.", + "entrypoint": "testplugpyproj.py", + "requirements": ["python3"] +} diff --git a/tests/data/recklessrepo/lightningd/testpluguv/manifest.json b/tests/data/recklessrepo/lightningd/testpluguv/manifest.json new file mode 100644 index 000000000000..31f9ce7027cd --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testpluguv/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testpluguv", + "short_description": "a plugin to test reckless installation using uv", + "long_description": "This plugin is one of several used in the reckless blackbox tests. This one specifies dependencies for uv in the pyproject.toml and has a corresponding uv.lock file.", + "entrypoint": "testpluguv.py", + "requirements": ["python3"] +} diff --git a/tools/reckless b/tools/reckless index 46ba97b69a85..5c78eff86fba 100755 --- a/tools/reckless +++ b/tools/reckless @@ -233,6 +233,20 @@ def remove_dir(directory: str) -> bool: return False +class GithubRepository(): + """extract the github user account and repository name.""" + def __init__(self, url: str): + assert 'github.com/' in url.lower() + url_parts = Path(str(url).lower().partition('github.com/')[2]).parts + assert len(url_parts) >= 2 + self.user = url_parts[0] + self.name = url_parts[1].removesuffix('.git') + self.url = url + + def __repr__(self): + return '' + + class Source(Enum): DIRECTORY = 1 LOCAL_REPO = 2 @@ -262,12 +276,11 @@ class Source(Enum): @classmethod def get_github_user_repo(cls, source: str) -> (str, str): 'extract a github username and repository name' - if 'github.com/' not in source.lower(): - return None, None - trailing = Path(source.lower().partition('github.com/')[2]).parts - if len(trailing) < 2: + try: + repo = GithubRepository(source) + return repo.user, repo.name + except: return None, None - return trailing[0], trailing[1].removesuffix('.git') class SubmoduleSource: @@ -325,7 +338,7 @@ class SourceDir(): if self.srctype == Source.DIRECTORY: self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: - self.contents = populate_local_repo(self.location, parent_source=self.parent_source) + self.contents = populate_local_repo(self.location, parent=self, parent_source=self.parent_source) elif self.srctype == Source.GITHUB_REPO: self.contents = populate_github_repo(self.location) else: @@ -351,7 +364,7 @@ class SourceDir(): return None def __repr__(self): - return f"" + return f"" def __eq__(self, compared): if isinstance(compared, str): @@ -576,7 +589,8 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: parentdir.name) else: relative_path = parentdir.name - child = SourceDir(p, srctype=Source.LOCAL_REPO, + child = SourceDir(p, srctype=parent.srctype, + parent_source=parent_source, relative=relative_path) # ls-tree lists every file in the repo with full path. # No need to populate each directory individually. @@ -611,8 +625,13 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: relative_path = filepath elif basedir.relative: relative_path = str(Path(basedir.relative) / filepath) - assert relative_path - submodule_dir = SourceDir(filepath, srctype=Source.LOCAL_REPO, + else: + relative_path = filepath + if parent: + srctype = parent.srctype + else: + srctype = Source.LOCAL_REPO + submodule_dir = SourceDir(filepath, srctype=srctype, relative=relative_path, parent_source=parent_source) populate_local_repo(Path(path) / filepath, parent=submodule_dir, @@ -710,7 +729,7 @@ def populate_github_repo(url: str) -> list: return contents -def copy_remote_git_source(github_source: InstInfo, verbose: bool=True): +def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> SourceDir: """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: @@ -2113,17 +2132,84 @@ def listinstalled(): return plugins -def find_plugin_candidates(source: SourceDir, depth=2) -> list: +def have_files(source: SourceDir): + """Do we have direct access to the files in this directory?""" + if source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, + Source.GIT_LOCAL_CLONE]: + return True + log.info(f'no files in {source.name} ({source.srctype})') + return False + + +def fetch_manifest(source: SourceDir) -> dict: + """read and ingest a manifest from the provided source.""" + log.debug(f'ingesting manifest from {source.name}: {source.location}/manifest.json ({source.srctype})') + # local_path = RECKLESS_DIR / '.remote_sources' / user + if source.srctype not in [Source.GIT_LOCAL_CLONE, Source.LOCAL_REPO, Source.DIRECTORY]: + log.info(f'oops! {source.srctype}') + return None + if source.srctype == Source.GIT_LOCAL_CLONE: + try: + repo = GithubRepository(source.parent_source.original_source) + path = RECKLESS_DIR / '.remote_sources' / repo.user / repo.name + except AssertionError: + log.info(f'could not parse github source {source.parent_source.original_source}') + return None + elif source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO]: + path = Path(source.location) + else: + raise Exception(f"cannot access manifest in {source.srctype}: {source}") + if source.relative: + path = path / source.relative + path = path / 'manifest.json' + if not path.exists(): + return None + with open(path, 'r+') as manifest_file: + try: + manifest = json.load(manifest_file) + return manifest + except json.decoder.JSONDecodeError: + log.warning(f'{source.name} contains malformed manifest ({source.parent_source.original_source})') + return None + + +def find_plugin_candidates(source: Union[LoadedSource, SourceDir], depth=2) -> list: """Filter through a source and return any candidates that appear to be installable plugins with the registered installers.""" + if isinstance(source, LoadedSource): + if source.local_clone: + return find_plugin_candidates(source.local_clone) + return find_plugin_candidates(source.content) + candidates = [] assert isinstance(source, SourceDir) if not source.contents and not source.prepopulated: source.populate() + for s in source.contents: + if isinstance(s, SourceDir): + assert s.srctype == source.srctype, f'source dir {s.name}, {s.srctype} did not inherit {source.srctype} from {source.name}' + assert s.parent_source == source.parent_source, f'source dir {s.name} did not inherit parent {source.parent_source} from {source.name}' guess = InstInfo(source.name, source.location, None, source_dir=source) + guess.srctype = source.srctype + manifest = None if guess.get_inst_details(): - candidates.append(source.name) + guess.srctype = source.srctype + guess.source_dir.srctype = source.srctype + if guess.source_dir.find('manifest.json'): + # FIXME: Handle github source case + if have_files(guess.source_dir): + manifest = fetch_manifest(guess.source_dir) + + if manifest: + candidate = manifest + else: + candidate = {'name': source.name, + 'short_description': None, + 'long_description': None, + 'entrypoint': guess.entry, + 'requirements': []} + candidates.append(candidate) if depth <= 1: return candidates @@ -2152,21 +2238,27 @@ def available_plugins() -> list: source.original_source, source_dir=source.content), verbose=False) + clone.srctype = Source.GIT_LOCAL_CLONE + clone.parent_source = source if not clone: log.warning(f"could not clone github source {source.original_source}") continue source.local_clone = clone source.local_clone.parent_source = source - if source.local_clone: - candidates.extend(find_plugin_candidates(source.local_clone)) - else: - candidates.extend(find_plugin_candidates(source.content)) + candidates.extend(find_plugin_candidates(source)) + + # json output requested + if log.capture: + return candidates + + for c in candidates: + log.info(c['name']) + if c['short_description']: + log.info(f'\tdescription: {c["short_description"]}') + if c['requirements']: + log.info(f'\trequirements: {c["requirements"]}') - # Order and deduplicate results - candidates = list(set(candidates)) - candidates.sort() - log.info(' '.join(candidates)) return candidates From 24bb1ac0348fd47b0f21817ac5e2077ca0a2427d Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 18 Dec 2025 17:31:46 -0600 Subject: [PATCH 15/47] reckless: add shebang installer for fancy plugins This allows plugins with a uv run script to install themselves. Unfortunately it requires file access to all potential entrypoints to check if they are installable. Changelog-Added: reckless can now install plugins executable by shebang. --- .../lightningd/testplugshebang/manifest.json | 7 + .../testplugshebang/requirements.txt | 2 + .../testplugshebang/testplugshebang.py | 27 ++++ .../rkls_api_lightningd_plugins.json | 9 ++ tests/test_reckless.py | 15 +++ tools/reckless | 126 +++++++++++++++++- 6 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 tests/data/recklessrepo/lightningd/testplugshebang/manifest.json create mode 100644 tests/data/recklessrepo/lightningd/testplugshebang/requirements.txt create mode 100755 tests/data/recklessrepo/lightningd/testplugshebang/testplugshebang.py diff --git a/tests/data/recklessrepo/lightningd/testplugshebang/manifest.json b/tests/data/recklessrepo/lightningd/testplugshebang/manifest.json new file mode 100644 index 000000000000..379447fe5ba8 --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugshebang/manifest.json @@ -0,0 +1,7 @@ +{ + "name": "testplugshebang", + "short_description": "a plugin to test reckless installation with a UV shebang", + "long_description": "This plugin is used in the reckless blackbox tests. This one manages its own dependency installation with uv invoked by #! from within the plugin.", + "entrypoint": "testplugshebang.py", + "requirements": ["python3"] +} diff --git a/tests/data/recklessrepo/lightningd/testplugshebang/requirements.txt b/tests/data/recklessrepo/lightningd/testplugshebang/requirements.txt new file mode 100644 index 000000000000..7b19e677138d --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugshebang/requirements.txt @@ -0,0 +1,2 @@ +pyln-client + diff --git a/tests/data/recklessrepo/lightningd/testplugshebang/testplugshebang.py b/tests/data/recklessrepo/lightningd/testplugshebang/testplugshebang.py new file mode 100755 index 000000000000..13c6a0caa425 --- /dev/null +++ b/tests/data/recklessrepo/lightningd/testplugshebang/testplugshebang.py @@ -0,0 +1,27 @@ +#!/usr/bin/env -S uv run --script + +# /// script +# requires-python = ">=3.9.2" +# dependencies = [ +# "pyln-client>=25.12", +# ] +# /// + +from pyln.client import Plugin + +plugin = Plugin() + +__version__ = 'v1' + + +@plugin.init() +def init(options, configuration, plugin, **kwargs): + plugin.log("testplugshebang initialized") + + +@plugin.method("plugintest") +def plugintest(plugin): + return ("success") + + +plugin.run() diff --git a/tests/data/recklessrepo/rkls_api_lightningd_plugins.json b/tests/data/recklessrepo/rkls_api_lightningd_plugins.json index a91c4844d898..e28e55c853e6 100644 --- a/tests/data/recklessrepo/rkls_api_lightningd_plugins.json +++ b/tests/data/recklessrepo/rkls_api_lightningd_plugins.json @@ -34,5 +34,14 @@ "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpyproj", "download_url": null, "type": "dir" + }, + { + "name": "testplugshebang", + "path": "testplugshebang", + "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", + "html_url": "https://github.com/lightningd/plugins/tree/master/testplugshebang", + "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugshebang", + "download_url": null, + "type": "dir" } ] diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 4d971825b886..858bbee0d50b 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -445,6 +445,21 @@ def test_reckless_uv_install(node_factory): r.check_stderr() +@unittest.skipIf(VALGRIND, "node too slow for starting plugin under valgrind") +def test_reckless_shebang_install(node_factory): + node = get_reckless_node(node_factory) + node.start() + r = reckless([f"--network={NETWORK}", "-v", "install", "testplugshebang"], + dir=node.lightning_dir) + assert r.returncode == 0 + installed_path = Path(node.lightning_dir) / 'reckless/testplugshebang' + assert installed_path.is_dir() + assert node.rpc.plugintest() == 'success' + + assert r.search_stdout('using installer shebang') + r.check_stderr() + + def test_reckless_available(node_factory): """list available plugins""" n = get_reckless_node(node_factory) diff --git a/tools/reckless b/tools/reckless index 5c78eff86fba..4d5006bd8f78 100755 --- a/tools/reckless +++ b/tools/reckless @@ -157,6 +157,9 @@ class Installer: self.manager = manager # dependency manager (if required) self.dependency_file = None self.dependency_call = None + # extra check routine to see if a source is installable by this Installer + self.check = None + def __repr__(self): return (f' bool: + def installable(self, source) -> bool: '''Validate the necessary compiler and package manager executables are available to install. If these are defined, they are considered mandatory even though the user may have the requisite packages already @@ -184,6 +187,8 @@ class Installer: return False if self.manager and not shutil.which(self.manager): return False + if self.check: + return self.check(source) return True def add_entrypoint(self, entry: str): @@ -484,9 +489,14 @@ class InstInfo: found_entry = sub.find(g, ftype=SourceFile) if found_entry: break - # FIXME: handle a list of dependencies - found_dep = sub.find(inst.dependency_file, - ftype=SourceFile) + + if inst.dependency_file: + # FIXME: handle a list of dependencies + found_dep = sub.find(inst.dependency_file, + ftype=SourceFile) + else: + found_dep = None + if found_entry: # Success! if found_dep: @@ -1031,6 +1041,47 @@ def install_to_python_virtual_environment(cloned_plugin: InstInfo): return cloned_plugin +def have_files(source: SourceDir): + """Do we have direct access to the files in this directory?""" + if source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, + Source.GIT_LOCAL_CLONE]: + return True + log.info(f'no files in {source.name} ({source.srctype})') + return False + + +def fetch_manifest(source: SourceDir) -> dict: + """read and ingest a manifest from the provided source.""" + log.debug(f'ingesting manifest from {source.name}: {source.location}/manifest.json ({source.srctype})') + # local_path = RECKLESS_DIR / '.remote_sources' / user + if source.srctype not in [Source.GIT_LOCAL_CLONE, Source.LOCAL_REPO, Source.DIRECTORY]: + log.info(f'oops! {source.srctype}') + return None + if source.srctype == Source.GIT_LOCAL_CLONE: + try: + repo = GithubRepository(source.parent_source.original_source) + path = RECKLESS_DIR / '.remote_sources' / repo.user / repo.name + except AssertionError: + log.info(f'could not parse github source {source.parent_source.original_source}') + return None + elif source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO]: + path = Path(source.location) + else: + raise Exception(f"cannot access manifest in {source.srctype}: {source}") + if source.relative: + path = path / source.relative + path = path / 'manifest.json' + if not path.exists(): + return None + with open(path, 'r+') as manifest_file: + try: + manifest = json.load(manifest_file) + return manifest + except json.decoder.JSONDecodeError: + log.warning(f'{source.name} contains malformed manifest ({source.parent_source.original_source})') + return None + + def cargo_installation(cloned_plugin: InstInfo): call = ['cargo', 'build', '--release', '-vv'] # FIXME: the symlinked Cargo.toml allows the installer to identify a valid @@ -1144,6 +1195,45 @@ def install_python_uv_legacy(cloned_plugin: InstInfo): return cloned_plugin +def open_source_entrypoint(source: InstInfo) -> str: + if source.srctype not in [Source.GIT_LOCAL_CLONE, Source.LOCAL_REPO, Source.DIRECTORY]: + log.info(f'oops! {source.srctype}') + return None + assert source.entry + file = Path(source.source_loc) + # if source.subdir: + # file /= source.subdir + file /= source.entry + log.debug(f'checking entry file {str(file)}') + if file.exists(): + # FIXME: check file encoding + try: + with open(file, 'r') as f: + return f.read() + except UnicodeDecodeError: + log.debug('failed to read source file') + return None + else: + log.debug('could not find source file') + + return None + +def check_for_shebang(source: InstInfo) -> bool: + log.debug(f'checking for shebang in {source}') + if source.source_dir: + source.get_inst_details() + if have_files(source.source_dir): + entrypoint_file = open_source_entrypoint(source) + if entrypoint_file.split('\n')[0].startswith('#!'): + # Calling the python interpreter will not manage dependencies. + # Leave this to another python installer. + for interpreter in ['bin/python', 'env python']: + if interpreter in entrypoint_file.split('\n')[0]: + return False + return True + return False + + python3venv = Installer('python3venv', exe='python3', manager='pip', entry='{name}.py') python3venv.add_entrypoint('{name}') @@ -1186,7 +1276,12 @@ rust_cargo = Installer('rust', manager='cargo', entry='Cargo.toml') rust_cargo.add_dependency_file('Cargo.toml') rust_cargo.dependency_call = cargo_installation -INSTALLERS = [pythonuv, pythonuvlegacy, python3venv, poetryvenv, +shebang = Installer('shebang', entry='{name}.py') +shebang.add_entrypoint('{name}') +# An extra installable check to see if a #! is present in the file +shebang.check = check_for_shebang + +INSTALLERS = [shebang, pythonuv, pythonuvlegacy, python3venv, poetryvenv, pyprojectViaPip, nodejs, rust_cargo] @@ -1376,6 +1471,8 @@ def _checkout_commit(orig_src: InstInfo, def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: """make sure the repo exists and clone it.""" log.debug(f'Install requested from {src}.') + if src.source_dir and src.source_dir.parent_source: + log.debug(f'source has parent {src.source_dir.parent_source}') if RECKLESS_CONFIG is None: log.error('reckless install directory unavailable') return None @@ -1411,6 +1508,8 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # FIXME: Validate path was cloned successfully. # Depending on how we accessed the original source, there may be install # details missing. Searching the cloned repo makes sure we have it. + # FIXME: This could be cloned to .remotesources and the global sources + # could then be updated with this new LoadedSource to save on additional cloning. clone = LoadedSource(plugin_path) clone.content.populate() # Make sure we don't try to fetch again! @@ -1426,14 +1525,29 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: if not plugin_path: return None + # FIXME: replace src wholesale + # We have a hunch it's in this directory/source, so link it here. + inst_check_src = copy.copy(src) + if not inst_check_src.source_dir: + inst_check_src.source_loc = plugin_path + inst_check_src.source_dir = clone.content + inst_check_src.source_dir.parent_source = clone + + if src.srctype == Source.GITHUB_REPO: + inst_check_src.srctype = Source.GIT_LOCAL_CLONE + else: + inst_check_src.srctype = clone.type + # Find a suitable installer INSTALLER = None for inst_method in INSTALLERS: - if not (inst_method.installable() and inst_method.executable()): + if not (inst_method.installable(inst_check_src) and inst_method.executable()): continue if inst_method.dependency_file is not None: if inst_method.dependency_file not in os.listdir(plugin_path): continue + if inst_method.check and not inst_method.check(inst_check_src): + continue log.debug(f"using installer {inst_method.name}") INSTALLER = inst_method break From 9de680033698a463224878163ff8eb8e0930881b Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 6 Jan 2026 11:59:38 -0600 Subject: [PATCH 16/47] reckless install: check and warn if a plugin is already installed. --- tests/test_reckless.py | 6 ++++++ tools/reckless | 26 ++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 858bbee0d50b..f69ff213b941 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -300,6 +300,12 @@ def test_install(node_factory): print(plugin_path) assert os.path.exists(plugin_path) + # Try to install again - should result in a warning. + r = reckless([f"--network={NETWORK}", "-v", "install", "testplugpass"], dir=n.lightning_dir) + r.check_stderr() + assert r.search_stdout('already installed') + assert r.returncode == 0 + def test_install_cleanup(node_factory): """test failed installation and post install cleanup""" diff --git a/tools/reckless b/tools/reckless index 4d5006bd8f78..2b3c8a717c40 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1731,6 +1731,20 @@ def install(plugin_name: str) -> Union[str, None]: LAST_FOUND = None return None + # Check if we already have this installed. + destination = Path(RECKLESS_CONFIG.reckless_dir) / name.lower() + + if Path(destination).exists(): + # should we run listinstalled first and see what's in the list? + installed = listinstalled(plugin_name) + if installed: + log.info(f'already installed: {list(installed.keys())[0]} in {str(destination)}') + return name + else: + log.warning(f'destination directory {destination} already exists.') + return None + + try: installed = _install_plugin(src) except FileExistsError as err: @@ -2189,8 +2203,9 @@ def extract_metadata(plugin_name: str) -> dict: return metadata -def listinstalled(): - """list all plugins currently managed by reckless""" +def listinstalled(name: str = None): + """list all plugins currently managed by reckless. Optionally passed + a plugin name.""" dir_contents = os.listdir(RECKLESS_CONFIG.reckless_dir) plugins = {} for plugin in dir_contents: @@ -2198,6 +2213,8 @@ def listinstalled(): # skip hidden dirs such as reckless' .remote_sources if plugin[0] == '.': continue + if name and name != plugin: + continue plugins.update({plugin: None}) # Format output in a simple table @@ -2230,8 +2247,9 @@ def listinstalled(): status = "disabled" else: print(f'cant handle {line}') - log.info(f"{plugin:<{name_len}} {md['installed commit']:<{inst_len}} " - f"{md['installation date']:<11} {status}") + if not name: + log.info(f"{plugin:<{name_len}} {md['installed commit']:<{inst_len}} " + f"{md['installation date']:<11} {status}") # This doesn't originate from the metadata, but we want to provide enabled status for json output md['enabled'] = status == "enabled" md['entrypoint'] = installed.entry From c22722035a3cc28f1d74de5bd6b3f53a411ed0fd Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 8 Jan 2026 13:32:48 -0600 Subject: [PATCH 17/47] reckless: remove github API access The shebang installer requires introspection of files, and the listavailable command reads the manifest for each plugin. Due to the need for file access, cloning all remote repositories is now simpler and faster, so it's time to rip out the github API access code. --- tools/reckless | 184 ++++++++++++++----------------------------------- 1 file changed, 50 insertions(+), 134 deletions(-) diff --git a/tools/reckless b/tools/reckless index 2b3c8a717c40..0a586e701367 100755 --- a/tools/reckless +++ b/tools/reckless @@ -255,7 +255,7 @@ class GithubRepository(): class Source(Enum): DIRECTORY = 1 LOCAL_REPO = 2 - GITHUB_REPO = 3 + REMOTE_GIT_REPO = 3 OTHER_URL = 4 UNKNOWN = 5 # Cloned from remote source before searching (rather than github API) @@ -309,11 +309,14 @@ class LoadedSource: self.content = SourceDir(source, self.type) self.local_clone = None self.local_clone_fetched = False - if self.type == Source.GITHUB_REPO: + if self.type == Source.REMOTE_GIT_REPO: local = _get_local_clone(source) if local: self.local_clone = SourceDir(local, Source.GIT_LOCAL_CLONE) - self.local_clone.parent_source = self + else: + self.local_clone = copy_remote_git_source(InstInfo(None, source)) + self.content = self.local_clone + self.local_clone.parent_source = self def __repr__(self): return f'' @@ -344,8 +347,9 @@ class SourceDir(): self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: self.contents = populate_local_repo(self.location, parent=self, parent_source=self.parent_source) - elif self.srctype == Source.GITHUB_REPO: - self.contents = populate_github_repo(self.location) + elif self.srctype == Source.REMOTE_GIT_REPO: + self.contents = copy_remote_git_source(InstInfo(self.name, self.location)).contents + else: raise Exception("populate method undefined for {self.srctype}") # Ensure the relative path of the contents is inherited. @@ -398,11 +402,10 @@ class SourceFile(): class InstInfo: - def __init__(self, name: str, location: str, git_url: str, source_dir: SourceDir=None): + def __init__(self, name: str, location: str, source_dir: SourceDir=None): self.name = name self.source_loc = str(location) # Used for 'git clone' self.source_dir = source_dir # Use this insead of source_loc to only fetch once. - self.git_url: str = git_url # API access for github repos self.srctype: Source = Source.get_type(location) self.entry: SourceFile = None # relative to source_loc or subdir self.deps: str = None @@ -410,7 +413,7 @@ class InstInfo: self.commit: str = None def __repr__(self): - return (f'InstInfo({self.name}, {self.source_loc}, {self.git_url}, ' + return (f'InstInfo({self.name}, {self.source_loc}, ' f'{self.entry}, {self.deps}, {self.subdir})') def get_repo_commit(self) -> Union[str, None]: @@ -422,26 +425,9 @@ class InstInfo: return None return git.stdout.splitlines()[0] - if self.srctype == Source.GITHUB_REPO: - parsed_url = urlparse(self.source_loc) - if 'github.com' not in parsed_url.netloc: - return None - if len(parsed_url.path.split('/')) < 2: - return None - start = 1 - # Maybe we were passed an api.github.com/repo/ url - if 'api' in parsed_url.netloc: - start += 1 - repo_user = parsed_url.path.split('/')[start] - repo_name = parsed_url.path.split('/')[start + 1] - api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/commits?ref=HEAD' - r = urlopen(api_url, timeout=5) - if r.status != 200: - return None - try: - return json.loads(r.read().decode())['0']['sha'] - except: - return None + if self.srctype == Source.REMOTE_GIT_REPO: + # The remote git source is not accessed directly. Use the local clone. + assert False def get_inst_details(self, permissive: bool=False) -> bool: """Search the source_loc for plugin install details. @@ -461,7 +447,7 @@ class InstInfo: if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: depth = 5 - elif self.srctype == Source.GITHUB_REPO: + elif self.srctype == Source.REMOTE_GIT_REPO: depth = 1 def search_dir(self, sub: SourceDir, subdir: bool, @@ -522,7 +508,7 @@ class InstInfo: # Fall back to cloning and searching the local copy instead. except HTTPError: result = None - if self.srctype == Source.GITHUB_REPO: + if self.srctype == Source.REMOTE_GIT_REPO: # clone source to reckless dir target = copy_remote_git_source(self) if not target: @@ -654,91 +640,6 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return basedir.contents -def source_element_from_repo_api(member: dict): - # api accessed via /contents/ - if 'type' in member and 'name' in member and 'git_url' in member: - if member['type'] == 'dir': - return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, - name=member['name']) - elif member['type'] == 'file': - # Likely a submodule - if member['size'] == 0: - return SourceDir(None, srctype=Source.GITHUB_REPO, - name=member['name']) - return SourceFile(member['name']) - elif member['type'] == 'commit': - # No path is given by the api here - return SourceDir(None, srctype=Source.GITHUB_REPO, - name=member['name']) - # git_url with /tree/ presents results a little differently - elif 'type' in member and 'path' in member and 'url' in member: - if member['type'] not in ['tree', 'blob']: - log.debug(f' skipping {member["path"]} type={member["type"]}') - if member['type'] == 'tree': - return SourceDir(member['url'], srctype=Source.GITHUB_REPO, - name=member['path']) - elif member['type'] == 'blob': - # This can be a submodule - if member['size'] == 0: - return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, - name=member['name']) - return SourceFile(member['path']) - elif member['type'] == 'commit': - # No path is given by the api here - return SourceDir(None, srctype=Source.GITHUB_REPO, - name=member['name']) - return None - - -def populate_github_repo(url: str) -> list: - """populate one level of a github repository via REST API""" - # Forces search to clone remote repos (for blackbox testing) - if GITHUB_API_FALLBACK: - with tempfile.NamedTemporaryFile() as tmp: - raise HTTPError(url, 403, 'simulated ratelimit', {}, tmp) - # FIXME: This probably contains leftover cruft. - repo = url.split('/') - while '' in repo: - repo.remove('') - repo_name = None - parsed_url = urlparse(url.removesuffix('.git')) - if 'github.com' not in parsed_url.netloc: - return None - if len(parsed_url.path.split('/')) < 2: - return None - start = 1 - # Maybe we were passed an api.github.com/repo/ url - if 'api' in parsed_url.netloc: - start += 1 - repo_user = parsed_url.path.split('/')[start] - repo_name = parsed_url.path.split('/')[start + 1] - - # Get details from the github API. - if API_GITHUB_COM in url: - api_url = url - else: - api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/contents/' - - git_url = api_url - if "api.github.com" in git_url: - # This lets us redirect to handle blackbox testing - log.debug(f'fetching from gh API: {git_url}') - git_url = (API_GITHUB_COM + git_url.split("api.github.com")[-1]) - # Ratelimiting occurs for non-authenticated GH API calls at 60 in 1 hour. - r = urlopen(git_url, timeout=5) - if r.status != 200: - return False - if 'git/tree' in git_url: - tree = json.loads(r.read().decode())['tree'] - else: - tree = json.loads(r.read().decode()) - contents = [] - for sub in tree: - if source_element_from_repo_api(sub): - contents.append(source_element_from_repo_api(sub)) - return contents - - def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> SourceDir: """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) @@ -1314,11 +1215,11 @@ def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: """Identify source type, retrieve contents, and populate InstInfo if the relevant contents are found.""" root_dir = src.content - source = InstInfo(name, root_dir.location, None) + source = InstInfo(name, root_dir.location) # If a local clone of a github source already exists, prefer searching # that instead of accessing the github API. - if src.type == Source.GITHUB_REPO: + if src.type == Source.REMOTE_GIT_REPO: if src.local_clone: if not src.local_clone_fetched: # FIXME: Pass the LoadedSource here? @@ -1326,10 +1227,18 @@ def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: src.local_clone_fetched = True log.debug(f'fetching local clone of {src.original_source}') log.debug(f"Using local clone of {src}: {src.local_clone.location}") + + # FIXME: ideally, the InstInfo object would have a concept of the + # original LoadedSource and get_inst_details would follow the local clone source.source_loc = str(src.local_clone.location) source.srctype = Source.GIT_LOCAL_CLONE if source.get_inst_details(permissive=True): + # If we have a local clone, report back the original location and type, + # not the clone that was traversed. + if source.srctype is Source.GIT_LOCAL_CLONE: + source.source_loc = src.original_source + source.srctype = src.type return source return None @@ -1339,9 +1248,11 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str], verbose: bool=True) - log.info(f'cloning {src.srctype} {src}') else: log.debug(f'cloning {src.srctype} {src}') - if src.srctype == Source.GITHUB_REPO: - assert 'github.com' in src.source_loc - source = f"{GITHUB_COM}" + src.source_loc.split("github.com")[-1] + if src.srctype == Source.REMOTE_GIT_REPO: + if 'github.com' in src.source_loc: + source = f"{GITHUB_COM}" + src.source_loc.split("github.com")[-1] + else: + source = src.source_loc elif src.srctype in [Source.LOCAL_REPO, Source.OTHER_URL, Source.GIT_LOCAL_CLONE]: source = src.source_loc @@ -1360,8 +1271,14 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str], verbose: bool=True) - def _git_update(github_source: str, local_copy: PosixPath): + + if 'github.com' in github_source: + source = GITHUB_COM + github_source.split('github.com')[-1] + else: + source = github_source + # Ensure this is the correct source - git = run(['git', 'remote', 'set-url', 'origin', github_source], + git = run(['git', 'remote', 'set-url', 'origin', source], cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) assert git.returncode == 0 @@ -1386,7 +1303,7 @@ def _git_update(github_source: str, local_copy: PosixPath): default_branch = git.stdout.splitlines()[0] if default_branch not in ['origin/master', 'origin/main']: log.debug(f'UNUSUAL: fetched default branch {default_branch} for ' - f'{github_source}') + f'{source}') # Checkout default branch git = run(['git', 'checkout', default_branch], @@ -1433,7 +1350,7 @@ def _checkout_commit(orig_src: InstInfo, cloned_src: InstInfo, cloned_path: PosixPath): # Check out and verify commit/tag if source was a repository - if orig_src.srctype in [Source.LOCAL_REPO, Source.GITHUB_REPO, + if orig_src.srctype in [Source.LOCAL_REPO, Source.REMOTE_GIT_REPO, Source.OTHER_URL, Source.GIT_LOCAL_CLONE]: if orig_src.commit: log.debug(f"Checking out {orig_src.commit}") @@ -1500,7 +1417,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: f" {full_source_path}")) create_dir(clone_path) shutil.copytree(full_source_path, plugin_path) - elif src.srctype in [Source.LOCAL_REPO, Source.GITHUB_REPO, + elif src.srctype in [Source.LOCAL_REPO, Source.REMOTE_GIT_REPO, Source.OTHER_URL, Source.GIT_LOCAL_CLONE]: # clone git repository to /tmp/reckless-... if not _git_clone(src, plugin_path): @@ -1533,7 +1450,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: inst_check_src.source_dir = clone.content inst_check_src.source_dir.parent_source = clone - if src.srctype == Source.GITHUB_REPO: + if src.srctype == Source.REMOTE_GIT_REPO: inst_check_src.srctype = Source.GIT_LOCAL_CLONE else: inst_check_src.srctype = clone.type @@ -1712,7 +1629,7 @@ def install(plugin_name: str) -> Union[str, None]: src = None if direct_location: log.debug(f"install of {name} requested from {direct_location}") - src = InstInfo(name, direct_location, name) + src = InstInfo(name, direct_location) # Treating a local git repo as a directory allows testing # uncommitted changes. if src and src.srctype == Source.LOCAL_REPO: @@ -1764,7 +1681,7 @@ def install(plugin_name: str) -> Union[str, None]: def uninstall(plugin_name: str) -> str: - """dDisables plugin and deletes the plugin's reckless dir. Returns the + """Disables plugin and deletes the plugin's reckless dir. Returns the status of the uninstall attempt.""" assert isinstance(plugin_name, str) log.debug(f'Uninstalling plugin {plugin_name}') @@ -1818,7 +1735,7 @@ def search(plugin_name: str) -> Union[InstInfo, None]: for src in RECKLESS_SOURCES: # Search repos named after the plugin before collections - if src.type == Source.GITHUB_REPO: + if src.type == Source.REMOTE_GIT_REPO: if src.original_source.split('/')[-1].lower().removesuffix('.git') == plugin_name.lower(): ordered_sources.remove(src) ordered_sources.insert(0, src) @@ -1848,7 +1765,7 @@ def search(plugin_name: str) -> Union[InstInfo, None]: log.debug(f'cannot search {source.type} {source.original_source}') continue if source.type in [Source.DIRECTORY, Source.LOCAL_REPO, - Source.GITHUB_REPO, Source.OTHER_URL]: + Source.REMOTE_GIT_REPO, Source.OTHER_URL]: found = _source_search(plugin_name, source) if found: log.debug(f"{found}, {found.srctype}") @@ -2130,7 +2047,7 @@ def update_plugin(plugin_name: str) -> tuple: return (None, UpdateStatus.REFUSING_UPDATE) src = InstInfo(plugin_name, - metadata['original source'], None) + metadata['original source']) if not src.get_inst_details(): log.error(f'cannot locate {plugin_name} in original source {metadata["original_source"]}') return (None, UpdateStatus.ERROR) @@ -2322,7 +2239,7 @@ def find_plugin_candidates(source: Union[LoadedSource, SourceDir], depth=2) -> l assert s.srctype == source.srctype, f'source dir {s.name}, {s.srctype} did not inherit {source.srctype} from {source.name}' assert s.parent_source == source.parent_source, f'source dir {s.name} did not inherit parent {source.parent_source} from {source.name}' - guess = InstInfo(source.name, source.location, None, source_dir=source) + guess = InstInfo(source.name, source.location, source_dir=source) guess.srctype = source.srctype manifest = None if guess.get_inst_details(): @@ -2362,11 +2279,10 @@ def available_plugins() -> list: log.debug(f'confusing source: {source.type}') continue # It takes too many API calls to query for installable plugins accurately. - if source.type == Source.GITHUB_REPO and not source.local_clone: + if source.type == Source.REMOTE_GIT_REPO and not source.local_clone: # FIXME: ignoring non-cloned repos for now. log.debug(f'cloning {source.original_source} in order to search') clone = copy_remote_git_source(InstInfo(None, - source.original_source, source.original_source, source_dir=source.content), verbose=False) @@ -2608,7 +2524,6 @@ if __name__ == '__main__': LIGHTNING_CONFIG = args.conf RECKLESS_CONFIG = load_config(reckless_dir=str(RECKLESS_DIR), network=NETWORK) - RECKLESS_SOURCES = load_sources() API_GITHUB_COM = 'https://api.github.com' GITHUB_COM = 'https://github.com' # Used for blackbox testing to avoid hitting github servers @@ -2621,6 +2536,7 @@ if __name__ == '__main__': if 'GITHUB_API_FALLBACK' in os.environ: GITHUB_API_FALLBACK = os.environ['GITHUB_API_FALLBACK'] + RECKLESS_SOURCES = load_sources() if 'targets' in args: # and len(args.targets) > 0: if args.func.__name__ == 'help_alias': log.add_result(args.func(args.targets)) From ee80ce163cf864f2628692fd9789a4541a4f69a4 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 8 Jan 2026 14:03:25 -0600 Subject: [PATCH 18/47] reckless: remove remaining traces of API access Including the canned server in the pytest infrastructure. --- .../rkls_api_lightningd_plugins.json | 47 ------------------- tests/test_reckless.py | 27 +++-------- tools/reckless | 46 +++--------------- 3 files changed, 12 insertions(+), 108 deletions(-) delete mode 100644 tests/data/recklessrepo/rkls_api_lightningd_plugins.json diff --git a/tests/data/recklessrepo/rkls_api_lightningd_plugins.json b/tests/data/recklessrepo/rkls_api_lightningd_plugins.json deleted file mode 100644 index e28e55c853e6..000000000000 --- a/tests/data/recklessrepo/rkls_api_lightningd_plugins.json +++ /dev/null @@ -1,47 +0,0 @@ -[ - { - "name": "testplugpass", - "path": "testplugpass", - "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testplugpass", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpass", - "download_url": null, - "type": "dir" - }, - { - "name": "testpluguv", - "path": "testpluguv", - "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testpluguv", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testpluguv", - "download_url": null, - "type": "dir" - }, - { - "name": "testplugfail", - "path": "testplugfail", - "url": "https://api.github.com/repos/lightningd/plugins/contents/testplugfail?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testplugfail", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugfail", - "download_url": null, - "type": "dir" - }, - { - "name": "testplugpyproj", - "path": "testplugpyproj", - "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testplugpyproj", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpyproj", - "download_url": null, - "type": "dir" - }, - { - "name": "testplugshebang", - "path": "testplugshebang", - "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master", - "html_url": "https://github.com/lightningd/plugins/tree/master/testplugshebang", - "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugshebang", - "download_url": null, - "type": "dir" - } -] diff --git a/tests/test_reckless.py b/tests/test_reckless.py index f69ff213b941..3bb1fde220f0 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -1,15 +1,13 @@ -from fixtures import * # noqa: F401,F403 -import subprocess -from pathlib import PosixPath, Path -import socket -from pyln.testing.utils import VALGRIND import json -import pytest import os +from pathlib import PosixPath, Path import re -import shutil +import subprocess import time import unittest +from fixtures import * # noqa: F401,F403 +from pyln.testing.utils import VALGRIND +import pytest @pytest.fixture(autouse=True) @@ -22,20 +20,10 @@ def canned_github_server(directory): if os.environ.get('LIGHTNING_CLI') is None: os.environ['LIGHTNING_CLI'] = str(FILE_PATH.parent / 'cli/lightning-cli') print('LIGHTNING_CALL: ', os.environ.get('LIGHTNING_CLI')) - # Use socket to provision a random free port - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind(('localhost', 0)) - free_port = str(sock.getsockname()[1]) - sock.close() global my_env my_env = os.environ.copy() - # This tells reckless to redirect to the canned server rather than github. - my_env['REDIR_GITHUB_API'] = f'http://127.0.0.1:{free_port}/api' + # This tells reckless to redirect to the local test plugins repo rather than github. my_env['REDIR_GITHUB'] = directory - my_env['FLASK_RUN_PORT'] = free_port - my_env['FLASK_APP'] = str(FILE_PATH / 'rkls_github_canned_server') - server = subprocess.Popen(["python3", "-m", "flask", "run"], - env=my_env) # Generate test plugin repository to test reckless against. repo_dir = os.path.join(directory, "lightningd") @@ -84,13 +72,10 @@ def canned_github_server(directory): del my_env['GIT_DIR'] del my_env['GIT_WORK_TREE'] del my_env['GIT_INDEX_FILE'] - # We also need the github api data for the repo which will be served via http - shutil.copyfile(str(FILE_PATH / 'data/recklessrepo/rkls_api_lightningd_plugins.json'), os.path.join(directory, 'rkls_api_lightningd_plugins.json')) yield # Delete requirements.txt from the testplugpass directory with open(requirements_file_path, 'w') as f: f.write(f"pyln-client\n\n") - server.terminate() class RecklessResult: diff --git a/tools/reckless b/tools/reckless index 0a586e701367..7bd55a6d4bd0 100755 --- a/tools/reckless +++ b/tools/reckless @@ -18,7 +18,6 @@ import types from typing import Union from urllib.parse import urlparse from urllib.request import urlopen -from urllib.error import HTTPError import venv @@ -258,7 +257,7 @@ class Source(Enum): REMOTE_GIT_REPO = 3 OTHER_URL = 4 UNKNOWN = 5 - # Cloned from remote source before searching (rather than github API) + # Cloned from remote source before searching GIT_LOCAL_CLONE = 6 @classmethod @@ -302,7 +301,7 @@ class SubmoduleSource: class LoadedSource: """Allows loading all sources only once per call of reckless. Initialized with a single line of the reckless .sources file. Keeping state also allows - minimizing API calls and refetching repositories.""" + minimizing refetching repositories.""" def __init__(self, source: str): self.original_source = source self.type = Source.get_type(source) @@ -445,16 +444,12 @@ class InstInfo: # Set recursion for how many directories deep we should search depth = 0 if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO, - Source.GIT_LOCAL_CLONE]: + Source.GIT_LOCAL_CLONE, Source.REMOTE_GIT_REPO]: depth = 5 - elif self.srctype == Source.REMOTE_GIT_REPO: - depth = 1 def search_dir(self, sub: SourceDir, subdir: bool, recursion: int) -> Union[SourceDir, None]: assert isinstance(recursion, int) - # carveout for archived plugins in lightningd/plugins. Other repos - # are only searched by API at the top level. if recursion == 0 and 'archive' in sub.name.lower(): pass # If unable to search deeper, resort to matching directory name @@ -501,27 +496,7 @@ class InstInfo: return success return None - try: - result = search_dir(self, target, False, depth) - # Using the rest API of github.com may result in a - # "Error 403: rate limit exceeded" or other access issues. - # Fall back to cloning and searching the local copy instead. - except HTTPError: - result = None - if self.srctype == Source.REMOTE_GIT_REPO: - # clone source to reckless dir - target = copy_remote_git_source(self) - if not target: - log.warning(f"could not clone github source {self}") - return False - log.debug(f"falling back to cloning remote repo {self}") - # Update to reflect use of a local clone - self.source_loc = str(target.location) - self.srctype = target.srctype - result = search_dir(self, target, False, 5) - - if not result: - return False + result = search_dir(self, target, False, depth) if result: if result != target: @@ -1202,8 +1177,7 @@ def help_alias(targets: list): def _get_local_clone(source: str) -> Union[Path, None]: - """Returns the path of a local repository clone of a github source. If one - already exists, prefer searching that to accessing the github API.""" + """Returns the path of a local repository clone of a github source.""" user, repo = Source.get_github_user_repo(source) local_clone_location = RECKLESS_DIR / '.remote_sources' / user / repo if local_clone_location.exists(): @@ -1217,8 +1191,7 @@ def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]: root_dir = src.content source = InstInfo(name, root_dir.location) - # If a local clone of a github source already exists, prefer searching - # that instead of accessing the github API. + # Remote git sources require a local clone before searching. if src.type == Source.REMOTE_GIT_REPO: if src.local_clone: if not src.local_clone_fetched: @@ -2524,18 +2497,11 @@ if __name__ == '__main__': LIGHTNING_CONFIG = args.conf RECKLESS_CONFIG = load_config(reckless_dir=str(RECKLESS_DIR), network=NETWORK) - API_GITHUB_COM = 'https://api.github.com' GITHUB_COM = 'https://github.com' # Used for blackbox testing to avoid hitting github servers - if 'REDIR_GITHUB_API' in os.environ: - API_GITHUB_COM = os.environ['REDIR_GITHUB_API'] if 'REDIR_GITHUB' in os.environ: GITHUB_COM = os.environ['REDIR_GITHUB'] - GITHUB_API_FALLBACK = False - if 'GITHUB_API_FALLBACK' in os.environ: - GITHUB_API_FALLBACK = os.environ['GITHUB_API_FALLBACK'] - RECKLESS_SOURCES = load_sources() if 'targets' in args: # and len(args.targets) > 0: if args.func.__name__ == 'help_alias': From 8f28733f3ff39553b9a8369af55570277aa454ca Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 15 Jan 2026 11:14:13 -0600 Subject: [PATCH 19/47] reckless: avoid populating uninitialized repo submodules with higher level repository contents. --- tools/reckless | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/reckless b/tools/reckless index 7bd55a6d4bd0..e20dfc9f36a2 100755 --- a/tools/reckless +++ b/tools/reckless @@ -579,11 +579,14 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return None submodules = [] for sub in proc.stdout.splitlines(): + # `git submodule status` can list higher level directory contents. + if sub.split()[1].startswith('..') or sub.split()[1].startswith('./'): + continue submodules.append(sub.split()[1]) # FIXME: Pass in tag or commit hash ver = 'HEAD' - git_call = ['git', '-C', path, 'ls-tree', '--full-tree', '-r', + git_call = ['git', '-C', path, 'ls-tree', '-r', '--name-only', ver] proc = run(git_call, stdout=PIPE, stderr=PIPE, text=True, timeout=5) if proc.returncode != 0: @@ -591,6 +594,9 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return None for filepath in proc.stdout.splitlines(): + # unfetched submodules can list the contents of the higher level repository here. + if filepath.startswith('./') or filepath.startswith('..'): + continue if filepath in submodules: if parent is None: relative_path = filepath @@ -615,7 +621,7 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return basedir.contents -def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> SourceDir: +def copy_remote_git_source(github_source: InstInfo, verbose: bool=True, parent_source=None) -> SourceDir: """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: @@ -635,7 +641,9 @@ def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> Sourc assert _git_update(github_source.source_loc, local_path) else: _git_clone(github_source, local_path, verbose) - return SourceDir(local_path, srctype=Source.GIT_LOCAL_CLONE) + local_clone = SourceDir(local_path, srctype=Source.GIT_LOCAL_CLONE, parent_source=parent_source) + local_clone.populate() + return local_clone class Config(): @@ -2258,14 +2266,14 @@ def available_plugins() -> list: clone = copy_remote_git_source(InstInfo(None, source.original_source, source_dir=source.content), - verbose=False) + verbose=False, + parent_source=source) clone.srctype = Source.GIT_LOCAL_CLONE clone.parent_source = source if not clone: log.warning(f"could not clone github source {source.original_source}") continue source.local_clone = clone - source.local_clone.parent_source = source candidates.extend(find_plugin_candidates(source)) From edb63d6998db5f8b1813f61902227ca143845d01 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 15 Jan 2026 12:14:14 -0600 Subject: [PATCH 20/47] reckless: ensure repository submodules are always fetched Changelog-Fixed: Fixes an issue where reckless would misread the contents of an uncloned repository submodule. --- tools/reckless | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tools/reckless b/tools/reckless index e20dfc9f36a2..ab0f8afa0eab 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1248,6 +1248,14 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str], verbose: bool=True) - remove_dir(str(dest)) log.error('Failed to clone repo') return False + + git = run(['git', 'submodule', 'update', '--init', '--recursive'], + cwd=str(dest), stdout=PIPE, stderr=PIPE, text=True, + check=False, timeout=120) + if git.returncode != 0: + log.warning(f'Failed to initialize submodules for {github_source}.') + return False + return True @@ -1278,7 +1286,6 @@ def _git_update(github_source: str, local_copy: PosixPath): git = run(['git', 'symbolic-ref', 'refs/remotes/origin/HEAD', '--short'], cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) - assert git.returncode == 0 if git.returncode != 0: return False default_branch = git.stdout.splitlines()[0] @@ -1290,8 +1297,16 @@ def _git_update(github_source: str, local_copy: PosixPath): git = run(['git', 'checkout', default_branch], cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) - assert git.returncode == 0 if git.returncode != 0: + log.warning(f'Failed to checkout branch {default_branch} of {github_source}.') + return False + + # Update all submodules to the referenced commit/branch/tag + git = run(['git', 'submodule', 'update', '--init', '--recursive'], + cwd=str(local_copy), stdout=PIPE, stderr=PIPE, text=True, + check=False, timeout=120) + if git.returncode != 0: + log.warning(f'Failed to initialize submodules for {github_source}.') return False return True @@ -2260,9 +2275,11 @@ def available_plugins() -> list: log.debug(f'confusing source: {source.type}') continue # It takes too many API calls to query for installable plugins accurately. - if source.type == Source.REMOTE_GIT_REPO and not source.local_clone: + if source.type == Source.REMOTE_GIT_REPO: # FIXME: ignoring non-cloned repos for now. - log.debug(f'cloning {source.original_source} in order to search') + if not source.local_clone: + log.debug(f'cloning {source.original_source} in order to search') + # Also updates existing clone and submodules clone = copy_remote_git_source(InstInfo(None, source.original_source, source_dir=source.content), From d1af4b7ea9239b0ac339d6a68e6fb886a9a0933b Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 22 Jan 2026 15:21:59 -0600 Subject: [PATCH 21/47] reckless: Add logging port to transmit log messages in real time. This will be useful for lightning-rpc so that logs can be read while waiting on reckless to return json output once complete. --- tools/reckless | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tools/reckless b/tools/reckless index ab0f8afa0eab..de85fea0349c 100755 --- a/tools/reckless +++ b/tools/reckless @@ -11,6 +11,7 @@ import logging import os from pathlib import Path, PosixPath import shutil +import socket from subprocess import Popen, PIPE, TimeoutExpired, run import tempfile import time @@ -51,11 +52,34 @@ class Logger: self.json_output = {"result": [], "log": []} self.capture = capture + self.socket = None + + def connect_socket(self, port: int): + """Streams log updates via this socket for lightningd notifications. + Used by the reckless-rpc plugin.""" + assert not self.socket + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + self.socket.connect(('localhost', port)) + except Exception as e: + logging.warning(f'socket failed to connect with {e}') + self.socket = None def str_esc(self, raw_string: str) -> str: assert isinstance(raw_string, str) return json.dumps(raw_string)[1:-1] + def push_to_socket(self, to_log: str, prefix: str): + if not self.socket or self.socket.fileno() <= 0: + return + try: + self.socket.sendall(f'{prefix}{to_log}\n'.encode('utf8')) + except Exception as e: + if self.capture: + self.json_output['log'].append(f'while feeding log to socket, encountered exception {e}') + else: + print(f'while feeding log to socket, encountered exception {e}') + def debug(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.DEBUG: @@ -65,6 +89,8 @@ class Logger: else: logging.debug(to_log) + self.push_to_socket(to_log, 'DEBUG: ') + def info(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.INFO: @@ -74,6 +100,8 @@ class Logger: else: print(to_log) + self.push_to_socket(to_log, 'INFO: ') + def warning(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.WARNING: @@ -83,6 +111,8 @@ class Logger: else: logging.warning(to_log) + self.push_to_socket(to_log, 'WARNING: ') + def error(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.ERROR: @@ -92,6 +122,8 @@ class Logger: else: logging.error(to_log) + self.push_to_socket(to_log, 'ERROR: ') + def add_result(self, result: Union[str, None]): assert json.dumps(result), "result must be json serializable" self.json_output["result"].append(result) @@ -2473,6 +2505,8 @@ if __name__ == '__main__': const=None) p.add_argument('-j', '--json', action=StoreTrueIdempotent, help='output in json format') + p.add_argument('--logging-port', action=StoreIdempotent, + help='lightning-rpc connects to this socket port to ingest log notifications') args = parser.parse_args() args = process_idempotent_args(args) @@ -2522,6 +2556,10 @@ if __name__ == '__main__': LIGHTNING_CONFIG = args.conf RECKLESS_CONFIG = load_config(reckless_dir=str(RECKLESS_DIR), network=NETWORK) + if args.logging_port: + log.connect_socket(int(args.logging_port)) + else: + log.debug('logging port argument not provided') GITHUB_COM = 'https://github.com' # Used for blackbox testing to avoid hitting github servers if 'REDIR_GITHUB' in os.environ: From 6c3031b77c03f643ddb5d6cf30dc256f8cc1e7d0 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 28 Jan 2026 10:55:14 -0600 Subject: [PATCH 22/47] reckless-rpc: open socket for listening to streaming logs --- plugins/recklessrpc.c | 110 +++++++++++++++++++++++++++++++++++++++++- tools/reckless | 7 ++- 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 794f985c94b3..5b1ae3b69ba3 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -2,6 +2,7 @@ */ #include "config.h" +#include #include #include #include @@ -10,8 +11,10 @@ #include #include #include +#include #include #include +#include #include static struct plugin *plugin; @@ -21,12 +24,17 @@ struct reckless { int stdinfd; int stdoutfd; int stderrfd; + int logfd; char *stdoutbuf; char *stderrbuf; + char *logbuf; size_t stdout_read; /* running total */ size_t stdout_new; /* new since last read */ size_t stderr_read; size_t stderr_new; + size_t log_read; + size_t log_new; + char* log_to_process; pid_t pid; char *process_failed; }; @@ -51,7 +59,7 @@ static void reckless_send_yes(struct reckless *reckless) static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) { rkls->stdout_read += rkls->stdout_new; - if (rkls->stdout_read == tal_count(rkls->stdoutbuf)) + if (rkls->stdout_read * 2 > tal_count(rkls->stdoutbuf)) tal_resize(&rkls->stdoutbuf, rkls->stdout_read * 2); return io_read_partial(conn, rkls->stdoutbuf + rkls->stdout_read, tal_count(rkls->stdoutbuf) - rkls->stdout_read, @@ -196,7 +204,7 @@ static struct io_plan *stderr_read_more(struct io_conn *conn, struct reckless *rkls) { rkls->stderr_read += rkls->stderr_new; - if (rkls->stderr_read == tal_count(rkls->stderrbuf)) + if (rkls->stderr_read * 2 > tal_count(rkls->stderrbuf)) tal_resize(&rkls->stderrbuf, rkls->stderr_read * 2); if (strends(rkls->stderrbuf, "[Y] to create one now.\n")) { plugin_log(plugin, LOG_DBG, "confirming config creation"); @@ -233,6 +241,82 @@ static bool is_single_arg_cmd(const char *command) { return false; } +static void log_conn_finish(struct io_conn *conn, struct reckless *reckless) +{ + io_close(conn); + close(reckless->logfd); + +} + +static struct io_plan *log_read_more(struct io_conn *conn, + struct reckless *rkls) +{ + rkls->log_read += rkls->log_new; + + if (rkls->log_read*2 >= tal_count(rkls->logbuf)) + tal_resize(&rkls->logbuf, rkls->log_read * 2); + + int unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); + char *lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); + + while (lineend != NULL) { + char * note; + note = tal_strndup(tmpctx, rkls->log_to_process, + lineend - rkls->log_to_process); + /* FIXME: Add notification for the utility logs. */ + plugin_log(plugin, LOG_DBG, "RECKLESS UTILITY: %s", note); + rkls->log_to_process = lineend + 1; + unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); + lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); + } + + return io_read_partial(conn, rkls->logbuf + rkls->log_read, + tal_count(rkls->logbuf) - rkls->log_read, + &rkls->log_new, log_read_more, rkls); +} + +static struct io_plan *log_conn_init(struct io_conn *conn, struct reckless *rkls) +{ + io_set_finish(conn, log_conn_finish, rkls); + return log_read_more(conn, rkls); +} + +static int open_socket(int *port) +{ + int sock; + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { + plugin_log(plugin, LOG_UNUSUAL, "could not open socket for " + "streaming logs"); + return -1; + } + struct sockaddr_in ai; + ai.sin_family = AF_INET; + ai.sin_port = htons(0); + inet_pton(AF_INET, "127.0.0.1", &ai.sin_addr); + + if (bind(sock, (struct sockaddr *)&ai, sizeof(ai)) < 0) { + plugin_log(plugin, LOG_UNUSUAL, "failed to bind socket: %s", strerror(errno)); + close(sock); + return -1; + } + + socklen_t len = sizeof(ai); + if (getsockname(sock, (struct sockaddr *)&ai, &len) < 0) { + plugin_log(plugin, LOG_DBG, "couldn't retrieve socket port"); + return -1; + } + *port = ntohs(ai.sin_port); + + if (listen(sock, 64) != 0) { + plugin_log(plugin, LOG_UNUSUAL, "failed to listen on socket: %s", strerror(errno)); + close(sock); + return -1; + } + + return sock; +} + static struct command_result *reckless_call(struct command *cmd, const char *subcommand, const char *target, @@ -242,6 +326,13 @@ static struct command_result *reckless_call(struct command *cmd, if (!subcommand || !target) return command_fail(cmd, PLUGIN_ERROR, "invalid reckless call"); } + int sock; + int *port = tal(tmpctx, int); + sock = open_socket(port); + if (sock < 0) + plugin_log(plugin, LOG_BROKEN, "not streaming logs " + "from reckless utility"); + char **my_call; my_call = tal_arrz(tmpctx, char *, 0); tal_arr_expand(&my_call, "reckless"); @@ -251,6 +342,11 @@ static struct command_result *reckless_call(struct command *cmd, tal_arr_expand(&my_call, lconfig.lightningdir); tal_arr_expand(&my_call, "--network"); tal_arr_expand(&my_call, lconfig.network); + if (sock > 0) { + tal_arr_expand(&my_call, "--logging-port"); + tal_arr_expand(&my_call, tal_fmt(tmpctx, "%i", *port)); + } + if (lconfig.config) { tal_arr_expand(&my_call, "--conf"); tal_arr_expand(&my_call, lconfig.config); @@ -266,11 +362,17 @@ static struct command_result *reckless_call(struct command *cmd, reckless->cmd = cmd; reckless->stdoutbuf = tal_arrz(reckless, char, 4096); reckless->stderrbuf = tal_arrz(reckless, char, 4096); + reckless->logbuf = tal_arrz(reckless, char, 4096); reckless->stdout_read = 0; reckless->stdout_new = 0; reckless->stderr_read = 0; reckless->stderr_new = 0; + reckless->log_read = 0; + reckless->log_new = 0; + reckless->log_to_process = reckless->logbuf; reckless->process_failed = NULL; + reckless->logfd = sock; + char * full_cmd; full_cmd = tal_fmt(tmpctx, "calling:"); for (int i=0; ipid = pipecmdarr(&reckless->stdinfd, &reckless->stdoutfd, &reckless->stderrfd, my_call); + if (sock > 0) + io_new_listener(reckless, reckless->logfd, + log_conn_init, reckless); /* FIXME: fail if invalid pid*/ io_new_conn(reckless, reckless->stdoutfd, conn_init, reckless); io_new_conn(reckless, reckless->stderrfd, stderr_conn_init, reckless); + tal_free(my_call); return command_still_pending(cmd); } diff --git a/tools/reckless b/tools/reckless index de85fea0349c..f97eb068722e 100755 --- a/tools/reckless +++ b/tools/reckless @@ -62,8 +62,13 @@ class Logger: try: self.socket.connect(('localhost', port)) except Exception as e: - logging.warning(f'socket failed to connect with {e}') self.socket = None + if logging.root.level <= logging.WARNING: + msg = f'socket failed to connect with {e}' + if self.capture: + self.json_output['log'].append(self.str_esc(msg)) + else: + logging.warning(msg) def str_esc(self, raw_string: str) -> str: assert isinstance(raw_string, str) From 1f0bf01578b794ad819b9369cd3d4f9a8dfeb636 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 28 Jan 2026 15:40:47 -0600 Subject: [PATCH 23/47] reckless-rpc: publish reckless log notifications Changelog-added: The reckless-rpc plugin streams logs via the notification topic 'reckless_log' --- plugins/recklessrpc.c | 17 ++++++++++++++--- tests/plugins/custom_notifications.py | 5 +++++ tests/test_reckless.py | 17 +++++++++++++++-- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 5b1ae3b69ba3..9a72b15829bc 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -241,6 +241,13 @@ static bool is_single_arg_cmd(const char *command) { return false; } +static void log_notify(char * log_line TAKES) +{ + struct json_stream *js = plugin_notification_start(NULL, "reckless_log"); + json_add_stringn(js, "log", log_line, tal_count(log_line)); + plugin_notification_end(plugin, js); +} + static void log_conn_finish(struct io_conn *conn, struct reckless *reckless) { io_close(conn); @@ -263,8 +270,8 @@ static struct io_plan *log_read_more(struct io_conn *conn, char * note; note = tal_strndup(tmpctx, rkls->log_to_process, lineend - rkls->log_to_process); - /* FIXME: Add notification for the utility logs. */ - plugin_log(plugin, LOG_DBG, "RECKLESS UTILITY: %s", note); + plugin_log(plugin, LOG_DBG, "reckless utility: %s", note); + log_notify(note); rkls->log_to_process = lineend + 1; unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); @@ -444,6 +451,10 @@ static const struct plugin_command commands[] = { }, }; +static const char *notifications[] = { + "reckless_log", +}; + int main(int argc, char **argv) { setup_locale(); @@ -453,7 +464,7 @@ int main(int argc, char **argv) commands, ARRAY_SIZE(commands), NULL, 0, /* Notifications */ NULL, 0, /* Hooks */ - NULL, 0, /* Notification topics */ + notifications, ARRAY_SIZE(notifications), /* Notification topics */ NULL); /* plugin options */ return 0; diff --git a/tests/plugins/custom_notifications.py b/tests/plugins/custom_notifications.py index 1a3d92f18fc7..7ac27f763423 100755 --- a/tests/plugins/custom_notifications.py +++ b/tests/plugins/custom_notifications.py @@ -51,5 +51,10 @@ def on_faulty_emit(origin, payload, **kwargs): plugin.log("Got the ididntannouncethis event") +@plugin.subscribe("reckless_log") +def on_reckless_log(origin, **kwargs): + plugin.log("Got reckless_log: {}".format(kwargs)) + + plugin.add_notification_topic("custom") plugin.run() diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 3bb1fde220f0..ce00e3f295d9 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -140,10 +140,10 @@ def reckless(cmds: list, dir: PosixPath = None, return RecklessResult(r, r.returncode, stdout, stderr) -def get_reckless_node(node_factory): +def get_reckless_node(node_factory, options={}, start=False): '''This may be unnecessary, but a preconfigured lightning dir is useful for reckless testing.''' - node = node_factory.get_node(options={}, start=False) + node = node_factory.get_node(options=options, start=start) return node @@ -461,3 +461,16 @@ def test_reckless_available(node_factory): assert r.search_stdout('testplugpass') assert r.search_stdout('testplugpyproj') assert r.search_stdout('testpluguv') + + +def test_reckless_notifications(node_factory): + """Reckless streams logs to the reckless-rpc plugin which are emitted + as 'reckless_log' notifications""" + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + node.start() + listconfig_log = node.rpc.reckless('listconfig')['log'] + # Some trouble escaping the clone url for searching + listconfig_log.pop(1) + for log in listconfig_log: + assert node.daemon.is_in_log(f"reckless_log: {{'reckless_log': {{'log': '{log}'", start=0) From 18efa06b986b62d4cc222fbe373136c4c0686594 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Fri, 30 Jan 2026 16:55:44 -0600 Subject: [PATCH 24/47] reckless: close log socket before stdout Otherwise reckless-rpc can be concerned that the reckless utility process didn't exit cleanly. --- plugins/recklessrpc.c | 14 +++++++++++++- tests/test_reckless.py | 7 ++++++- tools/reckless | 3 +++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 9a72b15829bc..c3337588e895 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -69,6 +69,7 @@ static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) static struct command_result *reckless_result(struct io_conn *conn, struct reckless *reckless) { + io_close(conn); struct json_stream *response; if (reckless->process_failed) { response = jsonrpc_stream_fail(reckless->cmd, @@ -152,6 +153,16 @@ static void reckless_conn_finish(struct io_conn *conn, /* FIXME: avoid EBADFD - leave stdin fd open? */ if (errno && errno != 9) plugin_log(plugin, LOG_DBG, "err: %s", strerror(errno)); + struct pollfd pfd = { .fd = reckless->logfd, .events = POLLIN }; + poll(&pfd, 1, 20); // wait for any remaining log data + + /* Close the log streaming socket. */ + if (reckless->logfd) { + if (close(reckless->logfd) != 0) + plugin_log(plugin, LOG_DBG, "closing log socket failed: %s", strerror(errno)); + reckless->logfd = 0; + } + if (reckless->pid > 0) { int status = 0; pid_t p; @@ -160,6 +171,7 @@ static void reckless_conn_finish(struct io_conn *conn, if (p != reckless->pid && reckless->pid) { plugin_log(plugin, LOG_DBG, "reckless failed to exit, " "killing now."); + io_close(conn); kill(reckless->pid, SIGKILL); reckless_fail(reckless, "reckless process hung"); /* Reckless process exited and with normal status? */ @@ -251,7 +263,7 @@ static void log_notify(char * log_line TAKES) static void log_conn_finish(struct io_conn *conn, struct reckless *reckless) { io_close(conn); - close(reckless->logfd); + reckless->logfd = 0; } diff --git a/tests/test_reckless.py b/tests/test_reckless.py index ce00e3f295d9..bff510e6f418 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -419,7 +419,7 @@ def test_tag_install(node_factory): # Note: uv timeouts from the GH network seem to happen? @pytest.mark.slow_test -@pytest.mark.flaky(reruns=3) +@pytest.mark.flaky(max_runs=3) def test_reckless_uv_install(node_factory): node = get_reckless_node(node_factory) node.start() @@ -468,6 +468,11 @@ def test_reckless_notifications(node_factory): as 'reckless_log' notifications""" notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + NETWORK = os.environ.get('TEST_NETWORK') + if not NETWORK: + NETWORK = 'regtest' + reckless(['listconfig', f'--network={NETWORK}', '--json'], + dir=node.lightning_dir) node.start() listconfig_log = node.rpc.reckless('listconfig')['log'] # Some trouble escaping the clone url for searching diff --git a/tools/reckless b/tools/reckless index f97eb068722e..7b7267fa6a1a 100755 --- a/tools/reckless +++ b/tools/reckless @@ -2597,3 +2597,6 @@ if __name__ == '__main__': if log.capture: log.reply_json() + # We're done streaming to this socket, but the rpc plugin will close it. + if log.socket: + log.socket.shutdown(socket.SHUT_WR) From d4a514770f153b9a3a389f51de524b93c8c5f41e Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Feb 2026 10:14:26 +1030 Subject: [PATCH 25/47] plugins/recklessrpc: use membuf for handling socket input. This is the standard battle-tested way of doing it, and it avoids the various bugs in the open-coded implementation. Inspired by common/jsonrpc_io.c which does a similar thing for JSON. Signed-off-by: Rusty Russell --- plugins/recklessrpc.c | 75 +++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index c3337588e895..e4a92f05b259 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -27,16 +28,16 @@ struct reckless { int logfd; char *stdoutbuf; char *stderrbuf; - char *logbuf; size_t stdout_read; /* running total */ size_t stdout_new; /* new since last read */ size_t stderr_read; size_t stderr_new; - size_t log_read; - size_t log_new; - char* log_to_process; pid_t pid; char *process_failed; + + MEMBUF(char) logbuf; + /* Amount just read by io_read_partial */ + size_t logbytes_read; }; struct lconfig { @@ -253,10 +254,10 @@ static bool is_single_arg_cmd(const char *command) { return false; } -static void log_notify(char * log_line TAKES) +static void log_notify(const char *log_line, size_t len) { struct json_stream *js = plugin_notification_start(NULL, "reckless_log"); - json_add_stringn(js, "log", log_line, tal_count(log_line)); + json_add_stringn(js, "log", log_line, len); plugin_notification_end(plugin, js); } @@ -267,31 +268,43 @@ static void log_conn_finish(struct io_conn *conn, struct reckless *reckless) } +/* len does NOT include the \n */ +static const char *get_line(const struct reckless *rkls, size_t *len) +{ + const char *line = membuf_elems(&rkls->logbuf); + const char *eol = memchr(line, '\n', membuf_num_elems(&rkls->logbuf)); + + if (eol) { + *len = eol - line; + return line; + } + return NULL; +} + static struct io_plan *log_read_more(struct io_conn *conn, - struct reckless *rkls) + struct reckless *rkls) { - rkls->log_read += rkls->log_new; - - if (rkls->log_read*2 >= tal_count(rkls->logbuf)) - tal_resize(&rkls->logbuf, rkls->log_read * 2); - - int unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); - char *lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); - - while (lineend != NULL) { - char * note; - note = tal_strndup(tmpctx, rkls->log_to_process, - lineend - rkls->log_to_process); - plugin_log(plugin, LOG_DBG, "reckless utility: %s", note); - log_notify(note); - rkls->log_to_process = lineend + 1; - unprocessed = rkls->log_read - (rkls->log_to_process - rkls->logbuf); - lineend = memchr(rkls->log_to_process, 0x0A, unprocessed); + size_t len; + const char *line; + + /* We read some more stuff in! */ + membuf_added(&rkls->logbuf, rkls->logbytes_read); + rkls->logbytes_read = 0; + + while ((line = get_line(rkls, &len)) != NULL) { + plugin_log(plugin, LOG_DBG, "reckless utility: %.*s", (int)len, line); + log_notify(line, len); + membuf_consume(&rkls->logbuf, len + 1); } - return io_read_partial(conn, rkls->logbuf + rkls->log_read, - tal_count(rkls->logbuf) - rkls->log_read, - &rkls->log_new, log_read_more, rkls); + /* Make sure there's more room */ + membuf_prepare_space(&rkls->logbuf, 4096); + + return io_read_partial(conn, + membuf_space(&rkls->logbuf), + membuf_num_space(&rkls->logbuf), + &rkls->logbytes_read, + log_read_more, rkls); } static struct io_plan *log_conn_init(struct io_conn *conn, struct reckless *rkls) @@ -381,16 +394,16 @@ static struct command_result *reckless_call(struct command *cmd, reckless->cmd = cmd; reckless->stdoutbuf = tal_arrz(reckless, char, 4096); reckless->stderrbuf = tal_arrz(reckless, char, 4096); - reckless->logbuf = tal_arrz(reckless, char, 4096); reckless->stdout_read = 0; reckless->stdout_new = 0; reckless->stderr_read = 0; reckless->stderr_new = 0; - reckless->log_read = 0; - reckless->log_new = 0; - reckless->log_to_process = reckless->logbuf; reckless->process_failed = NULL; reckless->logfd = sock; + membuf_init(&reckless->logbuf, + tal_arr(reckless, char, 10), + 10, membuf_tal_resize); + reckless->logbytes_read = 0; char * full_cmd; full_cmd = tal_fmt(tmpctx, "calling:"); From c312e282e109ab8182da8bb16e9c15e45aa09ad4 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 11 Feb 2026 09:25:19 -0600 Subject: [PATCH 26/47] reckless: json escape streaming logs --- tools/reckless | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/reckless b/tools/reckless index 7b7267fa6a1a..f7f21d5cd13b 100755 --- a/tools/reckless +++ b/tools/reckless @@ -71,7 +71,9 @@ class Logger: logging.warning(msg) def str_esc(self, raw_string: str) -> str: - assert isinstance(raw_string, str) + assert isinstance(raw_string, str) or hasattr(to_log, "__repr__") + if not isinstance(raw_string, str): + return json.dumps(str(raw_string))[1:-1] return json.dumps(raw_string)[1:-1] def push_to_socket(self, to_log: str, prefix: str): @@ -94,7 +96,7 @@ class Logger: else: logging.debug(to_log) - self.push_to_socket(to_log, 'DEBUG: ') + self.push_to_socket(self.str_esc(to_log), 'DEBUG: ') def info(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") @@ -105,7 +107,7 @@ class Logger: else: print(to_log) - self.push_to_socket(to_log, 'INFO: ') + self.push_to_socket(self.str_esc(to_log), 'INFO: ') def warning(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") @@ -116,7 +118,7 @@ class Logger: else: logging.warning(to_log) - self.push_to_socket(to_log, 'WARNING: ') + self.push_to_socket(self.str_esc(to_log), 'WARNING: ') def error(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") @@ -127,7 +129,7 @@ class Logger: else: logging.error(to_log) - self.push_to_socket(to_log, 'ERROR: ') + self.push_to_socket(self.str_esc(to_log), 'ERROR: ') def add_result(self, result: Union[str, None]): assert json.dumps(result), "result must be json serializable" @@ -1531,7 +1533,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # Create symlink in staging tree to redirect to the plugins entrypoint log.debug(f"linking source {staging_path / cloned_src.entry} to " f"{Path(staged_src.source_loc) / cloned_src.entry}") - log.debug(staged_src) + log.debug(str(staged_src)) (Path(staged_src.source_loc) / cloned_src.entry).\ symlink_to(staging_path / cloned_src.entry) @@ -1884,7 +1886,7 @@ def enable(plugin_name: str): return None else: log.error(f'reckless: {inst.name} failed to start!') - log.error(err) + log.error(str(err)) return None except RPCError: log.info(('lightningd rpc unavailable. ' From b941f67d9155714fb2c465a230b06547bd7825ca Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 11 Feb 2026 09:27:23 -0600 Subject: [PATCH 27/47] reklessrpc: duplicate json format of reckless listavailable output --- plugins/recklessrpc.c | 62 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index e4a92f05b259..38e3fead0e06 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -67,6 +67,51 @@ static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) &rkls->stdout_new, read_more, rkls); } +static void dup_listavailable_result(struct reckless *reckless, + struct json_stream *response, + char *reckless_result, + const jsmntok_t *results_tok) +{ + json_array_start(response, "result"); + size_t plugins, requirements; + const jsmntok_t *result, *requirement, *requirements_tok; + const char *plugin_name, *short_description, *long_description, *entrypoint; + + json_for_each_arr(plugins, result, results_tok) { + json_object_start(response, NULL); + + json_scan(tmpctx, reckless_result, result, + "{name:%," + "short_description:%," + "long_description:%," + "entrypoint:%}", + JSON_SCAN_TAL(tmpctx, json_strdup, &plugin_name), + JSON_SCAN_TAL(tmpctx, json_strdup, &short_description), + JSON_SCAN_TAL(tmpctx, json_strdup, &long_description), + JSON_SCAN_TAL(tmpctx, json_strdup, &entrypoint)); + + json_add_string(response, "name", plugin_name); + if (!streq(short_description, "null")) + json_add_string(response, "short_description", short_description); + if (!streq(long_description, "null")) + json_add_string(response, "long_description", long_description); + json_add_string(response, "entypoint", entrypoint); + + json_array_start(response, "requirements"); + requirements_tok = json_get_member(reckless_result, result, "requirements"); + if (requirements_tok) { + json_for_each_arr(requirements, requirement, requirements_tok) { + json_add_string(response, NULL, + json_strdup(tmpctx, reckless_result, requirement)); + } + } + json_array_end(response); + + json_object_end(response); + } + json_array_end(response); +} + static struct command_result *reckless_result(struct io_conn *conn, struct reckless *reckless) { @@ -78,7 +123,10 @@ static struct command_result *reckless_result(struct io_conn *conn, reckless->process_failed); return command_finished(reckless->cmd, response); } - const jsmntok_t *results, *result, *logs, *log, *conf; + + /* The reckless utility outputs utf-8 and ends the transmission with + * \u0004, which jsmn is unable to parse. */ + const jsmntok_t *results, *result, *logs, *log, *conf, *next; size_t i; jsmn_parser parser; jsmntok_t *toks; @@ -90,7 +138,7 @@ static struct command_result *reckless_result(struct io_conn *conn, const char *err; if (res == JSMN_ERROR_INVAL) err = tal_fmt(tmpctx, "reckless returned invalid character in json " - "output"); + "output. (total length %lu)", strlen(reckless->stdoutbuf)); else if (res == JSMN_ERROR_PART) err = tal_fmt(tmpctx, "reckless returned partial output"); else if (res == JSMN_ERROR_NOMEM ) @@ -100,7 +148,8 @@ static struct command_result *reckless_result(struct io_conn *conn, err = NULL; if (err) { - plugin_log(plugin, LOG_UNUSUAL, "failed to parse json: %s", err); + if (res == JSMN_ERROR_INVAL) + plugin_log(plugin, LOG_BROKEN, "invalid char in json"); response = jsonrpc_stream_fail(reckless->cmd, PLUGIN_ERROR, err); return command_finished(reckless->cmd, response); @@ -108,15 +157,20 @@ static struct command_result *reckless_result(struct io_conn *conn, response = jsonrpc_stream_success(reckless->cmd); results = json_get_member(reckless->stdoutbuf, toks, "result"); + next = json_get_arr(results, 0); conf = json_get_member(reckless->stdoutbuf, results, "requested_lightning_conf"); if (conf) { - plugin_log(plugin, LOG_DBG, "dealing with listconfigs output"); + plugin_log(plugin, LOG_DBG, "ingesting listconfigs output"); json_object_start(response, "result"); json_for_each_obj(i, result, results) { json_add_tok(response, json_strdup(tmpctx, reckless->stdoutbuf, result), result+1, reckless->stdoutbuf); } json_object_end(response); + } else if (next && next->type == JSMN_OBJECT) { + plugin_log(plugin, LOG_DBG, "ingesting listavailable output"); + dup_listavailable_result(reckless, response, reckless->stdoutbuf, results); + } else { json_array_start(response, "result"); json_for_each_arr(i, result, results) { From de72ad6d139424fed144364132216a1db9dc26de Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 12 Feb 2026 11:27:05 -0600 Subject: [PATCH 28/47] recklessrpc: fixup close connection --- plugins/recklessrpc.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 38e3fead0e06..c792251ccb88 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -112,10 +112,8 @@ static void dup_listavailable_result(struct reckless *reckless, json_array_end(response); } -static struct command_result *reckless_result(struct io_conn *conn, - struct reckless *reckless) +static struct command_result *reckless_result(struct reckless *reckless) { - io_close(conn); struct json_stream *response; if (reckless->process_failed) { response = jsonrpc_stream_fail(reckless->cmd, @@ -205,6 +203,7 @@ static struct command_result *reckless_fail(struct reckless *reckless, static void reckless_conn_finish(struct io_conn *conn, struct reckless *reckless) { + io_close(conn); /* FIXME: avoid EBADFD - leave stdin fd open? */ if (errno && errno != 9) plugin_log(plugin, LOG_DBG, "err: %s", strerror(errno)); @@ -226,15 +225,13 @@ static void reckless_conn_finish(struct io_conn *conn, if (p != reckless->pid && reckless->pid) { plugin_log(plugin, LOG_DBG, "reckless failed to exit, " "killing now."); - io_close(conn); kill(reckless->pid, SIGKILL); reckless_fail(reckless, "reckless process hung"); /* Reckless process exited and with normal status? */ } else if (WIFEXITED(status) && !WEXITSTATUS(status)) { plugin_log(plugin, LOG_DBG, - "Reckless subprocess complete: %s", - reckless->stdoutbuf); - reckless_result(conn, reckless); + "Reckless subprocess complete"); + reckless_result(reckless); /* Don't try to process json if python raised an error. */ } else { plugin_log(plugin, LOG_DBG, "%s", reckless->stderrbuf); @@ -252,7 +249,6 @@ static void reckless_conn_finish(struct io_conn *conn, "The reckless subprocess has failed."); } } - io_close(conn); tal_free(reckless); } From 940f9c80b17bf248cd7c1054f9a01950cfc34a8f Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 11 Feb 2026 16:14:06 -0600 Subject: [PATCH 29/47] recklessrpc: fix buffer resize --- plugins/recklessrpc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index c792251ccb88..9c1552edf4ba 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -60,8 +60,8 @@ static void reckless_send_yes(struct reckless *reckless) static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) { rkls->stdout_read += rkls->stdout_new; - if (rkls->stdout_read * 2 > tal_count(rkls->stdoutbuf)) - tal_resize(&rkls->stdoutbuf, rkls->stdout_read * 2); + while (rkls->stdout_read >= tal_count(rkls->stdoutbuf)) + tal_resizez(&rkls->stdoutbuf, tal_count(rkls->stdoutbuf) * 2); return io_read_partial(conn, rkls->stdoutbuf + rkls->stdout_read, tal_count(rkls->stdoutbuf) - rkls->stdout_read, &rkls->stdout_new, read_more, rkls); From e394e68006d6857b0b3bfb85aec2b43db00680cb Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 12 Feb 2026 11:41:04 -0600 Subject: [PATCH 30/47] pytest: skip reckless uv install under valgrind --- tests/test_reckless.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index bff510e6f418..099c715dbc41 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -292,6 +292,7 @@ def test_install(node_factory): assert r.returncode == 0 +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") def test_install_cleanup(node_factory): """test failed installation and post install cleanup""" n = get_reckless_node(node_factory) @@ -419,6 +420,7 @@ def test_tag_install(node_factory): # Note: uv timeouts from the GH network seem to happen? @pytest.mark.slow_test +@unittest.skipIf(VALGRIND, "node too slow for starting plugin under valgrind") @pytest.mark.flaky(max_runs=3) def test_reckless_uv_install(node_factory): node = get_reckless_node(node_factory) From b53e73f126748ef3589f668b08f8b553862dbf77 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Fri, 12 Dec 2025 10:47:54 -0600 Subject: [PATCH 31/47] reckless: don't be confused by python deps in rust plugins Python dependencies are often used for the test framework. Checking other installers first means they're less likely to be misinterpretted as python plugins. --- tools/reckless | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/reckless b/tools/reckless index f7f21d5cd13b..55ffe39269c6 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1204,8 +1204,10 @@ shebang.add_entrypoint('{name}') # An extra installable check to see if a #! is present in the file shebang.check = check_for_shebang -INSTALLERS = [shebang, pythonuv, pythonuvlegacy, python3venv, poetryvenv, - pyprojectViaPip, nodejs, rust_cargo] +# Projects may include python dependencies for testing, so give other installers +# first priority. +INSTALLERS = [rust_cargo, nodejs, shebang, pythonuv, pythonuvlegacy, python3venv, + poetryvenv, pyprojectViaPip] def help_alias(targets: list): From 0a9bcfb3dda1b97da5acb6678258fc0ebe7a1adf Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 15 Dec 2025 17:26:29 -0600 Subject: [PATCH 32/47] reckless: catch and raise usage errors While the reckless utility provided usage hints, the rpc plugin would simply exit with an unhelpful: { "code": -3, "message": "the reckless process has crashed" } This captures the usage hint from the utility and reports it from the plugin as well. Changelog-Fixed: reckless-rpc plugin now raises incorrect usage from the reckless utility. --- plugins/recklessrpc.c | 37 +++++++++++++++++++++++++++++-------- tests/test_reckless.py | 19 +++++++++++++++++++ 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 9c1552edf4ba..0b83a6d34f40 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -200,6 +200,23 @@ static struct command_result *reckless_fail(struct reckless *reckless, return command_finished(reckless->cmd, resp); } +/* Regurgitates the syntax error reported by the utility */ +static struct command_result *fail_bad_usage(struct reckless *reckless) +{ + char **lines; + lines = tal_strsplit(reckless, reckless->stderrbuf, "\n", STR_EMPTY_OK); + if (lines != NULL) + { + /* The last line of reckless output contains the usage error. + * Capture it for the user. */ + int i = 0; + while (lines[i + 1] != NULL) + i++; + return reckless_fail(reckless, lines[i]); + } + return reckless_fail(reckless, "the reckless process has crashed"); +} + static void reckless_conn_finish(struct io_conn *conn, struct reckless *reckless) { @@ -239,14 +256,18 @@ static void reckless_conn_finish(struct io_conn *conn, "Reckless process has crashed (%i).", WEXITSTATUS(status)); char * err; - if (reckless->process_failed) - err = reckless->process_failed; - else - err = tal_strdup(tmpctx, "the reckless process " - "has crashed"); - reckless_fail(reckless, err); - plugin_log(plugin, LOG_UNUSUAL, - "The reckless subprocess has failed."); + if (WEXITSTATUS(status) == 2) + fail_bad_usage(reckless); + else { + if (reckless->process_failed) + err = reckless->process_failed; + else + err = tal_strdup(tmpctx, "the reckless process " + "has crashed"); + reckless_fail(reckless, err); + plugin_log(plugin, LOG_UNUSUAL, + "The reckless subprocess has failed."); + } } } tal_free(reckless); diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 099c715dbc41..0fc9d2fa35c1 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -6,6 +6,7 @@ import time import unittest from fixtures import * # noqa: F401,F403 +from pyln.client import lightning from pyln.testing.utils import VALGRIND import pytest @@ -481,3 +482,21 @@ def test_reckless_notifications(node_factory): listconfig_log.pop(1) for log in listconfig_log: assert node.daemon.is_in_log(f"reckless_log: {{'reckless_log': {{'log': '{log}'", start=0) + + +def test_reckless_usage(node_factory): + """The reckless rpc response is more useful if it can pass back incorrect + usage errors.""" + node = node_factory.get_node(options={}, may_fail=True, start=False) + node.start(stderr_redir=True) + r = reckless(['searhc', 'testplugpass'], + dir=node.lightning_dir) + # The reckless utility should fail and argparse should provide a usage hint + # as the line of output. + assert r.returncode == 2 + assert "reckless: error: argument cmd1: invalid choice: 'searhc' (choose from " in r.stderr[-1] + + # The rpc plugin should capture and raise this usage error + with pytest.raises(lightning.RpcError, + match="reckless: error: argument cmd1: invalid choice: 'saerch'"): + node.rpc.reckless('saerch', 'testplugpass') From 3d45626215057cc4052987e19390d2391fe262f8 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 12 Feb 2026 14:52:33 -0600 Subject: [PATCH 33/47] pytest: add timeout to reckless tests Trying to debug pytest teardown under the github CI runner. --- tests/test_reckless.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 0fc9d2fa35c1..bc5b0c765b2a 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -157,6 +157,7 @@ def test_basic_help(): assert r.search_stdout("options:") or r.search_stdout("optional arguments:") +@pytest.mark.timeout(120) def test_reckless_version_listconfig(node_factory): '''Version should be reported without loading config and should advance with lightningd.''' @@ -466,6 +467,7 @@ def test_reckless_available(node_factory): assert r.search_stdout('testpluguv') +@pytest.mark.timeout(120) def test_reckless_notifications(node_factory): """Reckless streams logs to the reckless-rpc plugin which are emitted as 'reckless_log' notifications""" From 050c9c531af4816b14dc3cb086ee9d0604f25c28 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 12 Feb 2026 16:22:14 -0600 Subject: [PATCH 34/47] reckless: update json schema with listavailable, listconfig, listinstalled commands --- contrib/msggen/msggen/schema.json | 3 +++ doc/reckless.7.md | 2 +- doc/schemas/reckless.json | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json index 8e5862952203..319c00df483c 100644 --- a/contrib/msggen/msggen/schema.json +++ b/contrib/msggen/msggen/schema.json @@ -29992,6 +29992,9 @@ "enable", "disable", "source", + "listavailable", + "listconfig", + "listinstalled", "--version" ], "description": [ diff --git a/doc/reckless.7.md b/doc/reckless.7.md index acf5f4cd40d9..39f231ae47e2 100644 --- a/doc/reckless.7.md +++ b/doc/reckless.7.md @@ -11,7 +11,7 @@ DESCRIPTION The **reckless** RPC starts a reckless process with the *command* and *target* provided. Node configuration, network, and lightning direrctory are automatically passed to the reckless utility. -- **command** (string) (one of "install", "uninstall", "search", "enable", "disable", "source", "--version"): Determines which command to pass to reckless +- **command** (string) (one of "install", "uninstall", "search", "enable", "disable", "source", "listavailable", "listconfig", "listinstalled", "--version"): Determines which command to pass to reckless - *command* **install** takes a *plugin\_name* to search for and install a named plugin. - *command* **uninstall** takes a *plugin\_name* and attempts to uninstall a plugin of the same name. - *command* **search** takes a *plugin\_name* to search for a named plugin. diff --git a/doc/schemas/reckless.json b/doc/schemas/reckless.json index 372eb772dd3e..27bf2f8f5419 100644 --- a/doc/schemas/reckless.json +++ b/doc/schemas/reckless.json @@ -21,6 +21,9 @@ "enable", "disable", "source", + "listavailable", + "listconfig", + "listinstalled", "--version" ], "description": [ From 27e34108ec7a463d69f7acde20e025dcc795aaad Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Tue, 10 Feb 2026 10:14:45 +0100 Subject: [PATCH 35/47] reckless: fail reckless rpc if executable fails If the execution of reckless fails for some reason recklessrpc plugin should return an rpc failure message instead of hanging. Changelog-None. Signed-off-by: Lagrang3 --- plugins/recklessrpc.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 0b83a6d34f40..e62077dd8696 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -486,10 +486,14 @@ static struct command_result *reckless_call(struct command *cmd, reckless->pid = pipecmdarr(&reckless->stdinfd, &reckless->stdoutfd, &reckless->stderrfd, my_call); + if (reckless->pid < 0) { + return command_fail(cmd, LIGHTNINGD, "reckless failed: %s", + strerror(errno)); + } + if (sock > 0) io_new_listener(reckless, reckless->logfd, log_conn_init, reckless); - /* FIXME: fail if invalid pid*/ io_new_conn(reckless, reckless->stdoutfd, conn_init, reckless); io_new_conn(reckless, reckless->stderrfd, stderr_conn_init, reckless); From 0881bf506a8725cf482ad0921a38163ba4a1fcf1 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Fri, 13 Feb 2026 10:30:05 -0600 Subject: [PATCH 36/47] pytest: make sure tools/reckless is available on PATH so that reckless-rpc will use the correct executable. --- tests/test_reckless.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index bc5b0c765b2a..71ef7a2a2be6 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -2,6 +2,7 @@ import os from pathlib import PosixPath, Path import re +import shutil import subprocess import time import unittest @@ -79,6 +80,16 @@ def canned_github_server(directory): f.write(f"pyln-client\n\n") +@pytest.fixture(autouse=True) +def add_reckless_to_env_path(): + """Allows the reckless-rpc plugin to use the reckless executable from the + build directory, rather than whatever it finds already installed.""" + current_path = os.environ['PATH'] + tools_dir = Path(os.path.dirname(os.path.realpath(__file__))).parent / 'tools' + os.environ['PATH'] = f'{tools_dir}:{current_path}' + assert shutil.which('reckless') + + class RecklessResult: def __init__(self, process, returncode, stdout, stderr): self.process = process From 9eafa0c07d8defa94473f73af7b4a6242c302f43 Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 10:22:29 +0100 Subject: [PATCH 37/47] rebased --- tools/reckless | 99 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 91 insertions(+), 8 deletions(-) diff --git a/tools/reckless b/tools/reckless index 55ffe39269c6..36f3912b2458 100755 --- a/tools/reckless +++ b/tools/reckless @@ -385,9 +385,8 @@ class SourceDir(): self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: self.contents = populate_local_repo(self.location, parent=self, parent_source=self.parent_source) - elif self.srctype == Source.REMOTE_GIT_REPO: - self.contents = copy_remote_git_source(InstInfo(self.name, self.location)).contents - + elif self.srctype == Source.GITHUB_REPO: + self.contents = populate_github_repo(self.location) else: raise Exception("populate method undefined for {self.srctype}") # Ensure the relative path of the contents is inherited. @@ -660,7 +659,92 @@ def populate_local_repo(path: str, parent=None, parent_source=None) -> list: return basedir.contents -def copy_remote_git_source(github_source: InstInfo, verbose: bool=True, parent_source=None) -> SourceDir: +def source_element_from_repo_api(member: dict): + # api accessed via /contents/ + if 'type' in member and 'name' in member and 'git_url' in member: + if member['type'] == 'dir': + return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, + name=member['name']) + elif member['type'] == 'file': + # Likely a submodule + if member['size'] == 0: + return SourceDir(None, srctype=Source.GITHUB_REPO, + name=member['name']) + return SourceFile(member['name']) + elif member['type'] == 'commit': + # No path is given by the api here + return SourceDir(None, srctype=Source.GITHUB_REPO, + name=member['name']) + # git_url with /tree/ presents results a little differently + elif 'type' in member and 'path' in member and 'url' in member: + if member['type'] not in ['tree', 'blob']: + log.debug(f' skipping {member["path"]} type={member["type"]}') + if member['type'] == 'tree': + return SourceDir(member['url'], srctype=Source.GITHUB_REPO, + name=member['path']) + elif member['type'] == 'blob': + # This can be a submodule + if member['size'] == 0: + return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO, + name=member['name']) + return SourceFile(member['path']) + elif member['type'] == 'commit': + # No path is given by the api here + return SourceDir(None, srctype=Source.GITHUB_REPO, + name=member['name']) + return None + + +def populate_github_repo(url: str) -> list: + """populate one level of a github repository via REST API""" + # Forces search to clone remote repos (for blackbox testing) + if GITHUB_API_FALLBACK: + with tempfile.NamedTemporaryFile() as tmp: + raise HTTPError(url, 403, 'simulated ratelimit', {}, tmp) + # FIXME: This probably contains leftover cruft. + repo = url.split('/') + while '' in repo: + repo.remove('') + repo_name = None + parsed_url = urlparse(url.removesuffix('.git')) + if 'github.com' not in parsed_url.netloc: + return None + if len(parsed_url.path.split('/')) < 2: + return None + start = 1 + # Maybe we were passed an api.github.com/repo/ url + if 'api' in parsed_url.netloc: + start += 1 + repo_user = parsed_url.path.split('/')[start] + repo_name = parsed_url.path.split('/')[start + 1] + + # Get details from the github API. + if API_GITHUB_COM in url: + api_url = url + else: + api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/contents/' + + git_url = api_url + if "api.github.com" in git_url: + # This lets us redirect to handle blackbox testing + log.debug(f'fetching from gh API: {git_url}') + git_url = (API_GITHUB_COM + git_url.split("api.github.com")[-1]) + # Ratelimiting occurs for non-authenticated GH API calls at 60 in 1 hour. + r = urlopen(git_url, timeout=5) + if r.status != 200: + return False + if 'git/tree' in git_url: + tree = json.loads(r.read().decode())['tree'] + else: + tree = json.loads(r.read().decode()) + contents = [] + for sub in tree: + if source_element_from_repo_api(sub): + contents.append(source_element_from_repo_api(sub)) + return contents + + +def copy_remote_git_source(github_source: InstInfo, verbose: bool=True) -> SourceDir: """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: @@ -2252,7 +2336,7 @@ def fetch_manifest(source: SourceDir) -> dict: return None with open(path, 'r+') as manifest_file: try: - manifest = json.load(manifest_file) + manifest = json.loads(manifest_file.read()) return manifest except json.decoder.JSONDecodeError: log.warning(f'{source.name} contains malformed manifest ({source.parent_source.original_source})') @@ -2276,7 +2360,7 @@ def find_plugin_candidates(source: Union[LoadedSource, SourceDir], depth=2) -> l assert s.srctype == source.srctype, f'source dir {s.name}, {s.srctype} did not inherit {source.srctype} from {source.name}' assert s.parent_source == source.parent_source, f'source dir {s.name} did not inherit parent {source.parent_source} from {source.name}' - guess = InstInfo(source.name, source.location, source_dir=source) + guess = InstInfo(source.name, source.location, None, source_dir=source) guess.srctype = source.srctype manifest = None if guess.get_inst_details(): @@ -2324,8 +2408,7 @@ def available_plugins() -> list: clone = copy_remote_git_source(InstInfo(None, source.original_source, source_dir=source.content), - verbose=False, - parent_source=source) + verbose=False) clone.srctype = Source.GIT_LOCAL_CLONE clone.parent_source = source if not clone: From 6ed2b5c9171776540a6ff65720fbe7003ca0cd55 Mon Sep 17 00:00:00 2001 From: enaples Date: Thu, 12 Feb 2026 12:17:17 +0100 Subject: [PATCH 38/47] tests: makes `sed` works cross-platform --- tests/test_reckless.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 71ef7a2a2be6..8e84cf162d56 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -62,7 +62,8 @@ def canned_github_server(directory): 'git add --all;' 'git commit -m "initial commit - autogenerated by test_reckless.py";') tag_and_update = ('git tag v1;' - "sed -i 's/v1/v2/g' testplugpass/testplugpass.py;" + "sed -i.bak 's/v1/v2/g' testplugpass/testplugpass.py;" + "rm -f testplugpass/testplugpass.py.bak;" 'git add testplugpass/testplugpass.py;' 'git commit -m "update to v2";' 'git tag v2;') From 5a324163532c178449ee414066681b5e78295e8f Mon Sep 17 00:00:00 2001 From: enaples Date: Thu, 12 Feb 2026 17:32:44 +0100 Subject: [PATCH 39/47] tests: added rejection for shebang `#!/usr/bin/env -S uv run --script` --- tools/reckless | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index 36f3912b2458..0744fc1fc9bf 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1234,7 +1234,7 @@ def check_for_shebang(source: InstInfo) -> bool: if entrypoint_file.split('\n')[0].startswith('#!'): # Calling the python interpreter will not manage dependencies. # Leave this to another python installer. - for interpreter in ['bin/python', 'env python']: + for interpreter in ['bin/python', 'env python', 'uv run']: if interpreter in entrypoint_file.split('\n')[0]: return False return True From 562e403e406c55658db6fb160853570b8e4f0d46 Mon Sep 17 00:00:00 2001 From: enaples Date: Thu, 12 Feb 2026 17:33:44 +0100 Subject: [PATCH 40/47] tests: test reckless installs a local plugin cloned from GH --- tests/test_reckless.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 8e84cf162d56..832bb9c09a54 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -514,3 +514,49 @@ def test_reckless_usage(node_factory): with pytest.raises(lightning.RpcError, match="reckless: error: argument cmd1: invalid choice: 'saerch'"): node.rpc.reckless('saerch', 'testplugpass') + + +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_install_from_real_py(node_factory): + """Test reckless install a real plugin""" + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + # Temporarily bypass `canned_github_server` fixture so reckless hits real Github repo + saved_redir = my_env.pop('REDIR_GITHUB', None) + + try: + r = reckless([f"--network={NETWORK}", "-v", "install", "backup"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('plugin installed:') + + installed_path = (Path(node.lightning_dir) / 'reckless/backup').resolve() + assert installed_path.is_dir() + + network_dir = (Path(node.lightning_dir) / NETWORK).resolve() + backup_dest = str(network_dir / 'backup.bkp') + venv_python = str(installed_path / '.venv' / 'bin' / 'python') + backup_cli = str(installed_path / 'source' / 'backup' / 'backup-cli') + subprocess.run([venv_python, backup_cli, 'init', + '--lightning-dir', str(network_dir), + f'file://{backup_dest}'], + check=True, env=my_env, timeout=30) + + node.start() + + plugins = node.rpc.plugin_list()['plugins'] + plugin_names = [p['name'] for p in plugins] + assert any('backup' in name for name in plugin_names) + + r = reckless([f"--network={NETWORK}", "listavailable", "-v", "--json"], dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('backup') + + finally: + # Restore redirect for other tests + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + From f888c3e06e2c1a254da0636de65eb1637201d4c0 Mon Sep 17 00:00:00 2001 From: enaples Date: Mon, 16 Feb 2026 16:01:24 +0100 Subject: [PATCH 41/47] tests: test real py plugin installation from commit --- tests/test_reckless.py | 47 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 832bb9c09a54..59d8fdc9e2b5 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -519,8 +519,8 @@ def test_reckless_usage(node_factory): @pytest.mark.slow_test @pytest.mark.flaky(max_runs=3) @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") -def test_reckless_install_from_real_py(node_factory): - """Test reckless install a real plugin""" +def test_reckless_install_py(node_factory): + """Test reckless install a real python plugin""" notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) @@ -560,3 +560,46 @@ def test_reckless_install_from_real_py(node_factory): if saved_redir is not None: my_env['REDIR_GITHUB'] = saved_redir +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_install_from_commit_py(node_factory): + """Test reckless install a real python plugin from specifi commit""" + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + # Temporarily bypass `canned_github_server` fixture so reckless hits real Github repo + saved_redir = my_env.pop('REDIR_GITHUB', None) + from_commit = "478505fab37dd57a48834a5018016546729cf39a" + try: + r = reckless([f"--network={NETWORK}", "-v", "install", f"backup@{from_commit}"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('plugin installed:') + + installed_path = (Path(node.lightning_dir) / 'reckless/backup').resolve() + assert installed_path.is_dir() + + network_dir = (Path(node.lightning_dir) / NETWORK).resolve() + backup_dest = str(network_dir / 'backup.bkp') + venv_python = str(installed_path / '.venv' / 'bin' / 'python') + backup_cli = str(installed_path / 'source' / 'backup' / 'backup-cli') + subprocess.run([venv_python, backup_cli, 'init', + '--lightning-dir', str(network_dir), + f'file://{backup_dest}'], + check=True, env=my_env, timeout=30) + + node.start() + + plugins = node.rpc.plugin_list()['plugins'] + plugin_names = [p['name'] for p in plugins] + assert any('backup' in name for name in plugin_names) + + r = reckless([f"--network={NETWORK}", "listavailable", "-v", "--json"], dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('backup') + + finally: + # Restore redirect for other tests + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir \ No newline at end of file From fe94260eb69f9a09e0675c5460aff0c4e0248258 Mon Sep 17 00:00:00 2001 From: enaples Date: Mon, 16 Feb 2026 16:52:59 +0100 Subject: [PATCH 42/47] tests: test reckless install plugin from github url --- tests/test_reckless.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 59d8fdc9e2b5..d8f264739183 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -519,7 +519,7 @@ def test_reckless_usage(node_factory): @pytest.mark.slow_test @pytest.mark.flaky(max_runs=3) @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") -def test_reckless_install_py(node_factory): +def test_reckless_install_from_source_py(node_factory): """Test reckless install a real python plugin""" notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) @@ -560,6 +560,46 @@ def test_reckless_install_py(node_factory): if saved_redir is not None: my_env['REDIR_GITHUB'] = saved_redir +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_install_from_github_url(node_factory): + """Test reckless installs a plugin given a full GitHub URL. + """ + # Bypass the canned local redirect so reckless clones from real GitHub. + saved_redir = my_env.pop('REDIR_GITHUB', None) + + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + github_url = "https://github.com/ca-ruz/bumpit" + plugin_name = "bumpit" + + try: + r = reckless([f"--network={NETWORK}", "-v", "install", github_url], + dir=node.lightning_dir, timeout=300) + assert r.returncode == 0 + assert r.search_stdout('plugin installed:') + + installed_path = (Path(node.lightning_dir) / 'reckless' / plugin_name).resolve() + assert installed_path.is_dir() + + node.start() + + plugins = node.rpc.plugin_list()['plugins'] + plugin_names = [p['name'] for p in plugins] + assert any(plugin_name in name for name in plugin_names) + + r = reckless([f"--network={NETWORK}", "listavailable", "-v", "--json"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout(plugin_name) + + finally: + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + + @pytest.mark.slow_test @pytest.mark.flaky(max_runs=3) @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") From af1ae3dc39ddb8d7e57d3eb5dfcfaae849687083 Mon Sep 17 00:00:00 2001 From: enaples Date: Tue, 17 Feb 2026 09:39:19 +0100 Subject: [PATCH 43/47] tests: test reckless ability to disable a plugin that caused a cln crash --- tests/test_reckless.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index d8f264739183..a3589a035bbf 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -600,6 +600,58 @@ def test_reckless_install_from_github_url(node_factory): my_env['REDIR_GITHUB'] = saved_redir +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_install_crash_recovery(node_factory): + """Test reckless can uninstall a plugin that prevents CLN from starting. + + Installs the backup plugin via reckless, then replaces the entry + point with a script that exits immediately so CLN cannot start. + Verifies that reckless uninstall works while the node is stopped + and that CLN starts cleanly afterward. + """ + saved_redir = my_env.pop('REDIR_GITHUB', None) + + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, + options={"plugin": notification_plugin}, + start=False) + node.may_fail = True + + try: + r = reckless([f"--network={NETWORK}", "-v", "install", "backup"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('plugin installed:') + + installed_path = (Path(node.lightning_dir) / 'reckless' / 'backup').resolve() + assert installed_path.is_dir() + + node.daemon.start(wait_for_initialized=False) + rc = node.daemon.wait(timeout=60) + assert rc != 0, f"Expected CLN to crash but got exit code {rc}" + + r = reckless([f"--network={NETWORK}", "-v", "uninstall", "backup"], + dir=node.lightning_dir) + assert r.returncode == 0 + assert r.search_stdout('uninstalled') + + assert not installed_path.exists() + + config_path = Path(node.lightning_dir) / NETWORK / 'config' + if config_path.exists(): + config_text = config_path.read_text() + assert 'plugin=' not in config_text or 'backup' not in config_text + + node.may_fail = False + node.start() + + finally: + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + + @pytest.mark.slow_test @pytest.mark.flaky(max_runs=3) @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") From 3f6da2d58b2c3bf5ef12c1b648db446991f60a73 Mon Sep 17 00:00:00 2001 From: enaples Date: Tue, 17 Feb 2026 11:07:46 +0100 Subject: [PATCH 44/47] tests: making `test_install_crash_recovery` cln node to crash for real --- tests/test_reckless.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index a3589a035bbf..37f7228883f5 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -606,10 +606,9 @@ def test_reckless_install_from_github_url(node_factory): def test_install_crash_recovery(node_factory): """Test reckless can uninstall a plugin that prevents CLN from starting. - Installs the backup plugin via reckless, then replaces the entry - point with a script that exits immediately so CLN cannot start. - Verifies that reckless uninstall works while the node is stopped - and that CLN starts cleanly afterward. + Installs the backup plugin via reckless RPC call, then verify that + CLN crashed. Use reckless tool to unintall the plugin that caused + that prevents CLN from starting and verify that CLN starts cleanly afterward. """ saved_redir = my_env.pop('REDIR_GITHUB', None) @@ -620,17 +619,17 @@ def test_install_crash_recovery(node_factory): node.may_fail = True try: - r = reckless([f"--network={NETWORK}", "-v", "install", "backup"], - dir=node.lightning_dir) - assert r.returncode == 0 - assert r.search_stdout('plugin installed:') + node.start() + with pytest.raises(lightning.RpcError): + node.rpc.reckless('install', 'backup') + rc = node.daemon.wait(timeout=10) + assert rc != 0, f"Expected CLN to crash but got exit code {rc}" installed_path = (Path(node.lightning_dir) / 'reckless' / 'backup').resolve() assert installed_path.is_dir() - node.daemon.start(wait_for_initialized=False) - rc = node.daemon.wait(timeout=60) - assert rc != 0, f"Expected CLN to crash but got exit code {rc}" + installed_path = (Path(node.lightning_dir) / 'reckless' / 'backup').resolve() + assert installed_path.is_dir() r = reckless([f"--network={NETWORK}", "-v", "uninstall", "backup"], dir=node.lightning_dir) From 10fec7fe625eb77db79cb33227cc7f13f140cf9a Mon Sep 17 00:00:00 2001 From: enaples Date: Tue, 17 Feb 2026 11:11:23 +0100 Subject: [PATCH 45/47] tests: test reckless installs a plugin from a specific commit via RPC --- tests/test_reckless.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 37f7228883f5..5c2d3589e1c6 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -692,5 +692,48 @@ def test_reckless_install_from_commit_py(node_factory): finally: # Restore redirect for other tests + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + + +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_rpc_install_from_commit_py(node_factory): + """Test reckless installs a plugin at a specific commit via RPC.""" + saved_redir = my_env.pop('REDIR_GITHUB', None) + + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + plugin_name = "currencyrate" + from_commit = "306cfadcc90e98ce6fb2e27915efa72be0f66ad6" + + try: + node.start() + + result = node.rpc.reckless('install', f'{plugin_name}@{from_commit}') + assert 'plugin installed:' in ''.join(result.get('log', [])) + + installed_path = (Path(node.lightning_dir) / 'reckless' / plugin_name).resolve() + assert installed_path.is_dir() + + # Verify the .metadata file records the correct commit. + metadata_file = installed_path / '.metadata' + assert metadata_file.exists(), ".metadata file not found" + metadata_text = metadata_file.read_text() + metadata_lines = metadata_text.splitlines() + metadata = {} + for i, line in enumerate(metadata_lines): + if i > 0: + metadata[metadata_lines[i - 1].strip()] = line.strip() + + assert metadata.get('requested commit') == from_commit, \ + f"requested commit mismatch: {metadata.get('requested commit')}" + assert metadata.get('installed commit') is not None + assert metadata['installed commit'].startswith(from_commit[:7]), \ + f"installed commit {metadata['installed commit']} does not match {from_commit}" + + finally: if saved_redir is not None: my_env['REDIR_GITHUB'] = saved_redir \ No newline at end of file From 4623f697e787462cd89d98d5ce219c258a144fa3 Mon Sep 17 00:00:00 2001 From: enaples Date: Tue, 17 Feb 2026 11:30:30 +0100 Subject: [PATCH 46/47] tests: test reckless uninstall cleans confing from just-installed plugin --- tests/test_reckless.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index 5c2d3589e1c6..b2ba55ca231f 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -734,6 +734,41 @@ def test_reckless_rpc_install_from_commit_py(node_factory): assert metadata['installed commit'].startswith(from_commit[:7]), \ f"installed commit {metadata['installed commit']} does not match {from_commit}" + finally: + if saved_redir is not None: + my_env['REDIR_GITHUB'] = saved_redir + + +@pytest.mark.slow_test +@pytest.mark.flaky(max_runs=3) +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +def test_reckless_rpc_unistall_cleans_config(node_factory): + """Test reckless uninstall cleans confing from just-installed plugin.""" + saved_redir = my_env.pop('REDIR_GITHUB', None) + + notification_plugin = os.path.join(os.getcwd(), 'tests/plugins/custom_notifications.py') + node = get_reckless_node(node_factory, options={"plugin": notification_plugin}) + + plugin_name = "currencyrate" + + try: + node.start() + + result = node.rpc.reckless('install', plugin_name) + assert 'plugin installed:' in ''.join(result.get('log', [])) + + installed_path = (Path(node.lightning_dir) / 'reckless' / plugin_name).resolve() + assert installed_path.is_dir() + + result = node.rpc.reckless('uninstall', plugin_name) + assert f'{plugin_name} uninstalled successfully.' in ''.join(result.get('log', [])) + assert not installed_path.is_dir() + + config_path = Path(node.lightning_dir) / NETWORK / 'config' + if config_path.exists(): + config_text = config_path.read_text() + assert 'disable-plugin=' not in config_text or plugin_name not in config_text + finally: if saved_redir is not None: my_env['REDIR_GITHUB'] = saved_redir \ No newline at end of file From a9364ab6562b4c84bac28884c952bb81c62cdacc Mon Sep 17 00:00:00 2001 From: enaples Date: Wed, 18 Feb 2026 10:18:48 +0100 Subject: [PATCH 47/47] tests: test reckless returns well-formatted json --- tests/test_reckless.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index b2ba55ca231f..11a5a3e5b735 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -479,6 +479,37 @@ def test_reckless_available(node_factory): assert r.search_stdout('testpluguv') +def test_listavailable_json_format(node_factory): + """Verify listavailable --json returns properly structured JSON objects.""" + n = get_reckless_node(node_factory) + r = reckless([f"--network={NETWORK}", "listavailable", "--json"], + dir=n.lightning_dir) + assert r.returncode == 0 + + result = json.loads(''.join(r.stdout)) + plugins = result['result'] + assert isinstance(plugins, list) + assert len(plugins) > 0 + + expected_fields = {'name', 'short_description', 'long_description', + 'entrypoint', 'requirements'} + + for plugin in plugins: + + assert isinstance(plugin, dict), \ + f"expected dict but got {type(plugin).__name__}: {plugin!r}" + + missing = expected_fields - plugin.keys() + assert not missing, \ + f"plugin {plugin.get('name', '?')} missing fields: {missing}" + + assert isinstance(plugin['name'], str) and plugin['name'] + assert isinstance(plugin['entrypoint'], str) and plugin['entrypoint'] + + assert isinstance(plugin['requirements'], list), \ + f"plugin {plugin['name']}: requirements should be a list" + + @pytest.mark.timeout(120) def test_reckless_notifications(node_factory): """Reckless streams logs to the reckless-rpc plugin which are emitted