diff --git a/.github/workflows/check-self.yml b/.github/workflows/check-self.yml index a2e856618..b5a277eff 100644 --- a/.github/workflows/check-self.yml +++ b/.github/workflows/check-self.yml @@ -26,11 +26,20 @@ jobs: - name: Install the latest version of uv uses: astral-sh/setup-uv@v5 + env: + UV_PYTHON: 3.11 - uses: actions-rs/toolchain@v1 with: toolchain: stable + - name: Download CLN versions + env: + UV_PYTHON: 3.11 + run: | + cd libs/cln-version-manager + uv run --extra cli python3 clnvm/cli.py get-all + - name: Install Protoc uses: arduino/setup-protoc@v3 @@ -38,6 +47,8 @@ jobs: run: task docker:install-dependencies - name: Adjust PATH + env: + UV_PYTHON: 3.11 run: | echo "/home/runner/.cargo/bin" >> $GITHUB_PATH # Directory in which go-task deposits its binaries @@ -53,13 +64,16 @@ jobs: key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Build binaries + env: + UV_PYTHON: 3.11 run: task ci-build - + - name: Check Self env: GL_TESTING_IGNORE_HASH: 1 PYTEST_OPTS: -n 6 + UV_PYTHON: 3.11 run : | task clientpy:check task testing:check diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 55014853c..8a8de644e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -25,7 +25,7 @@ jobs: python-version: 3.9 architecture: x64 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 with: version: "latest" @@ -53,11 +53,9 @@ jobs: # - armv7 steps: - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: 3.9 - + - name: Install uv + uses: astral-sh/setup-uv@v7 + - name: Install Protoc uses: arduino/setup-protoc@v2 with: @@ -84,16 +82,17 @@ jobs: install: | apt-get update apt-get install -y --no-install-recommends python3 python3-pip - pip3 install -U pip pytest + uv sync run: | - pip install libs/gl-client-py/dist/gl_client*.whl --force-reinstall + uv pip install libs/gl-client-py/dist/gl_client*.whl --force-reinstall python3 -c "import glclient;creds=glclient.Credentials();signer=glclient.Signer(b'\x00'*32,'bitcoin', creds);print(repr(creds));print(signer.version())" - name: Install built wheel (native) if: matrix.target == 'x86_64' run: | - pip install libs/gl-client-py/dist/gl_client*.whl --force-reinstall - python3 -c "import glclient;creds=glclient.Credentials();signer=glclient.Signer(b'\x00'*32,'bitcoin', creds);print(repr(creds));print(signer.version())" + uv sync + uv pip install libs/gl-client-py/dist/gl_client*.whl --force-reinstall + uv run python3 -c "import glclient;creds=glclient.Credentials();signer=glclient.Signer(b'\x00'*32,'bitcoin', creds);print(repr(creds));print(signer.version())" - name: Upload wheels uses: actions/upload-artifact@v4 @@ -102,7 +101,7 @@ jobs: path: libs/gl-client-py/dist/ macos: - runs-on: macos-13 + runs-on: macos-latest strategy: fail-fast: false matrix: @@ -114,9 +113,13 @@ jobs: - uses: actions/setup-python@v4 with: python-version: 3.9 - architecture: x64 + - uses: dtolnay/rust-toolchain@nightly + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v7 + with: + version: "latest" - name: Install Protoc uses: arduino/setup-protoc@v2 with: @@ -134,10 +137,13 @@ jobs: MACOSX_DEPLOYMENT_TARGET: 10.9 - name: Install built wheel - if: matrix.target == 'x86_64' + env: + PATH: $PATH:$HOME/.local/bin + if: matrix.target == 'aarch64' run: | - pip install libs/gl-client-py/dist/gl_client*.whl --force-reinstall - python3 -c "import glclient;creds=glclient.Credentials();signer=glclient.Signer(b'\x00'*32,'bitcoin', creds);print(repr(creds));print(signer.version())" + uv sync + uv pip install libs/gl-client-py/dist/gl_client*.whl --force-reinstall + uv run python3 -c "import glclient;creds=glclient.Credentials();signer=glclient.Signer(b'\x00'*32,'bitcoin', creds);print(repr(creds));print(signer.version())" - name: Upload wheels uses: actions/upload-artifact@v4 diff --git a/libs/cln-version-manager/clnvm/cln_version_manager.py b/libs/cln-version-manager/clnvm/cln_version_manager.py index 9baa37e26..8e2e4f8da 100644 --- a/libs/cln-version-manager/clnvm/cln_version_manager.py +++ b/libs/cln-version-manager/clnvm/cln_version_manager.py @@ -13,96 +13,36 @@ import requests from clnvm.utils import NodeVersion -from clnvm.errors import UnrunnableVersion, HashMismatch, VersionMismatch +from clnvm.errors import ( + UnrunnableVersion, + HashMismatch, + VersionMismatch, + SignatureVerificationFailed, +) PathLike = Union[os.PathLike, str] +# Unless the URL is an absolute URL, use this prefix to complete the URL. +BASE_URL = "https://storage.googleapis.com/greenlight-artifacts/cln" +MANIFEST_URL = f"{BASE_URL}/manifest.json" +PUBKEY_FINGERPRINT = "1E832A80A25B69C6C7123285AB5187B13F8DD139" +PUBKEY_URL = f"{BASE_URL}/{PUBKEY_FINGERPRINT}.pub" +_LIGHTNINGD_REL_PATH = Path("usr/local/bin/lightningd") +_BIN_REL_PATH = Path("usr/local/bin") +GPG_OPTS = ['--no-default-keyring', '--keyring=/tmp/clnvm-keyring.gpg'] @dataclass class VersionDescriptor: tag: str url: str checksum: str + signature: Optional[str] = None logger = logging.getLogger(__name__) -# Stores all versions of cln that can be scheduled -CLN_VERSIONS = [ - VersionDescriptor( - tag="v0.10.1", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v0.10.1.tar.bz2", - checksum="928f09a46c707f68c8c5e1385f6a51e10f7b1e57c5cef52f5b73c7d661500af5", - ), - VersionDescriptor( - tag="v0.10.2", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v0.10.2.tar.bz2", - checksum="c323f2e41ffde962ac76b2aeaba3f2360b3aa6481028f11f12f114f408507bfe", - ), - VersionDescriptor( - tag="v0.11.0.1", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v0.11.0.1.tar.bz2", - checksum="0f1a49bb8696db44a9ab93d8a226e82b4d3de03c9bae2eb38b750d75d4bcaceb", - ), - VersionDescriptor( - tag="v0.11.2gl2", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v0.11.2gl2.tar.bz2", - checksum="b15866b7beea239aaf4e38931fe09ee85bf2e58ad61c2ec79b83bb361364bf97", - ), - VersionDescriptor( - tag="v0.11.2", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v0.11.2.tar.bz2", - checksum="95209242d8ddc4879b959fb5e4594b4d2dcf7bac7227ec7c421ab05019de8633", - ), - VersionDescriptor( - tag="v22.11gl1", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v22.11gl1.tar.bz2", - checksum="40b6c50babdc74d9fd251816efa46de0c12cac88d72e0c7b02457a8949d2690b", - ), - VersionDescriptor( - tag="v23.05gl1", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v23.05gl1.tar.bz2", - checksum="e1a57a8ced59fd92703fad8e34926c014b71ee0c13cc7f863cb18b2ca19a58b9", - ), - VersionDescriptor( - tag="v23.08gl1", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v23.08gl1.tar.bz2", - checksum="0e392c5117e14dc37cf72393f47657a09821f69ab8b45937d7e79ca8d91d17e9", - ), - VersionDescriptor( - tag="v24.02gl1", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v24.02gl1.tar.bz2", - checksum="31fc7e79eddfa5c4083d8d516b3c95477e65b0d2c6b671fd12170819db3217be", - ), - VersionDescriptor( - tag="v24.02", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v24.02.tar.bz2", - checksum="690f5b3ce0404504913bb7cde22d88efeabe72226aefe31a70916cf89905455d", - ), - VersionDescriptor( - tag="v24.11gl1", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v24.11gl1.tar.bz2", - checksum="06818569d3a68d578cf390d01a6d09a5c969b7c6fdef9291dfe6fb707bb02fcc", - ), - VersionDescriptor( - tag="v25.05gl1", - url="https://storage.googleapis.com/greenlight-artifacts/cln/lightningd-v25.05gl1.tar.bz2", - checksum="5f978b2a778f9e148bf09fec1fe5c132913b63a82e12a1bf74fd3ed507770e52", - ), -] - - -_LIGHTNINGD_REL_PATH = Path("usr/local/bin/lightningd") -_BIN_REL_PATH = Path("usr/local/bin") - - -def _get_cln_version_path(cln_path: Optional[PathLike] = None) -> Path: - """ - Retrieve the path where all cln-versions are stored - """ - if cln_path is not None: - return Path(cln_path).resolve() +def _get_cache_dir() -> Path: cln_cache_dir = os.environ.get("CLNVM_CACHE_DIR") if cln_cache_dir is not None: return Path(cln_cache_dir).resolve() @@ -114,6 +54,76 @@ def _get_cln_version_path(cln_path: Optional[PathLike] = None) -> Path: else: return Path("~/.cache").expanduser().resolve() / "clnvm" +def _ensure_pubkey() -> Path: + """Ensure we have the pubkey that signs the releases in the cache. + + Download if we don't yet. + """ + fpath = _get_cache_dir() + fpath.mkdir(parents=True, exist_ok=True) + pubkey_path = fpath / f"{PUBKEY_FINGERPRINT}.asc" + if not pubkey_path.exists(): + logger.debug("Fetching public key from %s", PUBKEY_URL) + pubkey_response = requests.get(PUBKEY_URL) + if pubkey_response.status_code != 200: + raise ValueError( + f"Failed to fetch public key: {pubkey_response.status_code}" + ) + with Path(pubkey_path).open(mode="w") as f: + f.write(pubkey_response.text) + try: + subprocess.check_call(["gpg", *GPG_OPTS, "--import", pubkey_path]) + except: + raise ValueError("Failed to import public key") + logger.debug("Imported public key.") + + return pubkey_path + +def _verify_signature( + file_path: Path, + signature: str, + tag: str, + pubkey_fingerprint: str = PUBKEY_FINGERPRINT, +) -> None: + """ + Verify the GPG signature of a file. + + Args: + file_path: Path to the file to verify + signature: The ASCII-armored PGP signature + tag: Version tag (for error messages) + pubkey_fingerprint: The fingerprint of the public key to use + + Raises: + SignatureVerificationFailed: If signature verification fails + """ + logger.debug("Verifying GPG signature for version %s", tag) + + pubkey_path = _ensure_pubkey() + sig_file = Path(file_path.parent / "signature.asc") + with sig_file.open(mode="w") as f: + f.write(signature) + try: + subprocess.check_call(["gpg", *GPG_OPTS, "--verify", sig_file, file_path]) + except Exception as e: + raise SignatureVerificationFailed( + tag=tag, reason=f"Invalid signature: {repr(e)}" + ) + + logger.debug( + "Successfully verified signature for version %s with key", + tag, + ) + + +def _get_cln_version_path(cln_path: Optional[PathLike] = None) -> Path: + """ + Retrieve the path where all cln-versions are stored + """ + if cln_path is not None: + return Path(cln_path).resolve() + return _get_cache_dir() + class ClnVersionManager: def __init__( @@ -125,7 +135,21 @@ def __init__( if cln_versions is not None: self._cln_versions = cln_versions else: - self._cln_versions = CLN_VERSIONS + self.update() + + def update(self) -> None: + """Fetch the manifest, and populate our list of versions.""" + manifest = requests.get(MANIFEST_URL).json() + versions = [ + VersionDescriptor( + tag=k, + url=f"{BASE_URL}/{v['filename']}", + checksum=v["sha256"], + signature=v.get("signature"), + ) + for k, v in manifest["versions"].items() + ] + self._cln_versions = versions def get_versions(self) -> List[VersionDescriptor]: """ @@ -170,7 +194,7 @@ def get_target_path(self, cln_version: VersionDescriptor) -> Path: return Path(self._cln_path) / cln_version.checksum / cln_version.tag def get_descriptor_from_tag(self, tag: str) -> VersionDescriptor: - cln_dict = {d.tag: d for d in CLN_VERSIONS} + cln_dict = {d.tag: d for d in self._cln_versions} descriptor = cln_dict.get(tag, None) if descriptor is None: @@ -208,13 +232,18 @@ def get(self, cln_version: VersionDescriptor, force: bool = False) -> NodeVersio root_path=target_path, ) - def _download(self, cln_version: VersionDescriptor, target_path: Path, verify_tag:bool = False) -> None: + def _download( + self, + cln_version: VersionDescriptor, + target_path: Path, + verify_tag: bool = False, + ) -> None: """Downloads the provided cln_version""" tag = cln_version.tag logger.info("Downloading version %s to %s", tag, target_path) # Retrieve the version from the provided url - response = requests.get(cln_version.url) + response = requests.get(cln_version.url, stream=True) if response.status_code != 200: logger.warning( "Failed to retrieve %s: %s - %s", @@ -224,50 +253,68 @@ def _download(self, cln_version: VersionDescriptor, target_path: Path, verify_ta ) raise Exception(f"Failed to find version {tag}") - data = response.content - - # We check the hash of the downloaded data - # If the hash doesn't match we stop and alert the user - m = hashlib.sha256() - m.update(data) - content_sha = m.hexdigest() - - logger.debug("Downloaded version %s with checksum %s", tag, content_sha) - ignore_hash = bool(os.environ.get("GL_TESTING_IGNORE_HASH", False)) - if ignore_hash: - logger.warning( - "Checking the hash of remote versions is disabled which is unsafe. " - "Try to unset GL_TESTING_IGNORE_HASH" - ) + # Write to a temporary file and compute hash in one pass + import tempfile + + tmp_dir = tempfile.mkdtemp() + tmp_file = Path(tmp_dir) / "download.tar" + + try: + m = hashlib.sha256() + with open(tmp_file, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + m.update(chunk) + + content_sha = m.hexdigest() + logger.debug("Downloaded version %s with checksum %s", tag, content_sha) + + # Verify signature if available + if cln_version.signature: + logger.info("Verifying GPG signature for version %s", tag) + _verify_signature(tmp_file, cln_version.signature, tag) + else: + logger.warning("No signature available for version %s", tag) + + # Check the hash + ignore_hash = bool(os.environ.get("GL_TESTING_IGNORE_HASH", False)) + if ignore_hash: + logger.warning( + "Checking the hash of remote versions is disabled which is unsafe. " + "Try to unset GL_TESTING_IGNORE_HASH" + ) - if (not ignore_hash) and content_sha != cln_version.checksum: - raise HashMismatch( - tag=cln_version.tag, expected=cln_version.checksum, actual=content_sha - ) + if (not ignore_hash) and content_sha != cln_version.checksum: + raise HashMismatch( + tag=cln_version.tag, + expected=cln_version.checksum, + actual=content_sha, + ) - # We extract the downloaded tar-file in the section below. - # Note, that we never put the tar-file on disk. We extract - # it straight from memory - # - # In python 3.12 the `filter`-argument was introduced - # to `TarFile.extractall`. - # - # Using this argument provide us extra security against - # malicious .tar-files. - # - # The extra security is nice to have. We used the hash to - # check the authenticity of our .tar-files a few lines above. - # - # We'll use the filter argument if it is available. - # In the other case we rely on the hash to keep our users safe - content_fh = BytesIO(data) - tf = tarfile.open(fileobj=content_fh) - - current_version = sys.version_info - if current_version.minor >= 12: - tf.extractall(path=target_path, filter="data") - else: - tf.extractall(path=target_path) + # We extract the downloaded tar-file in the section below. + # In python 3.12 the `filter`-argument was introduced + # to `TarFile.extractall`. + # + # Using this argument provide us extra security against + # malicious .tar-files. + # + # The extra security is nice to have. We used the hash and + # signature to check the authenticity of our .tar-files above. + # + # We'll use the filter argument if it is available. + # In the other case we rely on the hash to keep our users safe + tf = tarfile.open(str(tmp_file)) + + current_version = sys.version_info + if current_version.minor >= 12: + tf.extractall(path=target_path, filter="data") + else: + tf.extractall(path=target_path) + + finally: + # Clean up temporary file + shutil.rmtree(tmp_dir) if verify_tag: try: diff --git a/libs/cln-version-manager/clnvm/errors.py b/libs/cln-version-manager/clnvm/errors.py index f0442a463..53beabf7d 100644 --- a/libs/cln-version-manager/clnvm/errors.py +++ b/libs/cln-version-manager/clnvm/errors.py @@ -47,3 +47,16 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"HashMismatch(tag={self.tag}, actual={self.actual}, expected={self.expected})" + + +class SignatureVerificationFailed(Exception): + + def __init__(self, tag: str, reason: str): + self.tag = tag + self.reason = reason + + def __str__(self) -> str: + return self.__repr__() + + def __repr__(self) -> str: + return f"SignatureVerificationFailed(tag={self.tag}, reason={self.reason})" diff --git a/libs/cln-version-manager/tests/test_version_manager.py b/libs/cln-version-manager/tests/test_version_manager.py index caa451d61..be4dcf08b 100644 --- a/libs/cln-version-manager/tests/test_version_manager.py +++ b/libs/cln-version-manager/tests/test_version_manager.py @@ -5,8 +5,7 @@ import requests -from clnvm.cln_version_manager import ClnVersionManager -from clnvm.cln_version_manager import CLN_VERSIONS +from clnvm.cln_version_manager import ClnVersionManager, VersionDescriptor def get_tmp_dir(name: str) -> str: @@ -18,14 +17,18 @@ def get_tmp_dir(name: str) -> str: def test_download_cln_version() -> None: - # Only download the first 2 versions in the test - versions = CLN_VERSIONS[-2:] - vm = ClnVersionManager( + # Download the latest 2 versions from the manifest + vm = ClnVersionManager(cln_path=get_tmp_dir("test_download_cln_version")) + # Get all versions from the manifest and use the last 2 + all_versions = vm.get_versions() + versions = all_versions[-2:] if len(all_versions) >= 2 else all_versions + + vm_test = ClnVersionManager( cln_versions=versions, cln_path=get_tmp_dir("test_download_cln_version") ) - vm.get_all() + vm_test.get_all() # get them again to verify we don't download them with mock.patch("requests.get") as request_mock: - vm.get_all() + vm_test.get_all() assert not request_mock.get.called diff --git a/libs/gl-client-py/tests/test_plugin.py b/libs/gl-client-py/tests/test_plugin.py index c7f38baf7..b9f882b8d 100644 --- a/libs/gl-client-py/tests/test_plugin.py +++ b/libs/gl-client-py/tests/test_plugin.py @@ -5,6 +5,7 @@ import pytest import secrets from pathlib import Path +from flaky import flaky trmp_plugin_path = Path(__file__).parent / "plugins" / "trmp_htlc_hook.py" @@ -41,6 +42,7 @@ def test_max_message_size(clients): n1.datastore("some-key", hex=bytes.fromhex(secrets.token_hex(size))) +@flaky(max_runs=10) def test_trampoline_pay(bitcoind, clients, node_factory): c1 = clients.new() c1.register() @@ -61,6 +63,7 @@ def test_trampoline_pay(bitcoind, clients, node_factory): l2 = node_factory.get_node( options={ "plugin": trmp_plugin_path, + "disable-plugin": "cln-grpc", } ) n1.connect_peer(l2.info["id"], f"localhost:{l2.port}") @@ -116,7 +119,7 @@ def test_trampoline_pay(bitcoind, clients, node_factory): assert len(ch.htlcs) == 0 # new unknown unconnected node without the trampoline featurebit. - l3 = node_factory.get_node() + l3 = node_factory.get_node(options={"disable-plugin": "cln-grpc"}) inv = l2.rpc.invoice( amount_msat=1000000, label="trampoline-pay-test-2",