From 18e1af8d097fff0da33c2e9cb496de41db17f695 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 23 Jan 2025 13:10:45 +0100 Subject: [PATCH 01/15] chore(py): Migrate root and gl-client to `uv` --- Makefile | 8 +++-- libs/gl-client-py/pyproject.toml | 10 ++++-- libs/gl-testing/pyproject.toml | 8 ++--- pyproject.toml | 55 +++++++++----------------------- 4 files changed, 33 insertions(+), 48 deletions(-) diff --git a/Makefile b/Makefile index 246711042..b5cc130ca 100644 --- a/Makefile +++ b/Makefile @@ -102,14 +102,18 @@ gen: ${GENALL} build-self: ensure-docker cargo build --all - cd libs/gl-client-py; maturin develop + cd libs/gl-client-py; maturin develop --uv #mypy examples/python check-all: check-rs check-self check-py check-testing-py check-self: ensure-docker build-self + uv sync --package gl-testing + uv pip install libs/cln-version-manager + ls -lha .venv/bin PYTHONPATH=/repo/libs/gl-testing \ - pytest -vvv \ + GL_TESTING_IGNORE_HASH=1 \ + uv run python3 -m pytest -vvv -s\ /repo/libs/gl-testing \ ${PYTEST_OPTS} diff --git a/libs/gl-client-py/pyproject.toml b/libs/gl-client-py/pyproject.toml index 637e66bbd..862904816 100644 --- a/libs/gl-client-py/pyproject.toml +++ b/libs/gl-client-py/pyproject.toml @@ -1,6 +1,14 @@ [project] name = "gl-client" version = "0.3.0" +license = "MIT" +description = "Python bindings to the Greenlight service" +readme = "README.md" +requires-python = ">=3.8" + +authors = [ + {name = "Christian Decker", email = "decker@blockstream.com"}, +] dependencies = [ "protobuf>=3", @@ -12,8 +20,6 @@ dependencies = [ name = "gl-client" version = "0.3.0" description = "" -authors = ["Christian Decker "] -license = "MIT" packages = [ { include = "glclient" }, diff --git a/libs/gl-testing/pyproject.toml b/libs/gl-testing/pyproject.toml index 17868427e..97e82d2dc 100644 --- a/libs/gl-testing/pyproject.toml +++ b/libs/gl-testing/pyproject.toml @@ -1,11 +1,11 @@ [project] -name = "gltesting" +name = "gl-testing" version = "0.3.0" description = "" readme = "README.md" requires-python = ">=3.8" dependencies = [ - "cln-version-manager", +# "cln-version-manager", "flaky>=3.8.1", "gl-client", "grpcio-tools>=1.66", @@ -26,5 +26,5 @@ dependencies = [ package = true [tool.uv.sources] -gl-client = { path = "../gl-client-py" } -cln-version-manager = { path = "../cln-version-manager" } +gl-client = { workspace = true } +#cln-version-manager = { path = "../cln-version-manager" } diff --git a/pyproject.toml b/pyproject.toml index 410b4315f..97551796a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,43 +1,18 @@ -[tool.poetry] -package-mode=false +[project] +name = "gl-public" +version = "0.1.0" +description = "Public tools and utilities for Greenlight" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [] -[tool.poetry.dependencies] -python = ">=3.8,<4.0" -gl-client = {path = "./libs/gl-client-py", develop=true} -gltesting = {path = "./libs/gl-testing", develop=true} -cln-version-manager = {path = "./libs/cln-version-manager", develop=true} - -[tool.poetry.group.dev.dependencies] -black = "^24.0.0" -pytest = "^7" -mypy = "^1.7.0" -grpcio-tools = "^1.59.2" -maturin = {version = ">=1.0,<1.3.2", extras = ["patchelf"]} -mypy-protobuf = "^3.5" -typed-ast = "^1.5.4" -grpc-stubs= "^1.53.0" - -[tool.poetry.group.dev-cln-version-manager.dependencies] -types-requests = "^2.31" - -[tool.poetry.group.docs] -optional=true - -[tool.poetry.group.docs.dependencies] -python = ">=3.8,<4" -mkdocs-material = "^9.5.0" -mkdocs-redirects = "^1.1.0" -mkdocs-material-extensions = "^1.3.1" -pymdown-extensions = "^10.7" -ghp-import = "2.1.0" -pdoc = "^13.1.1" -cairosvg = "^2.7.0" -pillow = "^9.5.0" - -# Language server protocol used by the ide -# Not to be confused with lightning service provider -[tool.poetry.group.lsp_ide.dependencies] -python-lsp-server = "^1.10.0" +[tool.uv] +package = false [tool.uv.workspace] -members = ["libs/gl-testing", "libs/gl-testserver"] +members = [ + "libs/gl-testing", + "libs/gl-client-py", +# "libs/gl-testserver" +] +exclude = ["packages/seeds"] From 976e0c9e3faab7cc1f4540d136d6b9d791697a11 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 23 Jan 2025 13:52:50 +0100 Subject: [PATCH 02/15] clnvm: Fix the loggign.conf file not being loaded as a resource --- libs/cln-version-manager/clnvm/cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/cln-version-manager/clnvm/cli.py b/libs/cln-version-manager/clnvm/cli.py index 4a977f798..f7af082bd 100644 --- a/libs/cln-version-manager/clnvm/cli.py +++ b/libs/cln-version-manager/clnvm/cli.py @@ -1,18 +1,18 @@ import importlib import logging import logging.config -from pathlib import Path -import os import sys -from typing import Optional import clnvm -from clnvm.cln_version_manager import ClnVersionManager, VersionDescriptor +from clnvm.cln_version_manager import ClnVersionManager +from importlib import resources + def configure_logging() -> None: - dirname, _ = os.path.split(__file__) - logging.config.fileConfig(Path(dirname) / "logging.conf") + with resources.path(clnvm, "logging.conf") as fname: + assert fname is not None, "logging.conf must be bundled as a resource" + logging.config.fileConfig(fname) # Handle the optional import and provide a nice error message if it fails _click = importlib.util.find_spec("click") From 7ee09d0f863c3a2e3b2e2c597c20b4f3f224ace6 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 23 Jan 2025 15:22:52 +0100 Subject: [PATCH 03/15] chore(ci): Simplify the Dockerfile after switching to `uv` --- Makefile | 5 +- docker/gl-testing/Dockerfile | 100 +++++++++-------------------------- libs/gl-client-py/README.md | 1 + 3 files changed, 28 insertions(+), 78 deletions(-) create mode 100644 libs/gl-client-py/README.md diff --git a/Makefile b/Makefile index b5cc130ca..f4d9431e3 100644 --- a/Makefile +++ b/Makefile @@ -102,8 +102,7 @@ gen: ${GENALL} build-self: ensure-docker cargo build --all - cd libs/gl-client-py; maturin develop --uv - #mypy examples/python + cd libs/gl-client-py; uv tool run maturin develop --uv check-all: check-rs check-self check-py check-testing-py @@ -113,7 +112,7 @@ check-self: ensure-docker build-self ls -lha .venv/bin PYTHONPATH=/repo/libs/gl-testing \ GL_TESTING_IGNORE_HASH=1 \ - uv run python3 -m pytest -vvv -s\ + uv run python3 -m pytest -vvv -n 4 \ /repo/libs/gl-testing \ ${PYTEST_OPTS} diff --git a/docker/gl-testing/Dockerfile b/docker/gl-testing/Dockerfile index 4802701d9..3696e5055 100644 --- a/docker/gl-testing/Dockerfile +++ b/docker/gl-testing/Dockerfile @@ -45,39 +45,13 @@ RUN python3 -m venv /tmp/venv # This approach also ensures `poetry` and our environment don't overlap ENV POETRY_HOME=/tmp/poetry RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.2 -ENV PATH=$POETRY_HOME/bin:$PATH +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +ENV PATH=$POETRY_HOME/bin:$HOME/.local/bin:$PATH -# Uncomment below line after poetry v2.x.x upgrade -# RUN poetry self add poetry-plugin-export - -# The pyproject.toml files describe our project -# I purposefully ignore lock-files here -# We are developping a libraries and if a new version of a -# dependency breaks our code I want to be the first to know it -ADD pyproject.toml /repo/pyproject.toml -ADD libs/gl-client-py/pyproject.toml /repo/libs/gl-client-py/pyproject.toml -ADD libs/gl-testing/pyproject.toml /repo/libs/gl-testing/pyproject.toml -ADD libs/cln-version-manager/pyproject.toml /repo/libs/cln-version-manager/pyproject.toml +ADD . /repo WORKDIR /repo -# We don't want to reinstall depenendencies every time -# a single line of library code changes. -# -# Poetry install requires us to ADD the library-code to -# the dockerfile. So this approach doesn't work -# -# Using poetry export we can generate a list of all dependencies. -# We filter out all path-depenendencies from this list cause we -# don't have the code in the Docker-repo yet. -RUN poetry lock -RUN poetry export -f requirements.txt -o requirements.txt --with=dev -RUN cat requirements.txt | sed '/file:\/\/\//d' > requirements2.txt -RUN python -m pip install --upgrade pip && pip install wheel && pip install -r requirements2.txt - -# Later we will run this image under another user -# It is great if they can access this as well -RUN chmod a+rwx -R /tmp/venv # --------------------------------------------- # STAGE: rust-builder @@ -103,23 +77,6 @@ RUN rustup default stable # Ensure we can use cargo under a different user RUN chmod a+rwx -R $CARGO_HOME -# ------------------------------------------ -# STAGE: cln-downloader -# ----------------------------------------- -# gl-testing requires multiple versions of Core Lightning -# Download all versions so they can be copied to our dev-container -FROM python-builder AS cln-downloader - -ARG GL_TESTING_IGNORE_HASH=0 - -RUN mkdir -p /opt/cln/ -ENV CLNVM_CACHE_DIR=/opt/cln - -ADD libs/cln-version-manager /repo/libs/cln-version-manager -RUN cd /repo/libs/cln-version-manager; python -m pip install -e . - -RUN GL_TESTING_IGNORE_HASH=${GL_TESTING_IGNORE_HASH} python -m clnvm get-all - # ------------------------------------- # STAGE: bitcoin-downloader # ------------------------------------ @@ -192,6 +149,23 @@ ENV DEBIAN_FRONTEND=noninteractive ENV GL_DOCKER=1 # Tells cln to use shorter polling intervals ENV DEVELOPER=1 +ENV PYTHONUNBUFFERED=1 +ENV PATH=$HOME/.local/bin:/opt/cln-latest/usr/local/bin:/opt/bitcoin/bin:/opt/cln-latest/usr/local/bin:$PATH +ENV UV_INSTALL_DIR=/usr/local/bin/ +# grpcio == 1.46 produces spammy log messages, silence them +ENV GRPC_ENABLE_FORK_SUPPORT=0 + +# Install cln-versions +ENV CLNVM_CACHE_DIR=/opt/cln +ENV GL_TESTING_IGNORE_HASH=1 + +# Enumerate all versions that gl-testing should find +ENV CLN_PATH=/opt/cln/v0.10.1/usr/local/bin/:/opt/cln/v0.10.2/usr/local/bin/:/opt/cln/v0.11.0.1/usr/local/bin/:/opt/cln/v0.11.2gl2/usr/local/bin/:/opt/cln/v22.11gl1/usr/local/bin/:/opt/cln/v23.05gl1/usr/local/bin/:/opt/cln/v23.08gl1/usr/local/bin/:/opt/cln/v24.02gl1/usr/local/bin/ + +# Create a non-root user. +ARG GID=0 +ARG UID=0 +ARG DOCKER_USER=dev # Required dependencies and some dependencies which are nice to have # floating around @@ -214,20 +188,8 @@ RUN apt-get update && apt install -qqy \ # Set up a default logging filter. This includes all trace-level messages from greenlight components and validating lightning signer ENV RUST_LOG=gl_client=trace,tracing=warn,gl_signerproxy=trace,gl_plugin=trace,lightning_signer=trace,vls_protocol_signer=trace,vls_core=trace,vls_persist=trace,vls_protocol=trace,info -# grpcio == 1.46 produces spammy log messages, silence them -ENV GRPC_ENABLE_FORK_SUPPORT=0 - -# Install cln-versions -COPY --from=cln-downloader /opt/cln /opt/cln -ENV CLNVM_CACHE_DIR=/opt/cln -ENV PATH=/opt/cln-latest/usr/local/bin:$PATH - -# Enumerate all versions that gl-testing should find -ENV CLN_PATH=/opt/cln/v0.10.1/usr/local/bin/:/opt/cln/v0.10.2/usr/local/bin/:/opt/cln/v0.11.0.1/usr/local/bin/:/opt/cln/v0.11.2gl2/usr/local/bin/:/opt/cln/v22.11gl1/usr/local/bin/:/opt/cln/v23.05gl1/usr/local/bin/:/opt/cln/v23.08gl1/usr/local/bin/:/opt/cln/v24.02gl1/usr/local/bin/ - # Install bitcoin-core COPY --from=bitcoin-downloader /opt/bitcoin/bin /opt/bitcoin/bin -ENV PATH=/opt/bitcoin/bin:$PATH # Install cfssl COPY --from=cfssl-downloader /usr/bin/cfssl /usr/local/bin/cfssl @@ -236,18 +198,6 @@ COPY --from=cfssl-downloader /usr/bin/cfssljson /usr/local/bin/cfssljson # Install protoc COPY --from=protoc-downloader /usr/local/bin/protoc /usr/local/bin/protoc -# Copy the installs from the python-builder -COPY --from=python-builder /tmp/poetry /tmp/poetry -COPY --from=python-builder /tmp/venv /tmp/venv -COPY --from=python-builder /repo/poetry.lock /repo/poetry.lock - -ENV PATH=/tmp/poetry/bin:/tmp/venv/bin:$PATH -ENV VIRTUAL_ENV=/tmp/venv - -# Create a non-root user. -ARG GID=0 -ARG UID=0 -ARG DOCKER_USER=dev # Check if the group already exists before running groupadd RUN getent group $GID || groupadd -g $GID -o $DOCKER_USER && \ @@ -260,11 +210,11 @@ RUN mkdir -p /tmp/gltesting/cargo && mkdir -p /tmp/gltesting/tmp ADD . /repo WORKDIR /repo +ENV UV_LINK_MODE=copy -# Add the remaining repositories to the python-path -RUN poetry install -RUN chmod -R a+rw /tmp/venv +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +RUN uv sync && uv pip install libs/cln-version-manager +RUN uv run --package cln-version-manager clnvm --verbose get-all # Create a symlink to the latest cln-version and add it to the path -RUN ln -s $(clnvm latest --root-path) /opt/cln-latest -ENV PATH=/opt/cln-latest/usr/local/bin:$PATH +RUN ln -s $(uv run --package cln-version-manager latest --root-path) /opt/cln-latest diff --git a/libs/gl-client-py/README.md b/libs/gl-client-py/README.md new file mode 100644 index 000000000..778ac06b9 --- /dev/null +++ b/libs/gl-client-py/README.md @@ -0,0 +1 @@ +# `gl-client`: Greenlight Client Bindings for Python From f58b1d49a14a24527037d5bb7200fd6c482c67f5 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 23 Jan 2025 18:51:30 +0100 Subject: [PATCH 04/15] chore(py): Migrate gl-client to `uv` --- .github/workflows/python.yml | 21 +++++++------ .gitignore | 1 + Makefile | 2 +- docker/gl-testing/Dockerfile | 4 ++- libs/gl-client-py/pyproject.toml | 51 +++++++++----------------------- 5 files changed, 29 insertions(+), 50 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index ad820ad64..b2d9491f6 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -24,17 +24,19 @@ jobs: with: python-version: 3.9 architecture: x64 + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v5 + with: + version: "latest" - - run: pip3 install poetry - - run: | - cd libs/gl-client-py - poetry build --format=sdist + - name: Build Source Distribution + run: uv build --package gl-client --sdist - name: Upload wheels uses: actions/upload-artifact@v4 with: name: wheels-source - path: libs/gl-client-py/dist/gl_client-*.tar.gz + path: dist/gl_client-*.tar.gz linux: runs-on: ubuntu-20.04 @@ -204,17 +206,14 @@ jobs: sudo apt-get update -qq sudo apt-get install python3-pip sudo pip3 install -U \ - poetry \ - maturin \ - twine \ - keyring - + uv + - name: Publish wheels to PyPI env: TWINE_USERNAME: __token__ run: | cd libs/gl-client-py - twine upload \ + uv tool run twine upload \ --skip-existing \ --non-interactive \ --verbose \ diff --git a/.gitignore b/.gitignore index a422a36ac..02133d211 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ hsm_secret site .DS_Store target +*.egg-info # These files are auto generated and may contain sensible data that # we do not want to check-in! diff --git a/Makefile b/Makefile index f4d9431e3..d0bd7763d 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ gen: ${GENALL} build-self: ensure-docker cargo build --all - cd libs/gl-client-py; uv tool run maturin develop --uv + uv sync --package gl-client --extra dev check-all: check-rs check-self check-py check-testing-py diff --git a/docker/gl-testing/Dockerfile b/docker/gl-testing/Dockerfile index 3696e5055..4dcac10bf 100644 --- a/docker/gl-testing/Dockerfile +++ b/docker/gl-testing/Dockerfile @@ -150,7 +150,9 @@ ENV GL_DOCKER=1 # Tells cln to use shorter polling intervals ENV DEVELOPER=1 ENV PYTHONUNBUFFERED=1 -ENV PATH=$HOME/.local/bin:/opt/cln-latest/usr/local/bin:/opt/bitcoin/bin:/opt/cln-latest/usr/local/bin:$PATH +# Start by setting the PATH once and for all, they might not be there +# yet, but we will put the tools in those directories. +ENV PATH=$HOME/.local/bin:/opt/cln-latest/usr/local/bin:/opt/bitcoin/bin:/opt/cln-latest/usr/local/bin:/usr/local/bin:$PATH ENV UV_INSTALL_DIR=/usr/local/bin/ # grpcio == 1.46 produces spammy log messages, silence them ENV GRPC_ENABLE_FORK_SUPPORT=0 diff --git a/libs/gl-client-py/pyproject.toml b/libs/gl-client-py/pyproject.toml index 862904816..dc75a6829 100644 --- a/libs/gl-client-py/pyproject.toml +++ b/libs/gl-client-py/pyproject.toml @@ -1,53 +1,30 @@ [project] name = "gl-client" -version = "0.3.0" -license = "MIT" -description = "Python bindings to the Greenlight service" +version = "0.3.1" +description = "Add your description here" readme = "README.md" requires-python = ">=3.8" - -authors = [ - {name = "Christian Decker", email = "decker@blockstream.com"}, -] - dependencies = [ - "protobuf>=3", "grpcio>=1.56", + "protobuf>=4", "pyln-grpc-proto>=0.1", ] - -[tool.poetry] -name = "gl-client" -version = "0.3.0" -description = "" - -packages = [ - { include = "glclient" }, -] - -include = [ - { path = "glclient/*.pyi", format = ["wheel", "sdist"] }, - { path = "glclient/py.typed", format = ["wheel", "sdist"] }, +authors = [ + {name = "Christian Decker", email = "decker@blockstream.com"}, ] - -[tool.poetry.group.dev.dependencies] -black = "^23.1.0" -mypy-protobuf = "^3.5" -maturin = {version = ">=1.0,<1.3.2", extras = ["patchelf"]} -mypy = "^1.7.0" -grpcio-tools = "^1.67" - -[tool.poetry.dependencies] -python = ">=3.8,<4" -grpcio = ">=1.67" -pyln-grpc-proto = ">=0.1.2,<1.2" -protobuf = ">=4" -maturin = ">=1.0" - [build-system] requires = ["maturin>=1.0"] build-backend = "maturin" +[dependency-groups] +dev = [ + "grpcio>=1.67", + "maturin>=1", + "mypy>=1.14.1", + "protobuf>=4", + "pyln-grpc-proto>=0.1.2", +] + [tool.mypy] exclude = [ 'glclient/greenlight_pb2.py', From 5d824e745dbb61e2958420bbfbf90ed27021db1d Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 24 Jan 2025 18:47:16 +0100 Subject: [PATCH 05/15] py: Add clnvm to `uv` --- docker/gl-testing/Dockerfile | 4 ++-- libs/cln-version-manager/pyproject.toml | 18 +++++++++++------- libs/gl-client-py/Makefile | 4 ++-- libs/gl-client-py/pyproject.toml | 6 ++++++ libs/gl-testing/pyproject.toml | 14 ++++++++++++-- pyproject.toml | 3 ++- 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/docker/gl-testing/Dockerfile b/docker/gl-testing/Dockerfile index 4dcac10bf..b67d38084 100644 --- a/docker/gl-testing/Dockerfile +++ b/docker/gl-testing/Dockerfile @@ -215,8 +215,8 @@ WORKDIR /repo ENV UV_LINK_MODE=copy RUN curl -LsSf https://astral.sh/uv/install.sh | sh -RUN uv sync && uv pip install libs/cln-version-manager -RUN uv run --package cln-version-manager clnvm --verbose get-all +RUN uv sync --all-packages --extra dev +RUN uv run python3 -m clnvm --verbose get-all # Create a symlink to the latest cln-version and add it to the path RUN ln -s $(uv run --package cln-version-manager latest --root-path) /opt/cln-latest diff --git a/libs/cln-version-manager/pyproject.toml b/libs/cln-version-manager/pyproject.toml index 83e465ee9..b12804f9f 100644 --- a/libs/cln-version-manager/pyproject.toml +++ b/libs/cln-version-manager/pyproject.toml @@ -1,13 +1,12 @@ -[tool.poetry] +[project] name = "cln-version-manager" version = "0.1.1" description = "A version manager for Core Lightning Binaries" -authors = ["Erik De Smedt "] license = "MIT" readme = "README.md" - -packages = [ - { include = "clnvm" }, +authors = [ + {name = "Christian Decker", email = "decker@blockstream.com"}, + {name = "Erik De Smedt", email = "edesmedt@blockstream.com"}, ] [tool.poetry.dependencies] @@ -29,5 +28,10 @@ python-lsp-server = "^1.10.0" clnvm = "clnvm.cli:run" [build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = [ + "clnvm", +] \ No newline at end of file diff --git a/libs/gl-client-py/Makefile b/libs/gl-client-py/Makefile index 79ab98382..c1a120ee3 100644 --- a/libs/gl-client-py/Makefile +++ b/libs/gl-client-py/Makefile @@ -40,8 +40,8 @@ pygrpc: ${PROTOSRC} cd ${PYDIR}; poetry run python -m grpc_tools.protoc ${PYPROTOC_OPTS} glclient/greenlight.proto check-py: - cd ${PYDIR}; mypy glclient - cd libs/gl-client-py; pytest tests -n $(shell nproc) + #uv run --all-packages mypy ${PYDIR}/glclient + uv run --all-packages pytest tests -n $(shell nproc) ${PYDIR}/tests clean-py: rm -f ${PYPROTOS} ${PYDIR}/build ${PYDIR}/dist diff --git a/libs/gl-client-py/pyproject.toml b/libs/gl-client-py/pyproject.toml index dc75a6829..5035bf9e4 100644 --- a/libs/gl-client-py/pyproject.toml +++ b/libs/gl-client-py/pyproject.toml @@ -5,6 +5,7 @@ description = "Add your description here" readme = "README.md" requires-python = ">=3.8" dependencies = [ + "gl-testing", "grpcio>=1.56", "protobuf>=4", "pyln-grpc-proto>=0.1", @@ -18,11 +19,13 @@ build-backend = "maturin" [dependency-groups] dev = [ + "gl-testing", "grpcio>=1.67", "maturin>=1", "mypy>=1.14.1", "protobuf>=4", "pyln-grpc-proto>=0.1.2", + "types-protobuf>=5.29.1.20241207", ] [tool.mypy] @@ -35,3 +38,6 @@ exclude = [ [[tool.mypy.overrides]] module = 'glclient' + +[tool.uv.sources] +gl-testing = { workspace = true } diff --git a/libs/gl-testing/pyproject.toml b/libs/gl-testing/pyproject.toml index 97e82d2dc..eb17ce0cf 100644 --- a/libs/gl-testing/pyproject.toml +++ b/libs/gl-testing/pyproject.toml @@ -5,7 +5,7 @@ description = "" readme = "README.md" requires-python = ">=3.8" dependencies = [ -# "cln-version-manager", + # "cln-version-manager", "flaky>=3.8.1", "gl-client", "grpcio-tools>=1.66", @@ -19,6 +19,7 @@ dependencies = [ "rich>=13.9.3", "sh>=1.14.3", "sonora>=0.2.3", + "cln-version-manager", ] @@ -27,4 +28,13 @@ package = true [tool.uv.sources] gl-client = { workspace = true } -#cln-version-manager = { path = "../cln-version-manager" } +cln-version-manager = { workspace = true } + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = [ + "gltesting", +] diff --git a/pyproject.toml b/pyproject.toml index 97551796a..db9cc60b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ package = false members = [ "libs/gl-testing", "libs/gl-client-py", -# "libs/gl-testserver" +# "libs/gl-testserver", + "libs/cln-version-manager", ] exclude = ["packages/seeds"] From 389db57a304c4e9a8b1a8abf3908a841e972c303 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sat, 25 Jan 2025 12:54:33 +0100 Subject: [PATCH 06/15] py(test): Do not assume an absolute path for the trmp hook plugin --- libs/gl-client-py/tests/test_plugin.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libs/gl-client-py/tests/test_plugin.py b/libs/gl-client-py/tests/test_plugin.py index ad891c862..ad8b3cd34 100644 --- a/libs/gl-client-py/tests/test_plugin.py +++ b/libs/gl-client-py/tests/test_plugin.py @@ -3,6 +3,9 @@ from pyln import grpc as clnpb import pytest import secrets +from pathlib import Path + +trmp_plugin_path = Path(__file__).parent / "plugins" / "trmp_htlc_hook.py" def test_big_size_requests(clients): @@ -56,7 +59,7 @@ def test_trampoline_pay(bitcoind, clients, node_factory): # Fund channel between nodes. l2 = node_factory.get_node( options={ - "plugin": "/repo/libs/gl-client-py/tests/plugins/trmp_htlc_hook.py", + "plugin": trmp_plugin_path, } ) n1.connect_peer(l2.info["id"], f"localhost:{l2.port}") @@ -158,7 +161,7 @@ def test_trampoline_multi_htlc(bitcoind, clients, node_factory): # Fund channel between nodes. l2 = node_factory.get_node( options={ - "plugin": "/repo/libs/gl-client-py/tests/plugins/trmp_htlc_hook.py", + "plugin": trmp_plugin_path, } ) n1.connect_peer(l2.info["id"], f"localhost:{l2.port}") From 895199bc0d944bc0b9733201779190075183c214 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sat, 25 Jan 2025 14:17:10 +0100 Subject: [PATCH 07/15] clnvm(ci): Move CI operations to Taskfile This should eventually allow us to replay the CI locally without talking to GH. Should shorten the CI debugging cycle considerably. --- .github/workflows/check-self.yml | 32 ++++++++++---- .github/workflows/cln-version-manager-py.yml | 45 ++++++-------------- Taskfile.yml | 23 ++++++++++ docker/.tasks.yml | 25 +++++++++++ docker/Taskfile.yml | 25 +++++++++++ docker/gl-testing/Dockerfile | 16 +++---- libs/cln-version-manager/.tasks.yml | 11 +++++ libs/cln-version-manager/Taskfile.yml | 11 +++++ libs/cln-version-manager/clnvm/cli.py | 7 ++- libs/cln-version-manager/mypy.ini | 1 + libs/cln-version-manager/pyproject.toml | 7 ++- libs/gl-client-py/.tasks.yml | 10 +++++ libs/gl-client-py/Taskfile.yml | 10 +++++ 13 files changed, 170 insertions(+), 53 deletions(-) create mode 100644 Taskfile.yml create mode 100644 docker/.tasks.yml create mode 100644 docker/Taskfile.yml create mode 100644 libs/cln-version-manager/.tasks.yml create mode 100644 libs/cln-version-manager/Taskfile.yml create mode 100644 libs/gl-client-py/.tasks.yml create mode 100644 libs/gl-client-py/Taskfile.yml diff --git a/.github/workflows/check-self.yml b/.github/workflows/check-self.yml index e3b7982a5..142c0661a 100644 --- a/.github/workflows/check-self.yml +++ b/.github/workflows/check-self.yml @@ -18,10 +18,23 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v3 - - name: Build tester image - run: | - sudo chmod a+rwx -R libs/gl-client-py - make docker-image + + - name: Install Task + uses: arduino/setup-task@v2 + + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v5 + + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Install Protoc + uses: arduino/setup-protoc@v3 + + - name: Install dependencies + run: task docker:install-cfssl + - name: Rust Cache uses: actions/cache@v2 with: @@ -29,10 +42,11 @@ jobs: /tmp/gltesting/cargo /tmp/gltesting/target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Check Self - run : | - PYTEST_OPTS='-n 8' make docker-check-self - - name: Run python image + - name: Check Self + env: + GL_TESTING_IGNORE_HASH: 1 + PATH: /home/runner/.cargo/bin:/tmp/bin:${{ PATH }} + PYTEST_OPTS: -n 6 run : | - PYTEST_OPTS='-n 8' make docker-check-py + task clientpy:check diff --git a/.github/workflows/cln-version-manager-py.yml b/.github/workflows/cln-version-manager-py.yml index 0dbbbb383..d2ffc99e0 100644 --- a/.github/workflows/cln-version-manager-py.yml +++ b/.github/workflows/cln-version-manager-py.yml @@ -9,43 +9,26 @@ on: merge_group: push: branches: - - master + - main jobs: source: runs-on: ubuntu-20.04 steps: - # Check out the code - # We use a sparse check-out to ensure we only use the files we need - name: Checkout code uses: actions/checkout@v3 + + - name: Install Task + uses: arduino/setup-task@v2 + + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v5 with: - sparse-checkout: | - libs/cln-version-manager - - name: Move files to toplevel - run: mv ./libs/cln-version-manager/** . - # Install python-3.8 - - name: Install python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - # Load the poetry installation if it still cached - - name: Load cached Poetry installation - id: cached-poetry - uses: actions/cache@v3 - with: - path: ~/.local # the path depends on the OS - key: poetry-0 # increment to reset cache - # Install poetry. By default the poetry-files venv is created - # in ~/.cache/virtualenvs - - name: Install poetry - if: steps.cached-poetry.outputs.cache-hit != 'true' - uses: snok/install-poetry@v1 - with: - version: 1.8.2 - - name: Install dependencies - run: poetry update && poetry install --no-interaction - - name: Run mypy - run: poetry run mypy + version: "latest" + enable-cache: true + + - name: Build + run: task clnvm:build + - name: Run tests - run: poetry run pytest tests + run: task clnvm:check diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 000000000..09ffdda9b --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,23 @@ +version: '3' + +includes: + clnvm: + taskfile: "libs/cln-version-manager/Taskfile.yml" + dir: "libs/cln-version-manager/" + + clientpy: + taskfile: "libs/gl-client-py/Taskfile.yml" + dir: "libs/gl-client-py/" + + docker: + taskfile: "docker/Taskfile.yml" + dir: "." + +tasks: + check-py: + dir: "libs/gl-client-py" + env: + GL_TESTING_IGNORE_HASH: 1 + cmds: + - uv run --all-packages mypy glclient + - uv run --all-packages pytest tests diff --git a/docker/.tasks.yml b/docker/.tasks.yml new file mode 100644 index 000000000..94a36d60b --- /dev/null +++ b/docker/.tasks.yml @@ -0,0 +1,25 @@ +version: "3" + +tasks: + install-protoc: + dir: '/tmp' + env: + PROTOC_VERSION: 3.19.3 + cmds: + - wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --quiet -O protoc.zip + - unzip protoc.zip + - mkdir -p {{ .HOME }}/bin/ + - mv bin/protoc {{ .HOME }}/bin/ + - rm protoc.zip + + install-cfssl: + env: + BASE: "https://github.com/cloudflare/cfssl/releases/download" + VERSION: 1.6.5 + dir: /tmp/ + cmds: + - mkdir -p bin/ + - wget ${BASE}/v${VERSION}/cfssl_${VERSION}_linux_amd64 -O bin/cfssl + - wget ${BASE}/v${VERSION}/cfssljson_${VERSION}_linux_amd64 -O bin/cfssljson + - chmod a+x /tmp/bin/cfssljson + - chmod a+x /tmp/bin/cfssl diff --git a/docker/Taskfile.yml b/docker/Taskfile.yml new file mode 100644 index 000000000..d75c8a7fa --- /dev/null +++ b/docker/Taskfile.yml @@ -0,0 +1,25 @@ +version: "3" + +tasks: + install-protoc: + dir: '/tmp' + env: + PROTOC_VERSION: 3.19.3 + cmds: + - wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --quiet -O protoc.zip + - unzip protoc.zip + - mkdir -p {{ .HOME }}/bin/ + - mv bin/protoc {{ .HOME }}/bin/ + - rm protoc.zip + + install-cfssl: + env: + BASE: "https://github.com/cloudflare/cfssl/releases/download/" + VERSION: 1.5.0 + dir: /tmp/ + cmds: + - mkdir bin/ + - wget -q {{ .BASE }}/{{ .VERSION }}/cfssl_{{ .VERSION }}_linux_amd64 -O bin/cfssl + - wget -q {{ .BASE }}/{{ .VERSION }}/cfssljson_{{ .VERSION }}_linux_amd64 -O bin/cfssljson + - chmod a+x /tmp/bin/cfssljson + - chmod a+x /tmp/bin/cfssl diff --git a/docker/gl-testing/Dockerfile b/docker/gl-testing/Dockerfile index b67d38084..028e7de8c 100644 --- a/docker/gl-testing/Dockerfile +++ b/docker/gl-testing/Dockerfile @@ -168,6 +168,11 @@ ENV CLN_PATH=/opt/cln/v0.10.1/usr/local/bin/:/opt/cln/v0.10.2/usr/local/bin/:/op ARG GID=0 ARG UID=0 ARG DOCKER_USER=dev +ENV UV_LINK_MODE=copy +# Configure rust and related logging. Set up a default logging +# filter. This includes all trace-level messages from greenlight +# components and validating lightning signer +ENV RUST_LOG=gl_client=trace,tracing=warn,gl_signerproxy=trace,gl_plugin=trace,lightning_signer=trace,vls_protocol_signer=trace,vls_core=trace,vls_persist=trace,vls_protocol=trace,info # Required dependencies and some dependencies which are nice to have # floating around @@ -186,9 +191,6 @@ RUN apt-get update && apt install -qqy \ wget \ && rm -rf /var/lib/apt/lists/* -# Configure rust and related logging -# Set up a default logging filter. This includes all trace-level messages from greenlight components and validating lightning signer -ENV RUST_LOG=gl_client=trace,tracing=warn,gl_signerproxy=trace,gl_plugin=trace,lightning_signer=trace,vls_protocol_signer=trace,vls_core=trace,vls_persist=trace,vls_protocol=trace,info # Install bitcoin-core COPY --from=bitcoin-downloader /opt/bitcoin/bin /opt/bitcoin/bin @@ -200,22 +202,20 @@ COPY --from=cfssl-downloader /usr/bin/cfssljson /usr/local/bin/cfssljson # Install protoc COPY --from=protoc-downloader /usr/local/bin/protoc /usr/local/bin/protoc - # Check if the group already exists before running groupadd RUN getent group $GID || groupadd -g $GID -o $DOCKER_USER && \ id -u $UID || useradd -m -u $UID -g $GID -G sudo -o -s /bin/bash $DOCKER_USER && \ echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -# Create the required tmp-dicts -RUN chmod a+rw /tmp RUN mkdir -p /tmp/gltesting/cargo && mkdir -p /tmp/gltesting/tmp ADD . /repo WORKDIR /repo -ENV UV_LINK_MODE=copy -RUN curl -LsSf https://astral.sh/uv/install.sh | sh +# Populate `uv` cache RUN uv sync --all-packages --extra dev + +# Populate CLN versions RUN uv run python3 -m clnvm --verbose get-all # Create a symlink to the latest cln-version and add it to the path diff --git a/libs/cln-version-manager/.tasks.yml b/libs/cln-version-manager/.tasks.yml new file mode 100644 index 000000000..3b6526854 --- /dev/null +++ b/libs/cln-version-manager/.tasks.yml @@ -0,0 +1,11 @@ +version: '3' + +tasks: + check: + cmds: + - uv run --extra dev mypy clnvm + - uv run --extra dev pytest tests -vvv -s + + build: + cmds: + - uv build --sdist diff --git a/libs/cln-version-manager/Taskfile.yml b/libs/cln-version-manager/Taskfile.yml new file mode 100644 index 000000000..3b6526854 --- /dev/null +++ b/libs/cln-version-manager/Taskfile.yml @@ -0,0 +1,11 @@ +version: '3' + +tasks: + check: + cmds: + - uv run --extra dev mypy clnvm + - uv run --extra dev pytest tests -vvv -s + + build: + cmds: + - uv build --sdist diff --git a/libs/cln-version-manager/clnvm/cli.py b/libs/cln-version-manager/clnvm/cli.py index f7af082bd..b78ebfd99 100644 --- a/libs/cln-version-manager/clnvm/cli.py +++ b/libs/cln-version-manager/clnvm/cli.py @@ -15,15 +15,14 @@ def configure_logging() -> None: logging.config.fileConfig(fname) # Handle the optional import and provide a nice error message if it fails -_click = importlib.util.find_spec("click") -if _click is None: +try: + import click +except Exception: print("To use clnvm the `cli` feature must be installed") print("You can install the feature using") print("> pip install gltesting[cli]") sys.exit(1) -import click - @click.group() @click.option("--verbose", is_flag=True) diff --git a/libs/cln-version-manager/mypy.ini b/libs/cln-version-manager/mypy.ini index 3c0b9107c..367823110 100644 --- a/libs/cln-version-manager/mypy.ini +++ b/libs/cln-version-manager/mypy.ini @@ -4,3 +4,4 @@ files=clnvm,tests warn_return_any = True warn_unused_configs = True disallow_untyped_defs = True +ignore_missing_imports = True diff --git a/libs/cln-version-manager/pyproject.toml b/libs/cln-version-manager/pyproject.toml index b12804f9f..aa2a580a6 100644 --- a/libs/cln-version-manager/pyproject.toml +++ b/libs/cln-version-manager/pyproject.toml @@ -31,7 +31,12 @@ clnvm = "clnvm.cli:run" requires = ["hatchling"] build-backend = "hatchling.build" +[dependency-groups] +dev = [ + "types-requests>=2.31.0.6", +] + [tool.hatch.build] include = [ "clnvm", -] \ No newline at end of file +] diff --git a/libs/gl-client-py/.tasks.yml b/libs/gl-client-py/.tasks.yml new file mode 100644 index 000000000..5e31a3bd4 --- /dev/null +++ b/libs/gl-client-py/.tasks.yml @@ -0,0 +1,10 @@ +version: '3' + +tasks: + check: + cmds: + - uv run pytest -vvv tests + + build: + cmds: + - uv build --sdist diff --git a/libs/gl-client-py/Taskfile.yml b/libs/gl-client-py/Taskfile.yml new file mode 100644 index 000000000..5e31a3bd4 --- /dev/null +++ b/libs/gl-client-py/Taskfile.yml @@ -0,0 +1,10 @@ +version: '3' + +tasks: + check: + cmds: + - uv run pytest -vvv tests + + build: + cmds: + - uv build --sdist From e2c172991c26bd2170e797e94087f2a70c4e1955 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sun, 26 Jan 2025 09:57:53 +0100 Subject: [PATCH 08/15] chore(py): Remove pointless non-dev gl-testing dependency gl-client-py should only depend on gl-testing for... testing. --- libs/gl-client-py/pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/gl-client-py/pyproject.toml b/libs/gl-client-py/pyproject.toml index 5035bf9e4..121549ae2 100644 --- a/libs/gl-client-py/pyproject.toml +++ b/libs/gl-client-py/pyproject.toml @@ -5,8 +5,6 @@ description = "Add your description here" readme = "README.md" requires-python = ">=3.8" dependencies = [ - "gl-testing", - "grpcio>=1.56", "protobuf>=4", "pyln-grpc-proto>=0.1", ] From e06c1465494286af34004a9f711c67537bcaab11 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sun, 26 Jan 2025 12:07:13 +0100 Subject: [PATCH 09/15] chore(ci): Re-add the gl-testserver to the workspace --- Taskfile.yml | 29 ++++++++++++++++++--------- docker/Taskfile.yml | 25 ----------------------- libs/cln-version-manager/Taskfile.yml | 11 ---------- libs/gl-client-py/Taskfile.yml | 10 --------- libs/gl-testserver/pyproject.toml | 4 ++-- pyproject.toml | 2 +- 6 files changed, 23 insertions(+), 58 deletions(-) delete mode 100644 docker/Taskfile.yml delete mode 100644 libs/cln-version-manager/Taskfile.yml delete mode 100644 libs/gl-client-py/Taskfile.yml diff --git a/Taskfile.yml b/Taskfile.yml index 09ffdda9b..fc1a05a65 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -2,22 +2,33 @@ version: '3' includes: clnvm: - taskfile: "libs/cln-version-manager/Taskfile.yml" + taskfile: "libs/cln-version-manager/.tasks.yml" dir: "libs/cln-version-manager/" clientpy: - taskfile: "libs/gl-client-py/Taskfile.yml" + taskfile: "libs/gl-client-py/.tasks.yml" dir: "libs/gl-client-py/" docker: - taskfile: "docker/Taskfile.yml" + taskfile: "docker/.tasks.yml" dir: "." tasks: - check-py: - dir: "libs/gl-client-py" - env: - GL_TESTING_IGNORE_HASH: 1 + + ci-check: + deps: + - task: clientpy:check + - task: clnvm:check + + docker-ci-check: cmds: - - uv run --all-packages mypy glclient - - uv run --all-packages pytest tests + - > + docker buildx build \ + --load \ + --build-arg DOCKER_USER=$(whoami) \ + --build-arg UID=$(id -u) \ + --build-arg GID=$(id -g) \ + --build-arg GL_TESTING_IGNORE_HASH=1 \ + -t gltesting \ + -f docker/gl-testing/Dockerfile \ + . diff --git a/docker/Taskfile.yml b/docker/Taskfile.yml deleted file mode 100644 index d75c8a7fa..000000000 --- a/docker/Taskfile.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: "3" - -tasks: - install-protoc: - dir: '/tmp' - env: - PROTOC_VERSION: 3.19.3 - cmds: - - wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --quiet -O protoc.zip - - unzip protoc.zip - - mkdir -p {{ .HOME }}/bin/ - - mv bin/protoc {{ .HOME }}/bin/ - - rm protoc.zip - - install-cfssl: - env: - BASE: "https://github.com/cloudflare/cfssl/releases/download/" - VERSION: 1.5.0 - dir: /tmp/ - cmds: - - mkdir bin/ - - wget -q {{ .BASE }}/{{ .VERSION }}/cfssl_{{ .VERSION }}_linux_amd64 -O bin/cfssl - - wget -q {{ .BASE }}/{{ .VERSION }}/cfssljson_{{ .VERSION }}_linux_amd64 -O bin/cfssljson - - chmod a+x /tmp/bin/cfssljson - - chmod a+x /tmp/bin/cfssl diff --git a/libs/cln-version-manager/Taskfile.yml b/libs/cln-version-manager/Taskfile.yml deleted file mode 100644 index 3b6526854..000000000 --- a/libs/cln-version-manager/Taskfile.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: '3' - -tasks: - check: - cmds: - - uv run --extra dev mypy clnvm - - uv run --extra dev pytest tests -vvv -s - - build: - cmds: - - uv build --sdist diff --git a/libs/gl-client-py/Taskfile.yml b/libs/gl-client-py/Taskfile.yml deleted file mode 100644 index 5e31a3bd4..000000000 --- a/libs/gl-client-py/Taskfile.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '3' - -tasks: - check: - cmds: - - uv run pytest -vvv tests - - build: - cmds: - - uv build --sdist diff --git a/libs/gl-testserver/pyproject.toml b/libs/gl-testserver/pyproject.toml index e6eac8497..8f95b747d 100644 --- a/libs/gl-testserver/pyproject.toml +++ b/libs/gl-testserver/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.8" dependencies = [ "click>=8.1.7", - "gltesting", + "gl-testing", "rich>=13.9.3", ] @@ -14,4 +14,4 @@ dependencies = [ gltestserver = 'gltestserver.__main__:cli' [tool.uv.sources] -gltesting = { workspace = true } +gl-testing = { workspace = true } diff --git a/pyproject.toml b/pyproject.toml index db9cc60b0..fb189dbb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ package = false members = [ "libs/gl-testing", "libs/gl-client-py", -# "libs/gl-testserver", + "libs/gl-testserver", "libs/cln-version-manager", ] exclude = ["packages/seeds"] From d5a409bd8af54c4e0a2be2f98d00d7d0b6aeb610 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sun, 26 Jan 2025 12:19:46 +0100 Subject: [PATCH 10/15] chore(ci): Add GITHUB token for go-task --- .github/workflows/check-self.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-self.yml b/.github/workflows/check-self.yml index 142c0661a..cef368c8e 100644 --- a/.github/workflows/check-self.yml +++ b/.github/workflows/check-self.yml @@ -21,6 +21,8 @@ jobs: - name: Install Task uses: arduino/setup-task@v2 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Install the latest version of uv uses: astral-sh/setup-uv@v5 @@ -35,6 +37,13 @@ jobs: - name: Install dependencies run: task docker:install-cfssl + - name: Adjust PATH + run: | + echo "/home/runner/.cargo/bin" >> $GITHUB_PATH + # Directory in which go-task deposits its binaries + echo $(pwd)/bin >> $GITHUB_PATH + echo "/tmp/bin/" >> $GITHUB_PATH + - name: Rust Cache uses: actions/cache@v2 with: @@ -46,7 +55,6 @@ jobs: - name: Check Self env: GL_TESTING_IGNORE_HASH: 1 - PATH: /home/runner/.cargo/bin:/tmp/bin:${{ PATH }} PYTEST_OPTS: -n 6 run : | task clientpy:check From 4f17a7e363578ac5ea1298653c26b7b82aec77b0 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sun, 26 Jan 2025 13:43:01 +0100 Subject: [PATCH 11/15] clnvm(ci): Add missing mypy dependency --- docker/gl-testing/Dockerfile | 2 ++ libs/cln-version-manager/pyproject.toml | 1 + 2 files changed, 3 insertions(+) diff --git a/docker/gl-testing/Dockerfile b/docker/gl-testing/Dockerfile index 028e7de8c..78422b834 100644 --- a/docker/gl-testing/Dockerfile +++ b/docker/gl-testing/Dockerfile @@ -208,6 +208,8 @@ RUN getent group $GID || groupadd -g $GID -o $DOCKER_USER && \ echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers RUN mkdir -p /tmp/gltesting/cargo && mkdir -p /tmp/gltesting/tmp +RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d +RUN curl -LsSf https://astral.sh/uv/install.sh | sh ADD . /repo WORKDIR /repo diff --git a/libs/cln-version-manager/pyproject.toml b/libs/cln-version-manager/pyproject.toml index aa2a580a6..2f5f9e6ad 100644 --- a/libs/cln-version-manager/pyproject.toml +++ b/libs/cln-version-manager/pyproject.toml @@ -33,6 +33,7 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ + "mypy>=1.14.1", "types-requests>=2.31.0.6", ] From f3ac1a2e067fb6e13cbb8eda68997c29354fcc2a Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sun, 26 Jan 2025 13:44:26 +0100 Subject: [PATCH 12/15] ci: Add bitcoind download --- .github/workflows/check-self.yml | 2 +- Taskfile.yml | 1 + docker/.tasks.yml | 37 ++++++++++++++++++++++++++------ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check-self.yml b/.github/workflows/check-self.yml index cef368c8e..1dd5a8f51 100644 --- a/.github/workflows/check-self.yml +++ b/.github/workflows/check-self.yml @@ -35,7 +35,7 @@ jobs: uses: arduino/setup-protoc@v3 - name: Install dependencies - run: task docker:install-cfssl + run: task docker:install-dependencies - name: Adjust PATH run: | diff --git a/Taskfile.yml b/Taskfile.yml index fc1a05a65..6942cc110 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -32,3 +32,4 @@ tasks: -t gltesting \ -f docker/gl-testing/Dockerfile \ . + - docker run -ti -v $(pwd):/repo gltesting task ci-check diff --git a/docker/.tasks.yml b/docker/.tasks.yml index 94a36d60b..fe6bbb639 100644 --- a/docker/.tasks.yml +++ b/docker/.tasks.yml @@ -1,25 +1,50 @@ version: "3" tasks: + install-dependencies: + vars: + BINDIR: '{{.BINDIR | default "{{ .HOME }}/bin"}}' + deps: + - install-cfssl + - install-bitcoind + install-protoc: dir: '/tmp' env: PROTOC_VERSION: 3.19.3 + BINDIR: '{{.BINDIR | default "bin"}}' cmds: - wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --quiet -O protoc.zip + - defer: rm protoc.zip - unzip protoc.zip - - mkdir -p {{ .HOME }}/bin/ - - mv bin/protoc {{ .HOME }}/bin/ + - mkdir -p ${BINDIR} + - mv bin/protoc ${BINDIR} - rm protoc.zip install-cfssl: env: BASE: "https://github.com/cloudflare/cfssl/releases/download" VERSION: 1.6.5 + BINDIR: '{{.BINDIR | default "bin"}}' dir: /tmp/ cmds: - mkdir -p bin/ - - wget ${BASE}/v${VERSION}/cfssl_${VERSION}_linux_amd64 -O bin/cfssl - - wget ${BASE}/v${VERSION}/cfssljson_${VERSION}_linux_amd64 -O bin/cfssljson - - chmod a+x /tmp/bin/cfssljson - - chmod a+x /tmp/bin/cfssl + - wget -q ${BASE}/v${VERSION}/cfssl_${VERSION}_linux_amd64 -O ${BINDIR}/cfssl + - wget -q ${BASE}/v${VERSION}/cfssljson_${VERSION}_linux_amd64 -O ${BINDIR}/cfssljson + - chmod a+x ${BINDIR}/cfssljson + - chmod a+x ${BINDIR}/cfssl + + install-bitcoind: + dir: + vars: + env: + BITCOIN_VERSION: 28.1 + BINDIR: '{{.BINDIR | default "bin"}}' + cmds: + - mkdir -p {{ .HOME }}/bin + - wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz -O bitcoin.tar.gz + - defer: rm -rf bitcoin.tar.gz /tmp/bitcoin-${BITCOIN_VERSION} + - tar -xvzf bitcoin.tar.gz + - defer: rm -rf bitcoin-${BITCOIN_VERSION} + - mkdir -p ${BINDIR} + - mv bitcoin-${BITCOIN_VERSION}/bin/bitcoin* ${BINDIR} From b53ac212bf30d573c55ad22aefeb45013d31a633 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sun, 26 Jan 2025 16:14:51 +0100 Subject: [PATCH 13/15] ci: Fix some more minor things --- Taskfile.yml | 12 +++++++++++- libs/cln-version-manager/pyproject.toml | 25 +++++++------------------ libs/gl-plugin/.tasks.yml | 6 ++++++ libs/gl-signerproxy/.tasks.yml | 6 ++++++ 4 files changed, 30 insertions(+), 19 deletions(-) create mode 100644 libs/gl-plugin/.tasks.yml create mode 100644 libs/gl-signerproxy/.tasks.yml diff --git a/Taskfile.yml b/Taskfile.yml index 6942cc110..8c1cc4b7a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -13,10 +13,20 @@ includes: taskfile: "docker/.tasks.yml" dir: "." + plugin: + taskfile: "libs/gl-plugin/.tasks.yml" + dir: "." + + signerproxy: + taskfile: "libs/gl-signerproxy/.tasks.yml" + dir: "." + tasks: ci-check: - deps: + cmds: + - task: signerproxy:build + - task: plugin:build - task: clientpy:check - task: clnvm:check diff --git a/libs/cln-version-manager/pyproject.toml b/libs/cln-version-manager/pyproject.toml index 2f5f9e6ad..0cf57406f 100644 --- a/libs/cln-version-manager/pyproject.toml +++ b/libs/cln-version-manager/pyproject.toml @@ -8,24 +8,9 @@ authors = [ {name = "Christian Decker", email = "decker@blockstream.com"}, {name = "Erik De Smedt", email = "edesmedt@blockstream.com"}, ] - -[tool.poetry.dependencies] -python = "^3.8" -requests = "^2" -click = "^8" - -[tool.poetry.group.dev.dependencies] -types-requests = "^2.31" -pytest = "^8" -mypy = "^1.8.0" -types-click = "^7" -black = "^24.2.0" - -[tool.poetry.group.lsp_ide.dependencies] -python-lsp-server = "^1.10.0" - -[tool.poetry.scripts] -clnvm = "clnvm.cli:run" +dependencies = [ + "requests>=2.32.3", +] [build-system] requires = ["hatchling"] @@ -34,6 +19,7 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ "mypy>=1.14.1", + "pytest>=7.4.4", "types-requests>=2.31.0.6", ] @@ -41,3 +27,6 @@ dev = [ include = [ "clnvm", ] + +[project.scripts] +hello = "clnvm.cli:run" \ No newline at end of file diff --git a/libs/gl-plugin/.tasks.yml b/libs/gl-plugin/.tasks.yml new file mode 100644 index 000000000..2a7de4ba2 --- /dev/null +++ b/libs/gl-plugin/.tasks.yml @@ -0,0 +1,6 @@ +version: '3' + +tasks: + build: + cmds: + - cargo build diff --git a/libs/gl-signerproxy/.tasks.yml b/libs/gl-signerproxy/.tasks.yml new file mode 100644 index 000000000..2a7de4ba2 --- /dev/null +++ b/libs/gl-signerproxy/.tasks.yml @@ -0,0 +1,6 @@ +version: '3' + +tasks: + build: + cmds: + - cargo build From ad9a353461a00920a901baeb90ea3ac32fc4ba3a Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sun, 26 Jan 2025 16:30:50 +0100 Subject: [PATCH 14/15] chore(py): Fix a broken path to the target dir --- libs/gl-testing/gltesting/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/gl-testing/gltesting/node.py b/libs/gl-testing/gltesting/node.py index 0f6f73578..23fcf890a 100644 --- a/libs/gl-testing/gltesting/node.py +++ b/libs/gl-testing/gltesting/node.py @@ -18,7 +18,7 @@ target_dir = Path( os.environ.get( 'CARGO_TARGET_DIR', - Path(__file__).parent / ".." / ".." / "target" + Path(__file__).parent / ".." / ".." / ".." / "target" ) ) From 87d379aadfc4b5f0bd2dd18cbca79f526c042022 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 27 Jan 2025 11:17:50 +0100 Subject: [PATCH 15/15] ci: Add build step before testing --- .github/workflows/check-self.yml | 6 +++++- Taskfile.yml | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-self.yml b/.github/workflows/check-self.yml index 1dd5a8f51..e57c02042 100644 --- a/.github/workflows/check-self.yml +++ b/.github/workflows/check-self.yml @@ -45,13 +45,17 @@ jobs: echo "/tmp/bin/" >> $GITHUB_PATH - name: Rust Cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: | /tmp/gltesting/cargo /tmp/gltesting/target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Build binaries + run: + task ci-build + - name: Check Self env: GL_TESTING_IGNORE_HASH: 1 diff --git a/Taskfile.yml b/Taskfile.yml index 8c1cc4b7a..c8186dd16 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -24,9 +24,9 @@ includes: tasks: ci-check: + deps: + - ci-build cmds: - - task: signerproxy:build - - task: plugin:build - task: clientpy:check - task: clnvm:check @@ -43,3 +43,8 @@ tasks: -f docker/gl-testing/Dockerfile \ . - docker run -ti -v $(pwd):/repo gltesting task ci-check + + ci-build: + cmds: + - task: signerproxy:build + - task: plugin:build