diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7dda61720455..1876f0d73f7d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,9 +11,8 @@ concurrency: cancel-in-progress: true env: - # Makes the upload-artifact work more reliably at the cost - # of a bit of compile time. - RUST_PROFILE: release + # Reduce size, helps upload-artifact work more reliably + RUST_PROFILE: small SLOW_MACHINE: 1 CI_SERVER_URL: "http://35.239.136.52:3170" PYTEST_OPTS_BASE: "-vvv --junit-xml=report.xml --timeout=1800 --durations=10" @@ -82,6 +81,8 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh # We're going to check BOLT quotes, so get the latest version @@ -93,7 +94,7 @@ jobs: VALGRIND: 0 PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} run: | - uv run make check-source BASE_REF="origin/${{ github.base_ref }}" + uv run make -j $(nproc) check-source BASE_REF="origin/${{ github.base_ref }}" CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1 - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -101,10 +102,6 @@ jobs: name: pytest-results-prebuild path: report.xml if-no-files-found: ignore - - name: Check Generated Files have been updated - run: uv run make check-gen-updated - - name: Check docs - run: uv run make check-doc compile: name: Compile CLN ${{ matrix.cfg }} @@ -145,6 +142,8 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh @@ -155,21 +154,62 @@ jobs: UBSAN: ${{ matrix.UBSAN }} VALGRIND: ${{ matrix.VALGRIND }} COMPAT: 1 - CFG: ${{ matrix.CFG }} run: | set -e ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }} - uv run make -j $(nproc) testpack.tar.bz2 + uv run make -j $(nproc) testpack.tar.gz # Rename now so we don't clash - mv testpack.tar.bz2 cln-${CFG}.tar.bz2 - - name: Check rust packages - run: cargo test --all + mv testpack.tar.gz cln-${{ matrix.CFG }}.tar.gz - uses: actions/upload-artifact@v4 with: - name: cln-${{ matrix.CFG }}.tar.bz2 - path: cln-${{ matrix.CFG }}.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz + path: cln-${{ matrix.CFG }}.tar.gz + + check-compiled-source: + runs-on: ubuntu-24.04 + needs: + - compile + strategy: + matrix: + include: + - CFG: compile-gcc + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} + run: | + bash -x .github/scripts/setup.sh + + - name: Download build + uses: actions/download-artifact@v4 + with: + name: cln-${{ matrix.CFG }}.tar.gz + + - name: Unpack prebuilt binaries + run: | + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz + + - name: Check + run: | + uv run eatmydata make -j $(nproc) check-source-bolt check-python check-gen-updated check-doc CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1 + - name: Check rust packages + run: cargo test --all check-units: # The unit test checks are not in the critical path (not dependent @@ -202,21 +242,28 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh - sudo apt-get update -qq # We're going to check BOLT quotes, so get the latest version git clone https://github.com/lightning/bolts.git ../${BOLTDIR} - name: Download build uses: actions/download-artifact@v4 with: - name: cln-${{ matrix.CFG }}.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz + + - name: Unpack prebuilt binaries + run: | + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz - name: Check run: | - tar -xaf cln-${{ matrix.CFG }}.tar.bz2 - uv run eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }} + uv run eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }} CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1 check-fuzz: name: Run fuzz regression tests @@ -236,6 +283,8 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh @@ -248,7 +297,7 @@ jobs: name: Check we can downgrade the node runs-on: ubuntu-24.04 needs: - - compile + - check-compiled-source strategy: fail-fast: false matrix: @@ -276,29 +325,27 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Install bitcoind env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - run: .github/scripts/install-bitcoind.sh + run: | + bash -x .github/scripts/setup.sh - name: Download build uses: actions/download-artifact@v4 with: - name: cln-${{ matrix.CFG }}.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz - - name: Unpack pre-built CLN - env: - CFG: ${{ matrix.CFG }} + - name: Unpack prebuilt binaries run: | - tar -xaf cln-${CFG}.tar.bz2 + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz - name: Fetch and unpack previous CLN run: | - mkdir /tmp/old-cln - cd /tmp/old-cln + mkdir old-cln + cd old-cln wget https://github.com/ElementsProject/lightning/releases/download/v25.12/clightning-v25.12-ubuntu-24.04-amd64.tar.xz tar -xaf clightning-v25.12-ubuntu-24.04-amd64.tar.xz @@ -308,22 +355,22 @@ jobs: # Loading the network from config.vars rather than the envvar is a terrible idea... sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars cat config.vars + touch -d yesterday config.vars - name: Test env: SLOW_MACHINE: 1 - PYTEST_PAR: 4 TEST_DEBUG: 1 TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} TEST_NETWORK: ${{ matrix.TEST_NETWORK }} LIGHTNINGD_POSTGRES_NO_VACUUM: 1 VALGRIND: ${{ matrix.VALGRIND }} - PREV_LIGHTNINGD: /tmp/old-cln/usr/bin/lightningd + PREV_LIGHTNINGD: old-cln/usr/bin/lightningd PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} run: | env cat config.vars - uv run eatmydata pytest tests/test_downgrade.py -n ${PYTEST_PAR} ${PYTEST_OPTS} + uv run eatmydata pytest tests/test_downgrade.py -n $(($(nproc) + 1)) ${PYTEST_OPTS} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -337,7 +384,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 120 env: - RUST_PROFILE: release # Has to match the one in the compile step + RUST_PROFILE: small # Has to match the one in the compile step PYTEST_OPTS: -vvv --junit-xml=report.xml --timeout=1800 --durations=10 needs: - compile @@ -396,24 +443,22 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Install bitcoind env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - run: .github/scripts/install-bitcoind.sh + run: | + bash -x .github/scripts/setup.sh - name: Download build uses: actions/download-artifact@v4 with: - name: cln-${{ matrix.CFG }}.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz - - name: Unpack pre-built CLN - env: - CFG: ${{ matrix.CFG }} + - name: Unpack prebuilt binaries run: | - tar -xaf cln-${CFG}.tar.bz2 + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz - name: Switch network if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }} @@ -428,9 +473,7 @@ jobs: EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }} EXPERIMENTAL_SPLICING: ${{ matrix.EXPERIMENTAL_SPLICING }} COMPAT: 1 - CFG: ${{ matrix.CFG }} SLOW_MACHINE: 1 - PYTEST_PAR: 4 TEST_DEBUG: 1 TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -438,7 +481,7 @@ jobs: run: | env cat config.vars - VALGRIND=0 uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} + VALGRIND=0 uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -452,14 +495,14 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 120 env: - RUST_PROFILE: release # Has to match the one in the compile step - CFG: compile-gcc + RUST_PROFILE: small # Has to match the one in the compile step PYTEST_OPTS: -vvv --junit-xml=report.xml --timeout=1800 --durations=10 --test-group-random-seed=42 needs: - - compile + - check-compiled-source strategy: fail-fast: false matrix: + CFG: [compile-gcc] include: - NAME: Valgrind (01/10) GROUP: 1 @@ -505,8 +548,6 @@ jobs: - name: Install dependencies run: | - sudo apt-get update -qq - sudo apt-get install -yyq valgrind bash -x .github/scripts/setup.sh - name: Install bitcoind @@ -515,18 +556,21 @@ jobs: - name: Download build uses: actions/download-artifact@v4 with: - name: cln-compile-gcc.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz - - name: Unpack build - run: tar -xvjf cln-compile-gcc.tar.bz2 + - name: Unpack prebuilt binaries + run: | + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz - name: Test env: SLOW_MACHINE: 1 TEST_DEBUG: 1 - PYTEST_PAR: 2 run: | - VALGRIND=1 uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} + VALGRIND=1 uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -540,15 +584,16 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 120 env: - RUST_PROFILE: release + RUST_PROFILE: small SLOW_MACHINE: 1 TEST_DEBUG: 1 PYTEST_OPTS: -vvv --junit-xml=report.xml --timeout=1800 --durations=10 --test-group-random-seed=42 needs: - - compile + - check-compiled-source strategy: fail-fast: false matrix: + CFG: [compile-clang-sanitizers] include: - NAME: ASan/UBSan (01/12) GROUP: 1 @@ -599,25 +644,26 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh - - name: Install bitcoind - run: .github/scripts/install-bitcoind.sh - - name: Download build uses: actions/download-artifact@v4 with: - name: cln-compile-clang-sanitizers.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz - - name: Unpack build - run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2 + - name: Unpack prebuilt binaries + run: | + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz - name: Test - env: - PYTEST_PAR: 2 run: | - uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} + uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -633,6 +679,9 @@ jobs: timeout-minutes: 30 strategy: fail-fast: false + matrix: + include: + - CFG: compile-gcc env: VALGRIND: 0 GENERATE_EXAMPLES: 1 @@ -661,13 +710,17 @@ jobs: - name: Download build uses: actions/download-artifact@v4 with: - name: cln-compile-gcc.tar.bz2 - - name: Unpack pre-built CLN + name: cln-${{ matrix.CFG }}.tar.gz + - name: Unpack prebuilt binaries run: | - tar -xaf cln-compile-gcc.tar.bz2 + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz + - name: Test run: | - uv run eatmydata make -j $(nproc) check-doc-examples + uv run eatmydata make -j $(($(nproc) + 1)) check-doc-examples CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1 - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -681,7 +734,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 120 env: - RUST_PROFILE: release # Has to match the one in the compile step + RUST_PROFILE: small # Has to match the one in the compile step PYTEST_OPTS: -vvv --junit-xml=report.xml --timeout=1800 --durations=10 needs: - compile @@ -708,6 +761,8 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh @@ -726,21 +781,20 @@ jobs: - name: Download build uses: actions/download-artifact@v4 with: - name: cln-${{ matrix.CFG }}.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz - - name: Unpack pre-built CLN - env: - CFG: ${{ matrix.CFG }} + - name: Unpack prebuilt binaries run: | - tar -xaf cln-${CFG}.tar.bz2 + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz - name: Test env: COMPILER: ${{ matrix.COMPILER }} COMPAT: 1 - CFG: ${{ matrix.CFG }} SLOW_MACHINE: 1 - PYTEST_PAR: 4 TEST_DEBUG: 1 TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -748,7 +802,7 @@ jobs: run: | env cat config.vars - VALGRIND=0 uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} + VALGRIND=0 uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -771,11 +825,12 @@ jobs: - integration-sanitizers - min-btc-support - check-downgrade + - check-compiled-source if: ${{ always() }} steps: - name: Complete env: - JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC" + JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC CHECK_COMPILED_SOURCE" INTEGRATION: ${{ needs.integration.result }} CHECK_UNITS: ${{ needs['check-units'].result }} VALGRIND: ${{ needs['integration-valgrind'].result }} @@ -783,6 +838,7 @@ jobs: DOCS: ${{ needs['update-docs-examples'].result }} BTC: ${{ needs['min-btc-support'].result }} CHECK_DOWNGRADE: ${{ needs['check-downgrade'].result }} + CHECK_COMPILED_SOURCE: ${{ needs['check-compiled-source'].result }} run: | failed="" for name in $JOB_NAMES; do diff --git a/Cargo.toml b/Cargo.toml index a3b8e55dafa3..ef9ca9e801ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,12 @@ [profile.release] strip = "debuginfo" +[profile.small] +inherits = "dev" +opt-level = 0 +debug = 0 +codegen-units = 32 + [workspace] resolver = "2" members = [ diff --git a/Makefile b/Makefile index e59e9f0b5bc1..fb9bd9110205 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,7 @@ ECHO := echo SUPPRESS_OUTPUT := endif +CARGO := cargo DISTRO=$(shell lsb_release -is 2>/dev/null || echo unknown)-$(shell lsb_release -rs 2>/dev/null || echo unknown) OS=$(shell uname -s) ARCH=$(shell uname -m) @@ -104,6 +105,7 @@ CCAN_OBJS := \ ccan-bitmap.o \ ccan-bitops.o \ ccan-breakpoint.o \ + ccan-cdump.o \ ccan-closefrom.o \ ccan-crc32c.o \ ccan-crypto-hmac.o \ @@ -229,8 +231,6 @@ CCAN_HEADERS := \ $(CCANDIR)/ccan/typesafe_cb/typesafe_cb.h \ $(CCANDIR)/ccan/utf8/utf8.h -CDUMP_OBJS := ccan-cdump.o ccan-strmap.o - BOLT_GEN := tools/generate-wire.py WIRE_GEN := $(BOLT_GEN) @@ -483,10 +483,6 @@ mkdocs.yml: $(MANPAGES:=.md) ) - -# Don't delete these intermediaries. -.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) $(PYTHON_GENERATED) - # Every single object file. ALL_OBJS := $(ALL_C_SOURCES:.c=.o) @@ -649,10 +645,8 @@ update-doc-examples: check-doc-examples: update-doc-examples git diff --exit-code HEAD -# For those without working cppcheck -check-source-no-cppcheck: check-makefile check-source-bolt check-whitespace check-spelling check-python check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access check-bad-sprintf - -check-source: check-source-no-cppcheck +# This should NOT compile things! +check-source: check-makefile check-whitespace check-spelling check-python-flake8 check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access check-bad-sprintf full-check: check check-source @@ -725,7 +719,7 @@ TAGS: tags: $(RM) tags; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 ctags --append -ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS) +ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o libccan.a ALL_PROGRAMS += ccan/ccan/cdump/tools/cdump-enumstr # Can't add to ALL_OBJS, as that makes a circular dep. @@ -747,6 +741,9 @@ endif header_versions_gen.h: tools/headerversions $(FORCE) @tools/headerversions $@ +# Once you have libccan.a, you don't need these. +.INTERMEDIATE: $(CCAN_OBJS) + # We make a static library, this way linker can discard unused parts. libccan.a: $(CCAN_OBJS) @$(call VERBOSE, "ar $@", $(AR) r $@ $(CCAN_OBJS)) @@ -968,25 +965,21 @@ install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $( install: install-program install-data -# Non-artifacts that are needed for testing. These are added to the -# testpack.tar, used to transfer things between builder and tester -# phase. If you get a missing file/executable while testing on CI it -# is likely missing from this variable. -TESTBINS = \ - $(CLN_PLUGIN_EXAMPLES) \ - tests/plugins/test_libplugin \ - tests/plugins/channeld_fakenet \ - tests/plugins/test_selfdisable_after_getmanifest +# We exclude most of target/ and external, but we need: +# 1. config files (we only tar up files *newer* than these) +# 2. $(DEFAULT_TARGETS) for rust stuff. +# 3. $(EXTERNAL_LIBS) for prebuild external libraries. +TESTPACK_EXTRAS := \ + config.vars ccan/config.h \ + header_versions_gen.h \ + $(DEFAULT_TARGETS) \ + $(EXTERNAL_LIBS) # The testpack is used in CI to transfer built artefacts between the -# build and the test phase. This is necessary because the fixtures in -# `tests/` explicitly use the binaries built in the current directory -# rather than using `$PATH`, as that may pick up some other installed -# version of `lightningd` leading to bogus results. We bundle up all -# built artefacts here, and will unpack them on the tester (overlaying -# on top of the checked out repo as if we had just built it in place). -testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS) $(TOOLS) - tar -caf $@ $^ +# build and the test phase. Only useful on a freshly build tree! +# We use Posix format for timestamps with subsecond accuracy. +testpack.tar.gz: all-programs all-fuzz-programs all-test-programs default-targets + (find * -path external -prune -o -path target -prune -o -newer config.vars -type f -print; ls $(TESTPACK_EXTRAS)) | tar --verbatim-files-from -T- -c --format=posix -f - | gzip -5 > $@ uninstall: @$(NORMAL_UNINSTALL) diff --git a/cln-rpc/Makefile b/cln-rpc/Makefile index c43467ab0c94..808dd71cca13 100644 --- a/cln-rpc/Makefile +++ b/cln-rpc/Makefile @@ -9,13 +9,13 @@ DEFAULT_TARGETS += $(CLN_RPC_EXAMPLES) $(CLN_RPC_GENALL) MSGGEN_GENALL += $(CLN_RPC_GENALL) target/${RUST_PROFILE}/examples/cln-rpc-getinfo: ${CLN_RPC_SOURCES} cln-rpc/examples/getinfo.rs - cargo build ${CARGO_OPTS} --example cln-rpc-getinfo + $(CARGO) build ${CARGO_OPTS} --example cln-rpc-getinfo target/${RUST_PROFILE}/examples/cln-plugin-startup: ${CLN_RPC_SOURCES} plugins/examples/cln-plugin-startup.rs - cargo build ${CARGO_OPTS} --example cln-plugin-startup + $(CARGO) build ${CARGO_OPTS} --example cln-plugin-startup target/${RUST_PROFILE}/examples/cln-plugin-reentrant: ${CLN_RPC_SOURCES} plugins/examples/cln-plugin-reentrant.rs - cargo build ${CARGO_OPTS} --example cln-plugin-reentrant + $(CARGO) build ${CARGO_OPTS} --example cln-plugin-reentrant cln-rpc-all: ${CLN_RPC_GENALL} ${CLN_RPC_EXAMPLES} diff --git a/common/Makefile b/common/Makefile index 419261166fce..62ceb252a8ba 100644 --- a/common/Makefile +++ b/common/Makefile @@ -157,8 +157,8 @@ ALL_C_SOURCES += $(COMMON_SRC) common/htlc_state_names_gen.h: common/htlc_state.h ccan/ccan/cdump/tools/cdump-enumstr ccan/ccan/cdump/tools/cdump-enumstr common/htlc_state.h > $@ -check-source-bolt: $(COMMON_SRC_NOGEN:%=bolt-check/%) $(COMMON_HEADERS:%=bolt-check/%) -check-whitespace: $(COMMON_SRC_NOGEN:%=check-whitespace/%) $(COMMON_HEADERS:%=check-whitespace/%) +check-source-bolt: $(COMMON_SRC_NOGEN:%=bolt-check/%) $(COMMON_HEADERS_NOGEN:%=bolt-check/%) +check-whitespace: $(COMMON_SRC_NOGEN:%=check-whitespace/%) $(COMMON_HEADERS_NOGEN:%=check-whitespace/%) clean: common-clean diff --git a/connectd/connectd.c b/connectd/connectd.c index 44f81b6c2518..3de511cbf344 100644 --- a/connectd/connectd.c +++ b/connectd/connectd.c @@ -2224,6 +2224,10 @@ static void dev_report_fds(struct daemon *daemon, const u8 *msg) status_info("dev_report_fds: %i -> gossip_store", fd); continue; } + if (fd == tor_service_fd) { + status_info("dev_report_fds: %i -> tor service fd", fd); + continue; + } c = io_have_fd(fd, &listener); if (!c) { /* We consider a single CHR as expected */ diff --git a/connectd/tor_autoservice.c b/connectd/tor_autoservice.c index fef93a80fad5..cc867995a22d 100644 --- a/connectd/tor_autoservice.c +++ b/connectd/tor_autoservice.c @@ -12,6 +12,7 @@ #include #include +int tor_service_fd = -1; static void *buf_resize(struct membuf *mb, void *buf, size_t len) { @@ -331,5 +332,6 @@ struct wireaddr *tor_fixed_service(const tal_t *ctx, * read_partial to keep it open until LN drops * DO NOT CLOSE FD TO KEEP ADDRESS ALIVE AS WE DO NOT DETACH WITH STATIC ADDRESS */ + tor_service_fd = fd; return onion; } diff --git a/connectd/tor_autoservice.h b/connectd/tor_autoservice.h index dce91a637e49..950c6a97d3f1 100644 --- a/connectd/tor_autoservice.h +++ b/connectd/tor_autoservice.h @@ -18,4 +18,7 @@ struct wireaddr *tor_fixed_service(const tal_t *ctx, const struct wireaddr *bind, const u8 index); +/* For dev_report_fds */ +extern int tor_service_fd; + #endif /* LIGHTNING_CONNECTD_TOR_AUTOSERVICE_H */ diff --git a/devtools/cc-nobuild b/devtools/cc-nobuild new file mode 100755 index 000000000000..118385581f41 --- /dev/null +++ b/devtools/cc-nobuild @@ -0,0 +1,9 @@ +#! /bin/sh +# Version of CC which only supports -dumpmachine (for external/Makefile), and fails otherwise +set -e + +if [ x"$*" = x"-dumpmachine" ]; then + CC="$(grep ^CC= config.vars | cut -d= -f2-)" + exec ${CC:=cc} "$@" +fi +exit 1 diff --git a/external/Makefile b/external/Makefile index 839cbac2c6d2..4ff220539eb5 100644 --- a/external/Makefile +++ b/external/Makefile @@ -42,6 +42,8 @@ endif EXTERNAL_LDLIBS := -L${TARGET_DIR} $(patsubst lib%.a,-l%,$(notdir $(EXTERNAL_LIBS))) +.PHONY: submodcheck + submodcheck: $(FORCE) ifneq ($(VERSION),) @tools/refresh-submodules.sh $(SUBMODULES) @@ -60,6 +62,8 @@ ifeq ($(DEBUGBUILD),1) WALLY_OPTS=--enable-debug endif +.INTERMEDIATE: $(TARGET_DIR)/libwally-core-build/src/secp256k1/libsecp256k1.la $(TARGET_DIR)/libwally-core-build/src/libwallycore.la + $(TARGET_DIR)/libwally-core-build/src/libwallycore.% $(TARGET_DIR)/libwally-core-build/src/secp256k1/libsecp256k1.%: $(LIBWALLY_HEADERS) $(LIBSECP_HEADERS) cd external/libwally-core && ./tools/autogen.sh mkdir -p ${TARGET_DIR}/libwally-core-build @@ -85,6 +89,8 @@ $(TARGET_DIR)/jsmn-build/jsmn.o: external/jsmn/jsmn.c Makefile $(TARGET_DIR)/libjsmn.a: $(TARGET_DIR)/jsmn-build/jsmn.o $(AR) rc $@ $< +.INTERMEDIATE: $(TARGET_DIR)/jsmn-build/jsmn.o + # Need separate build dir: changes inside submodule make git think it's dirty. $(TARGET_DIR)/libbacktrace.a: external/libbacktrace/backtrace.h mkdir -p $(TARGET_DIR)/libbacktrace-build diff --git a/plugins/Makefile b/plugins/Makefile index 8a01fa339741..9dd006cbff36 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -138,23 +138,19 @@ $(shell test -d plugins/clnrest && $(RM) -r plugins/clnrest || true) $(shell test -d plugins/wss-proxy && $(RM) -r plugins/wss-proxy || true) ifneq ($(RUST),0) +RUST_PLUGIN_NAMES := cln-grpc clnrest cln-lsps-client cln-lsps-service wss-proxy cln-bip353 + # Builtin plugins must be in this plugins dir to work when we're executed # *without* make install. -plugins/cln-grpc: $(RUST_TARGET_DIR)/cln-grpc - @cp $< $@ -plugins/clnrest: $(RUST_TARGET_DIR)/clnrest - @cp $< $@ -plugins/cln-lsps-client: $(RUST_TARGET_DIR)/cln-lsps-client - @cp $< $@ -plugins/cln-lsps-service: $(RUST_TARGET_DIR)/cln-lsps-service - @cp $< $@ -plugins/wss-proxy: $(RUST_TARGET_DIR)/wss-proxy - @cp $< $@ -plugins/cln-bip353: $(RUST_TARGET_DIR)/cln-bip353 +RUST_PLUGINS := $(addprefix plugins/,$(RUST_PLUGIN_NAMES)) +$(RUST_PLUGINS): plugins/%: $(RUST_TARGET_DIR)/% @cp $< $@ -PLUGINS += plugins/cln-grpc plugins/clnrest plugins/cln-lsps-client plugins/cln-lsps-service plugins/wss-proxy plugins/cln-bip353 -endif +PLUGINS += $(RUST_PLUGINS) + +# You don't need the originals if you have the final ones. +.INTERMEDIATE: $(RUST_PLUGIN_NAMES:%=$(RUST_TARGET_DIR)/%) +endif # RUST include plugins/askrene/Makefile include plugins/bkpr/Makefile @@ -241,7 +237,7 @@ plugins/list_of_builtin_plugins_gen.h: plugins/Makefile Makefile config.vars @$(call VERBOSE,GEN $@,echo "static const char *list_of_builtin_plugins[] = { $(PLUGIN_BASES:%=\"%\",) NULL };" > $@) $(RUST_TARGET_DIR)/examples/cln-subscribe-wildcard: ${CLN_PLUGIN_SRC} plugins/examples/cln-subscribe-wildcard.rs - cargo build ${CARGO_OPTS} --example cln-subscribe-wildcard + $(CARGO) build ${CARGO_OPTS} --example cln-subscribe-wildcard CLN_PLUGIN_EXAMPLES := \ $(RUST_TARGET_DIR)/examples/cln-plugin-startup \ @@ -257,17 +253,17 @@ CLN_WSS_PROXY_PLUGIN_SRC = $(shell find plugins/wss-proxy-plugin/src -name "*.rs CLN_BIP353_PLUGIN_SRC = $(shell find plugins/bip353-plugin/src -name "*.rs") $(RUST_TARGET_DIR)/cln-grpc: ${CLN_PLUGIN_SRC} ${CLN_GRPC_PLUGIN_SRC} $(MSGGEN_GENALL) $(MSGGEN_GEN_ALL) - cargo build ${CARGO_OPTS} --bin cln-grpc + $(CARGO) build ${CARGO_OPTS} --bin cln-grpc $(RUST_TARGET_DIR)/clnrest: ${CLN_REST_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin clnrest + $(CARGO) build ${CARGO_OPTS} --bin clnrest $(RUST_TARGET_DIR)/cln-lsps-client: ${CLN_LSPS_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin cln-lsps-client + $(CARGO) build ${CARGO_OPTS} --bin cln-lsps-client $(RUST_TARGET_DIR)/cln-lsps-service: ${CLN_LSPS_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin cln-lsps-service + $(CARGO) build ${CARGO_OPTS} --bin cln-lsps-service $(RUST_TARGET_DIR)/wss-proxy: ${CLN_WSS_PROXY_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin wss-proxy + $(CARGO) build ${CARGO_OPTS} --bin wss-proxy $(RUST_TARGET_DIR)/cln-bip353: ${CLN_BIP353_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin cln-bip353 + $(CARGO) build ${CARGO_OPTS} --bin cln-bip353 ifneq ($(RUST),0) include plugins/rest-plugin/Makefile diff --git a/wire/Makefile b/wire/Makefile index 6d10c3e6b984..e52e1f16dc4d 100644 --- a/wire/Makefile +++ b/wire/Makefile @@ -106,7 +106,11 @@ wire/channel_type_wiregen.h_args := -s wire/channel_type_wiregen.c_args := $(wire/channel_type_wiregen.h_args) # All generated wire/ files depend on this Makefile -$(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(WIRE_SRC) $(WIRE_HEADERS)): wire/Makefile +WIRE_GENERATED := $(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(WIRE_SRC) $(WIRE_HEADERS)) +$(WIRE_GENERATED): wire/Makefile + +# You don't need to keep these if they don't exist. +.INTERMEDIATE: $(WIRE_GENERATED) maintainer-clean: wire-maintainer-clean