CI: stop rebuilding everything in every step!

Cargo.toml

@@ -1,6 +1,12 @@
 [profile.release]
 strip = "debuginfo"
 
+[profile.small]
+inherits = "dev"
+opt-level = 0
+debug = 0
+codegen-units = 32
+
 [workspace]
 resolver = "2"
 members = [

Makefile

@@ -21,6 +21,7 @@ ECHO := echo
 SUPPRESS_OUTPUT :=
 endif
 
+CARGO := cargo
 DISTRO=$(shell lsb_release -is 2>/dev/null || echo unknown)-$(shell lsb_release -rs 2>/dev/null || echo unknown)
 OS=$(shell uname -s)
 ARCH=$(shell uname -m)
@@ -104,6 +105,7 @@ CCAN_OBJS :=					\
 	ccan-bitmap.o				\
 	ccan-bitops.o				\
 	ccan-breakpoint.o			\
+	ccan-cdump.o				\
 	ccan-closefrom.o			\
 	ccan-crc32c.o				\
 	ccan-crypto-hmac.o			\
@@ -229,8 +231,6 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/typesafe_cb/typesafe_cb.h	\
 	$(CCANDIR)/ccan/utf8/utf8.h
 
-CDUMP_OBJS := ccan-cdump.o ccan-strmap.o
-
 BOLT_GEN := tools/generate-wire.py
 WIRE_GEN := $(BOLT_GEN)
 
@@ -483,10 +483,6 @@ mkdocs.yml: $(MANPAGES:=.md)
 	)
 
 
-
-# Don't delete these intermediaries.
-.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) $(PYTHON_GENERATED)
-
 # Every single object file.
 ALL_OBJS := $(ALL_C_SOURCES:.c=.o)
 
@@ -649,10 +645,8 @@ update-doc-examples:
 check-doc-examples: update-doc-examples
 	git diff --exit-code HEAD
 
-# For those without working cppcheck
-check-source-no-cppcheck: check-makefile check-source-bolt check-whitespace check-spelling check-python check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access check-bad-sprintf
-
-check-source: check-source-no-cppcheck
+# This should NOT compile things!
+check-source: check-makefile check-whitespace check-spelling check-python-flake8 check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access check-bad-sprintf
 
 full-check: check check-source
 
@@ -725,7 +719,7 @@ TAGS:
 tags:
 	$(RM) tags; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 ctags --append
 
-ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS)
+ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o libccan.a
 
 ALL_PROGRAMS += ccan/ccan/cdump/tools/cdump-enumstr
 # Can't add to ALL_OBJS, as that makes a circular dep.
@@ -747,6 +741,9 @@ endif
 header_versions_gen.h: tools/headerversions $(FORCE)
 	@tools/headerversions $@
 
+# Once you have libccan.a, you don't need these.
+.INTERMEDIATE: $(CCAN_OBJS)
+
 # We make a static library, this way linker can discard unused parts.
 libccan.a: $(CCAN_OBJS)
 	@$(call VERBOSE, "ar $@", $(AR) r $@ $(CCAN_OBJS))
@@ -968,25 +965,21 @@ install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(
 
 install: install-program install-data
 
-# Non-artifacts that are needed for testing. These are added to the
-# testpack.tar, used to transfer things between builder and tester
-# phase. If you get a missing file/executable while testing on CI it
-# is likely missing from this variable.
-TESTBINS = \
-	$(CLN_PLUGIN_EXAMPLES) \
-	tests/plugins/test_libplugin \
-	tests/plugins/channeld_fakenet \
-	tests/plugins/test_selfdisable_after_getmanifest
+# We exclude most of target/ and external, but we need:
+# 1. config files (we only tar up files *newer* than these)
+# 2. $(DEFAULT_TARGETS) for rust stuff.
+# 3. $(EXTERNAL_LIBS) for prebuild external libraries.
+TESTPACK_EXTRAS :=			\
+	config.vars ccan/config.h	\
+	header_versions_gen.h		\
+	$(DEFAULT_TARGETS)		\
+	$(EXTERNAL_LIBS)
 
 # The testpack is used in CI to transfer built artefacts between the
-# build and the test phase. This is necessary because the fixtures in
-# `tests/` explicitly use the binaries built in the current directory
-# rather than using `$PATH`, as that may pick up some other installed
-# version of `lightningd` leading to bogus results. We bundle up all
-# built artefacts here, and will unpack them on the tester (overlaying
-# on top of the checked out repo as if we had just built it in place).
-testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS) $(TOOLS)
-	tar -caf $@ $^
+# build and the test phase.  Only useful on a freshly build tree!
+# We use Posix format for timestamps with subsecond accuracy.
+testpack.tar.gz: all-programs all-fuzz-programs all-test-programs default-targets
+	(find * -path external -prune -o -path target -prune -o -newer config.vars -type f -print; ls $(TESTPACK_EXTRAS)) | tar --verbatim-files-from -T- -c --format=posix -f - | gzip -5 > $@
 
 uninstall:
 	@$(NORMAL_UNINSTALL)

cln-rpc/Makefile

@@ -9,13 +9,13 @@ DEFAULT_TARGETS += $(CLN_RPC_EXAMPLES) $(CLN_RPC_GENALL)
 MSGGEN_GENALL += $(CLN_RPC_GENALL)
 
 target/${RUST_PROFILE}/examples/cln-rpc-getinfo: ${CLN_RPC_SOURCES} cln-rpc/examples/getinfo.rs
-	cargo build ${CARGO_OPTS} --example cln-rpc-getinfo
+	$(CARGO) build ${CARGO_OPTS} --example cln-rpc-getinfo
 
 target/${RUST_PROFILE}/examples/cln-plugin-startup: ${CLN_RPC_SOURCES} plugins/examples/cln-plugin-startup.rs
-	cargo build ${CARGO_OPTS} --example cln-plugin-startup
+	$(CARGO) build ${CARGO_OPTS} --example cln-plugin-startup
 
 target/${RUST_PROFILE}/examples/cln-plugin-reentrant: ${CLN_RPC_SOURCES} plugins/examples/cln-plugin-reentrant.rs
-	cargo build ${CARGO_OPTS} --example cln-plugin-reentrant
+	$(CARGO) build ${CARGO_OPTS} --example cln-plugin-reentrant
 
 
 cln-rpc-all: ${CLN_RPC_GENALL} ${CLN_RPC_EXAMPLES}

common/Makefile

@@ -157,8 +157,8 @@ ALL_C_SOURCES += $(COMMON_SRC)
 common/htlc_state_names_gen.h: common/htlc_state.h ccan/ccan/cdump/tools/cdump-enumstr
 	ccan/ccan/cdump/tools/cdump-enumstr common/htlc_state.h > $@
 
-check-source-bolt: $(COMMON_SRC_NOGEN:%=bolt-check/%) $(COMMON_HEADERS:%=bolt-check/%)
-check-whitespace: $(COMMON_SRC_NOGEN:%=check-whitespace/%) $(COMMON_HEADERS:%=check-whitespace/%)
+check-source-bolt: $(COMMON_SRC_NOGEN:%=bolt-check/%) $(COMMON_HEADERS_NOGEN:%=bolt-check/%)
+check-whitespace: $(COMMON_SRC_NOGEN:%=check-whitespace/%) $(COMMON_HEADERS_NOGEN:%=check-whitespace/%)
 
 clean: common-clean
 

connectd/connectd.c

@@ -2224,6 +2224,10 @@ static void dev_report_fds(struct daemon *daemon, const u8 *msg)
 			status_info("dev_report_fds: %i -> gossip_store", fd);
 			continue;
 		}
+		if (fd == tor_service_fd) {
+			status_info("dev_report_fds: %i -> tor service fd", fd);
+			continue;
+		}
 		c = io_have_fd(fd, &listener);
 		if (!c) {
 			/* We consider a single CHR as expected */

connectd/tor_autoservice.c

@@ -12,6 +12,7 @@
 #include <netdb.h>
 #include <unistd.h>
 
+int tor_service_fd = -1;
 
 static void *buf_resize(struct membuf *mb, void *buf, size_t len)
 {
@@ -331,5 +332,6 @@ struct wireaddr *tor_fixed_service(const tal_t *ctx,
 	* read_partial to keep it open until LN drops
 	* DO NOT CLOSE FD TO KEEP ADDRESS ALIVE AS WE DO NOT DETACH WITH STATIC ADDRESS
 	*/
+	tor_service_fd = fd;
 	return onion;
 }

connectd/tor_autoservice.h

@@ -18,4 +18,7 @@ struct wireaddr *tor_fixed_service(const tal_t *ctx,
 				 const struct wireaddr *bind,
 				 const u8 index);
 
+/* For dev_report_fds */
+extern int tor_service_fd;
+
 #endif /* LIGHTNING_CONNECTD_TOR_AUTOSERVICE_H */

devtools/cc-nobuild

@@ -0,0 +1,9 @@
+#! /bin/sh
+# Version of CC which only supports -dumpmachine (for external/Makefile), and fails otherwise
+set -e
+
+if [ x"$*" = x"-dumpmachine" ]; then
+    CC="$(grep ^CC= config.vars | cut -d= -f2-)"
+    exec ${CC:=cc} "$@"
+fi
+exit 1

external/Makefile

@@ -42,6 +42,8 @@ endif
 
 EXTERNAL_LDLIBS := -L${TARGET_DIR} $(patsubst lib%.a,-l%,$(notdir $(EXTERNAL_LIBS)))
 
+.PHONY: submodcheck
+
 submodcheck: $(FORCE)
 ifneq ($(VERSION),)
 	@tools/refresh-submodules.sh $(SUBMODULES)
@@ -60,6 +62,8 @@ ifeq ($(DEBUGBUILD),1)
 WALLY_OPTS=--enable-debug
 endif
 
+.INTERMEDIATE: $(TARGET_DIR)/libwally-core-build/src/secp256k1/libsecp256k1.la $(TARGET_DIR)/libwally-core-build/src/libwallycore.la
+
 $(TARGET_DIR)/libwally-core-build/src/libwallycore.% $(TARGET_DIR)/libwally-core-build/src/secp256k1/libsecp256k1.%: $(LIBWALLY_HEADERS) $(LIBSECP_HEADERS)
 	cd external/libwally-core && ./tools/autogen.sh
 	mkdir -p ${TARGET_DIR}/libwally-core-build
@@ -85,6 +89,8 @@ $(TARGET_DIR)/jsmn-build/jsmn.o: external/jsmn/jsmn.c Makefile
 $(TARGET_DIR)/libjsmn.a: $(TARGET_DIR)/jsmn-build/jsmn.o
 	$(AR) rc $@ $<
 
+.INTERMEDIATE: $(TARGET_DIR)/jsmn-build/jsmn.o
+
 # Need separate build dir: changes inside submodule make git think it's dirty.
 $(TARGET_DIR)/libbacktrace.a: external/libbacktrace/backtrace.h
 	mkdir -p $(TARGET_DIR)/libbacktrace-build

plugins/Makefile

@@ -138,23 +138,19 @@ $(shell test -d plugins/clnrest && $(RM) -r plugins/clnrest || true)
 $(shell test -d plugins/wss-proxy && $(RM) -r plugins/wss-proxy || true)
 
 ifneq ($(RUST),0)
+RUST_PLUGIN_NAMES := cln-grpc clnrest cln-lsps-client cln-lsps-service wss-proxy cln-bip353
+
 # Builtin plugins must be in this plugins dir to work when we're executed
 # *without* make install.
-plugins/cln-grpc: $(RUST_TARGET_DIR)/cln-grpc
-	@cp $< $@
-plugins/clnrest: $(RUST_TARGET_DIR)/clnrest
-	@cp $< $@
-plugins/cln-lsps-client: $(RUST_TARGET_DIR)/cln-lsps-client
-	@cp $< $@
-plugins/cln-lsps-service: $(RUST_TARGET_DIR)/cln-lsps-service
-	@cp $< $@
-plugins/wss-proxy: $(RUST_TARGET_DIR)/wss-proxy
-	@cp $< $@
-plugins/cln-bip353: $(RUST_TARGET_DIR)/cln-bip353
+RUST_PLUGINS := $(addprefix plugins/,$(RUST_PLUGIN_NAMES))
+$(RUST_PLUGINS): plugins/%: $(RUST_TARGET_DIR)/%
 	@cp $< $@
 
-PLUGINS += plugins/cln-grpc plugins/clnrest plugins/cln-lsps-client plugins/cln-lsps-service plugins/wss-proxy plugins/cln-bip353
-endif
+PLUGINS += $(RUST_PLUGINS)
+
+# You don't need the originals if you have the final ones.
+.INTERMEDIATE: $(RUST_PLUGIN_NAMES:%=$(RUST_TARGET_DIR)/%)
+endif # RUST
 
 include plugins/askrene/Makefile
 include plugins/bkpr/Makefile
@@ -241,7 +237,7 @@ plugins/list_of_builtin_plugins_gen.h: plugins/Makefile Makefile config.vars
 	@$(call VERBOSE,GEN $@,echo "static const char *list_of_builtin_plugins[] = { $(PLUGIN_BASES:%=\"%\",) NULL };" > $@)
 
 $(RUST_TARGET_DIR)/examples/cln-subscribe-wildcard: ${CLN_PLUGIN_SRC} plugins/examples/cln-subscribe-wildcard.rs
-	cargo build ${CARGO_OPTS} --example cln-subscribe-wildcard
+	$(CARGO) build ${CARGO_OPTS} --example cln-subscribe-wildcard
 
 CLN_PLUGIN_EXAMPLES := \
 	$(RUST_TARGET_DIR)/examples/cln-plugin-startup \
@@ -257,17 +253,17 @@ CLN_WSS_PROXY_PLUGIN_SRC = $(shell find plugins/wss-proxy-plugin/src -name "*.rs
 CLN_BIP353_PLUGIN_SRC = $(shell find plugins/bip353-plugin/src -name "*.rs")
 
 $(RUST_TARGET_DIR)/cln-grpc: ${CLN_PLUGIN_SRC} ${CLN_GRPC_PLUGIN_SRC} $(MSGGEN_GENALL) $(MSGGEN_GEN_ALL)
-	cargo build ${CARGO_OPTS} --bin cln-grpc
+	$(CARGO) build ${CARGO_OPTS} --bin cln-grpc
 $(RUST_TARGET_DIR)/clnrest: ${CLN_REST_PLUGIN_SRC}
-	cargo build ${CARGO_OPTS} --bin clnrest
+	$(CARGO) build ${CARGO_OPTS} --bin clnrest
 $(RUST_TARGET_DIR)/cln-lsps-client: ${CLN_LSPS_PLUGIN_SRC}
-	cargo build ${CARGO_OPTS} --bin cln-lsps-client
+	$(CARGO) build ${CARGO_OPTS} --bin cln-lsps-client
 $(RUST_TARGET_DIR)/cln-lsps-service: ${CLN_LSPS_PLUGIN_SRC}
-	cargo build ${CARGO_OPTS} --bin cln-lsps-service
+	$(CARGO) build ${CARGO_OPTS} --bin cln-lsps-service
 $(RUST_TARGET_DIR)/wss-proxy: ${CLN_WSS_PROXY_PLUGIN_SRC}
-	cargo build ${CARGO_OPTS} --bin wss-proxy
+	$(CARGO) build ${CARGO_OPTS} --bin wss-proxy
 $(RUST_TARGET_DIR)/cln-bip353: ${CLN_BIP353_PLUGIN_SRC}
-	cargo build ${CARGO_OPTS} --bin cln-bip353
+	$(CARGO) build ${CARGO_OPTS} --bin cln-bip353
 
 ifneq ($(RUST),0)
 include plugins/rest-plugin/Makefile

wire/Makefile

@@ -106,7 +106,11 @@ wire/channel_type_wiregen.h_args := -s
 wire/channel_type_wiregen.c_args := $(wire/channel_type_wiregen.h_args)
 
 # All generated wire/ files depend on this Makefile
-$(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(WIRE_SRC) $(WIRE_HEADERS)): wire/Makefile
+WIRE_GENERATED := $(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(WIRE_SRC) $(WIRE_HEADERS))
+$(WIRE_GENERATED): wire/Makefile
+
+# You don't need to keep these if they don't exist.
+.INTERMEDIATE: $(WIRE_GENERATED)
 
 maintainer-clean: wire-maintainer-clean