Gossmap compaction

Makefile

@@ -468,6 +468,7 @@ PKGLIBEXEC_PROGRAMS = \
 	       lightningd/lightning_connectd \
 	       lightningd/lightning_dualopend \
 	       lightningd/lightning_gossipd \
+	       lightningd/lightning_gossip_compactd \
 	       lightningd/lightning_hsmd \
 	       lightningd/lightning_onchaind \
 	       lightningd/lightning_openingd \
@@ -489,9 +490,14 @@ mkdocs.yml: $(MANPAGES:=.md)
 # Every single object file.
 ALL_OBJS := $(ALL_C_SOURCES:.c=.o)
 
+WIREGEN_FILES := $(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(ALL_C_HEADERS) $(ALL_C_SOURCES))
+
+# Always make wiregen files before any object file
+$(ALL_OBJS): $(WIREGEN_FILES)
+
 # We always regen wiregen and printgen files, since SHA256STAMP protects against
 # spurious rebuilds.
-$(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(ALL_C_HEADERS) $(ALL_C_SOURCES)): $(FORCE)
+$(WIREGEN_FILES): $(FORCE)
 
 ifneq ($(TEST_GROUP_COUNT),)
 PYTEST_OPTS += --test-group=$(TEST_GROUP) --test-group-count=$(TEST_GROUP_COUNT)

common/Makefile

@@ -111,7 +111,7 @@ COMMON_SRC_NOGEN :=				\
 	common/wire_error.c
 
 
-COMMON_SRC_GEN := common/status_wiregen.c common/peer_status_wiregen.c common/scb_wiregen.c
+COMMON_SRC_GEN := common/status_wiregen.c common/peer_status_wiregen.c common/scb_wiregen.c common/gossip_store_wiregen.c
 
 COMMON_HEADERS_NOGEN := $(COMMON_SRC_NOGEN:.c=.h)	\
 	common/closing_fee.h				\
@@ -124,7 +124,7 @@ COMMON_HEADERS_NOGEN := $(COMMON_SRC_NOGEN:.c=.h)	\
 	common/jsonrpc_errors.h				\
 	common/overflows.h
 
-COMMON_HEADERS_GEN := common/htlc_state_names_gen.h common/status_wiregen.h common/peer_status_wiregen.h common/scb_wiregen.h
+COMMON_HEADERS_GEN := common/htlc_state_names_gen.h common/status_wiregen.h common/peer_status_wiregen.h common/scb_wiregen.h common/gossip_store_wiregen.h
 
 COMMON_HEADERS := $(COMMON_HEADERS_GEN) $(COMMON_HEADERS_NOGEN)
 COMMON_SRC := $(COMMON_SRC_NOGEN) $(COMMON_SRC_GEN)
@@ -157,8 +157,6 @@ ALL_C_SOURCES += $(COMMON_SRC)
 common/htlc_state_names_gen.h: common/htlc_state.h ccan/ccan/cdump/tools/cdump-enumstr
 	ccan/ccan/cdump/tools/cdump-enumstr common/htlc_state.h > $@
 
-common/gossip_store.o: gossipd/gossip_store_wiregen.h
-
 check-source-bolt: $(COMMON_SRC_NOGEN:%=bolt-check/%) $(COMMON_HEADERS:%=bolt-check/%)
 check-whitespace: $(COMMON_SRC_NOGEN:%=check-whitespace/%) $(COMMON_HEADERS:%=check-whitespace/%)
 

common/gossip_store.c

@@ -1,27 +1,19 @@
 #include "config.h"
 #include <common/gossip_store.h>
-#include <gossipd/gossip_store_wiregen.h>
+#include <common/gossip_store_wiregen.h>
 #include <unistd.h>
 
-/* We cheat and read first two bytes of message too. */
-struct hdr_and_type {
-	struct gossip_hdr hdr;
-	be16 type;
-};
-/* Beware padding! */
-#define HDR_AND_TYPE_SIZE (sizeof(struct gossip_hdr) + sizeof(u16))
-
 bool gossip_store_readhdr(int gossip_store_fd, size_t off,
 			  size_t *len,
 			  u32 *timestamp,
 			  u16 *flags,
 			  u16 *type)
 {
-	struct hdr_and_type buf;
+	struct gossip_hdr_and_type buf;
 	int r;
 
-	r = pread(gossip_store_fd, &buf, HDR_AND_TYPE_SIZE, off);
-	if (r != HDR_AND_TYPE_SIZE)
+	r = pread(gossip_store_fd, &buf, GOSSIP_HDR_AND_TYPE_SIZE, off);
+	if (r != GOSSIP_HDR_AND_TYPE_SIZE)
 		return false;
 	if (!(buf.hdr.flags & CPU_TO_BE16(GOSSIP_STORE_COMPLETED_BIT)))
 		return false;

common/gossip_store.h

@@ -15,7 +15,7 @@ struct gossip_rcvd_filter;
 /* First byte of file is the version.
  *
  * Top three bits mean incompatible change.
- * As of this writing, major == 0, minor == 15.
+ * As of this writing, major == 0, minor == 16.
  */
 #define GOSSIP_STORE_MAJOR_VERSION_MASK 0xE0
 #define GOSSIP_STORE_MINOR_VERSION_MASK 0x1F
@@ -50,6 +50,14 @@ struct gossip_hdr {
 	beint32_t timestamp; /* timestamp of msg. */
 };
 
+/* Useful to read gossip_hdr and type of msg.  */
+struct gossip_hdr_and_type {
+	struct gossip_hdr hdr;
+	be16 type;
+};
+/* Beware padding! */
+#define GOSSIP_HDR_AND_TYPE_SIZE (sizeof(struct gossip_hdr) + sizeof(u16))
+
 /**
  * Direct store accessor: read gossip msg hdr from store.
  * @gossip_store_fd: the readable file descriptor
@@ -59,11 +67,8 @@ struct gossip_hdr {
  * @flags (out): if non-NULL, set to the flags.
  * @type (out): if non-NULL, set to the msg type.
  *
- * Returns false if there are no more gossip msgs.  If you
- * want to read the message, use gossip_store_next, if you
- * want to skip, simply add sizeof(gossip_hdr) + *len to *off.
- * Note: it's possible that entire record isn't there yet,
- * so gossip_store_next can fail.
+ * Returns false if there are no more gossip msgs, or message
+ * is incomplete.  To iterate, simply add sizeof(gossip_hdr) + *len to off.
  */
 bool gossip_store_readhdr(int gossip_store_fd, size_t off,
 			  size_t *len,

gossipd/gossip_store_wire.csv → common/gossip_store_wire.csv

@@ -23,7 +23,11 @@ msgdata,gossip_store_delete_chan,scid,short_channel_id,
 
 msgtype,gossip_store_ended,4105
 msgdata,gossip_store_ended,equivalent_offset,u64,
+msgdata,gossip_store_ended,uuid,u8,32
 
 msgtype,gossip_store_chan_dying,4106
 msgdata,gossip_store_chan_dying,scid,short_channel_id,
 msgdata,gossip_store_chan_dying,blockheight,u32,
+
+msgtype,gossip_store_uuid,4107
+msgdata,gossip_store_uuid,uuid,u8,32