fix(node): race on caught up (#3133)

feat: add `Syncer.PendingCount` and use it to ensure the sync pipeline is drained before marking the node as caught up in failover.

Author: alpe

block/internal/syncing/syncer.go

@@ -467,6 +467,11 @@ func (s *Syncer) fetchDAUntilCaughtUp(ctx context.Context) error {
 	}
 }
 
+// PendingCount returns the number of unprocessed height events in the pipeline.
+func (s *Syncer) PendingCount() int {
+	return len(s.heightInCh)
+}
+
 func (s *Syncer) pendingWorkerLoop(ctx context.Context) {
 	defer s.wg.Done()
 

node/failover.go

@@ -315,7 +315,8 @@ func (f *failoverState) waitForCatchup(ctx context.Context) (bool, error) {
 				p2pCaughtUp = true
 			}
 
-			if daCaughtUp && p2pCaughtUp {
+			pipelineDrained := f.bc.Syncer == nil || f.bc.Syncer.PendingCount() == 0
+			if daCaughtUp && p2pCaughtUp && pipelineDrained {
 				f.logger.Info().
 					Uint64("store_height", storeHeight).
 					Uint64("max_p2p_height", maxP2PHeight).