feat(p2p): detect and track announce IP changes at runtime

.github/workflows/deploy-network.yml

@@ -10,11 +10,11 @@ on:
         required: true
         type: string
       semver:
-        description: "Semver version (e.g., 2.3.4)"
-        required: true
+        description: "Semver version (e.g., 2.3.4). Used to construct docker image if aztec_docker_image is not set."
+        required: false
         type: string
-      docker_image_tag:
-        description: "Full docker image tag (optional, defaults to semver)"
+      aztec_docker_image:
+        description: "Full Aztec docker image (e.g., aztecprotocol/aztec:2.3.4). If not set, constructed from semver."
         required: false
         type: string
       ref:
@@ -50,11 +50,11 @@ on:
           - testnet
           - mainnet
       semver:
-        description: "Semver version (e.g., 2.3.4)"
-        required: true
+        description: "Semver version (e.g., 2.3.4). Used to construct docker image if aztec_docker_image is not set."
+        required: false
         type: string
-      docker_image_tag:
-        description: "Full docker image tag (optional, defaults to semver)"
+      aztec_docker_image:
+        description: "Full Aztec docker image (e.g., aztecprotocol/aztec:2.3.4). If not set, constructed from semver."
         required: false
         type: string
       namespace:
@@ -76,7 +76,7 @@ on:
         type: string
 
 concurrency:
-  group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.semver }}-${{ github.ref || github.ref_name }}
+  group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.aztec_docker_image || inputs.semver }}-${{ github.ref || github.ref_name }}
   cancel-in-progress: true
 
 jobs:
@@ -120,16 +120,33 @@ jobs:
             exit 1
           fi
 
-          # Validate semver format
-          if ! echo "${{ inputs.semver }}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-.*)?$'; then
-            echo "Error: Invalid semver format '${{ inputs.semver }}'. Expected format: X.Y.Z or X.Y.Z-suffix"
+          # Require at least one of aztec_docker_image or semver
+          if [[ -z "${{ inputs.aztec_docker_image }}" && -z "${{ inputs.semver }}" ]]; then
+            echo "Error: Either 'aztec_docker_image' or 'semver' must be provided"
             exit 1
           fi
 
-          # Extract major version for v2 check
-          major_version="${{ inputs.semver }}"
-          major_version="${major_version%%.*}"
-          echo "MAJOR_VERSION=$major_version" >> $GITHUB_ENV
+          # Validate semver format if provided
+          if [[ -n "${{ inputs.semver }}" ]]; then
+            if ! echo "${{ inputs.semver }}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-.*)?$'; then
+              echo "Error: Invalid semver format '${{ inputs.semver }}'. Expected format: X.Y.Z or X.Y.Z-suffix"
+              exit 1
+            fi
+          fi
+
+          # Resolve the docker image
+          if [[ -n "${{ inputs.aztec_docker_image }}" ]]; then
+            AZTEC_DOCKER_IMAGE="${{ inputs.aztec_docker_image }}"
+          else
+            AZTEC_DOCKER_IMAGE="aztecprotocol/aztec:${{ inputs.semver }}"
+          fi
+          echo "AZTEC_DOCKER_IMAGE=$AZTEC_DOCKER_IMAGE" >> $GITHUB_ENV
+
+          # Only use the separate prover-agent image for official semver builds;
+          # for custom images, let the deploy script fall back to AZTEC_DOCKER_IMAGE
+          if [[ -n "${{ inputs.semver }}" ]]; then
+            echo "PROVER_AGENT_DOCKER_IMAGE=aztecprotocol/aztec-prover-agent:${{ inputs.semver }}" >> $GITHUB_ENV
+          fi
 
       - name: Store the GCP key in a file
         env:
@@ -174,12 +191,12 @@ jobs:
           RUN_ID: ${{ github.run_id }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           GOOGLE_APPLICATION_CREDENTIALS: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
-          REF_NAME: "v${{ inputs.semver }}"
+          REF_NAME: ${{ inputs.semver && format('v{0}', inputs.semver) || '' }}
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
           NAMESPACE: ${{ inputs.namespace }}
-          AZTEC_DOCKER_IMAGE: "aztecprotocol/aztec:${{ inputs.docker_image_tag || inputs.semver }}"
+          AZTEC_DOCKER_IMAGE: ${{ env.AZTEC_DOCKER_IMAGE }}
           CREATE_ROLLUP_CONTRACTS: ${{ inputs.deploy_contracts == true && 'true' || '' }}
-          PROVER_AGENT_DOCKER_IMAGE: "aztecprotocol/aztec-prover-agent:${{ inputs.docker_image_tag || inputs.semver }}"
+          PROVER_AGENT_DOCKER_IMAGE: ${{ env.PROVER_AGENT_DOCKER_IMAGE || env.AZTEC_DOCKER_IMAGE }}
           VALIDATOR_HA_DOCKER_IMAGE: ${{ inputs.ha_docker_image || '' }}
         run: |
           echo "Deploying network: ${{ inputs.network }}"
@@ -209,7 +226,7 @@ jobs:
             echo "| Item | Value |"
             echo "|------|-------|"
             echo "| Network | \`${{ inputs.network }}\` |"
-            echo "| Semver | \`${{ inputs.semver }}\` |"
+            echo "| Docker Image | \`${{ env.AZTEC_DOCKER_IMAGE }}\` |"
             echo "| Ref | \`${{ steps.checkout-ref.outputs.ref }}\` |"
             if [[ -n "${{ inputs.source_tag }}" ]]; then
               echo "| Source Tag | [\`${{ inputs.source_tag }}\`](https://github.com/${{ github.repository }}/releases/tag/${{ inputs.source_tag }}) |"
@@ -229,7 +246,7 @@ jobs:
 
           CHANNEL="#alerts-${{ inputs.network }}"
           RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-          TEXT="Deploy Network workflow FAILED for *${{ inputs.network }}* (version ${{ inputs.semver }}): <${RUN_URL}|View Run> (🤖)"
+          TEXT="Deploy Network workflow FAILED for *${{ inputs.network }}* (image ${{ env.AZTEC_DOCKER_IMAGE }}): <${RUN_URL}|View Run> (🤖)"
 
           # Post to Slack and capture timestamp for permalink
           RESP=$(curl -sS -X POST https://slack.com/api/chat.postMessage \
@@ -247,11 +264,11 @@ jobs:
           fi
 
           # Dispatch ClaudeBox to investigate the failure
-          PROMPT="Deployment of ${{ inputs.network }} (version ${{ inputs.semver }}) failed. \
+          PROMPT="Deployment of ${{ inputs.network }} (image ${{ env.AZTEC_DOCKER_IMAGE }}) failed. \
           Follow .claude/claudebox/deploy-investigation.md to investigate. \
           GitHub Actions run: ${RUN_URL}. \
-          Network: ${{ inputs.network }}. Version: ${{ inputs.semver }}. \
-          Docker image: ${{ inputs.docker_image_tag || inputs.semver }}. \
+          Network: ${{ inputs.network }}. \
+          Docker image: ${{ env.AZTEC_DOCKER_IMAGE }}. \
           Git ref: ${{ steps.checkout-ref.outputs.ref }}. \
           Namespace: ${{ inputs.namespace || inputs.network }}. \
           Deploy contracts: ${{ inputs.deploy_contracts }}."

.github/workflows/deploy-next-net.yml

@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
     inputs:
       image_tag:
-        description: 'Docker image tag (e.g., 2.3.4, 3.0.0-nightly.20251004-amd64, or leave empty for latest nightly)'
+        description: "Docker image tag (e.g., 2.3.4, 3.0.0-nightly.20251004-amd64, or leave empty for latest nightly)"
         required: false
         type: string
 
@@ -67,6 +67,6 @@ jobs:
     with:
       network: next-net
       semver: ${{ needs.get-image-tag.outputs.semver }}
-      docker_image_tag: ${{ needs.get-image-tag.outputs.tag }}
+      docker_image: "aztecprotocol/aztec:${{ needs.get-image-tag.outputs.tag }}"
       ref: ${{ github.ref }}
     secrets: inherit

spartan/environments/next-net.env

@@ -29,6 +29,8 @@ L1_TX_FAILED_STORE=gs://aztec-develop/next-net/failed-l1-txs
 TEST_ACCOUNTS=true
 SPONSORED_FPC=true
 
+LOG_LEVEL=debug
+
 SEQ_MIN_TX_PER_BLOCK=1
 
 SEQ_MAX_TX_PER_CHECKPOINT=7

yarn-project/p2p/src/bootstrap/bootstrap.ts

@@ -86,6 +86,14 @@ export class BootstrapNode implements P2PBootstrapApi {
     this.node.on('discovered', async (enr: SignableENR) => {
       const addr = await enr.getFullMultiaddr('udp');
       this.logger.verbose(`Discovered new peer`, { enr: enr.encodeTxt(), addr: addr?.toString() });
+      // discv5's discovered() only updates routing table entries that already exist. Nodes that
+      // established a session with an empty-IP ENR are never inserted, so even after their ENR
+      // gains a valid socket address the routing table stays empty and FINDNODE always returns 0
+      // peers. Calling addEnr() here does an insertOrUpdate regardless of prior state, fixing
+      // the routing table so these nodes become discoverable to other peers.
+      if (addr) {
+        this.node.addEnr(enr);
+      }
     });
 
     try {

yarn-project/p2p/src/client/factory.ts

@@ -64,6 +64,15 @@ export async function createP2PClient(
 
   const logger = deps.logger ?? createLogger('p2p');
 
+  logger.info('P2P client address config resolved', {
+    p2pIp: config.p2pIp ?? 'not set',
+    queryForIp: config.queryForIp,
+    p2pPort: config.p2pPort,
+    p2pBroadcastPort: config.p2pBroadcastPort,
+    listenAddress: config.listenAddress,
+    bootstrapNodeCount: config.bootstrapNodes.length,
+  });
+
   if (config.bootstrapNodes.length === 0) {
     logger.warn(
       'No bootstrap nodes have been provided. Set the BOOTSTRAP_NODES environment variable in order to join the P2P network',

yarn-project/p2p/src/services/discv5/discV5_service.ts

@@ -53,6 +53,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
     configOverrides: Partial<IDiscv5CreateOptions> = {},
   ) {
     super();
+
     const { p2pIp, p2pPort, p2pBroadcastPort, bootstrapNodes, trustedPeers, privatePeers } = config;
 
     this.bootstrapNodeEnrs = bootstrapNodes.map(x => ENR.decodeTxt(x));
@@ -96,7 +97,9 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
         lookupTimeout: 2000,
         requestTimeout: 2000,
         allowUnverifiedSessions: true,
-        enrUpdate: !p2pIp ? true : false, // If no p2p IP is set, enrUpdate can automatically resolve it
+        enrUpdate: config.queryForIp || !p2pIp,
+        pingInterval: config.queryForIp ? 10_000 : 300_000,
+        addrVotesToUpdateEnr: config.queryForIp ? 1 : 10,
         ...configOverrides.config,
       },
       metricsRegistry,
@@ -127,11 +130,19 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
   }
 
   private onMultiaddrUpdated(m: Multiaddr) {
+    const address = m.nodeAddress().address;
     // We want to update our tcp port to match the udp port
     // p2pBroadcastPort is optional on config, however it is set to default within the p2p client factory
-    const multiAddrTcp = multiaddr(convertToMultiaddr(m.nodeAddress().address, this.config.p2pBroadcastPort!, 'tcp'));
+    const multiAddrTcp = multiaddr(convertToMultiaddr(address, this.config.p2pBroadcastPort!, 'tcp'));
+    const prevIp = this.enr.toENR().ip ?? 'none';
     this.enr.setLocationMultiaddr(multiAddrTcp);
-    this.logger.info('Multiaddr updated', { multiaddr: multiAddrTcp.toString() });
+    this.logger.info('Multiaddr updated via discv5 PONG vote', {
+      prevIp,
+      newIp: address,
+      multiaddr: multiAddrTcp.toString(),
+    });
+
+    this.emit('ip:changed', address);
   }
 
   public async start(): Promise<void> {
@@ -201,11 +212,21 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
       await sleep(delayBeforeStart - msSinceStart);
     }
 
+    const kadBefore = this.discv5.kadValues().length;
     try {
       await this.discv5.findRandomNode();
     } catch (err) {
       this.logger.error(`Error running discV5 random node query: ${err}`);
     }
+    const kadAfter = this.discv5.kadValues().length;
+    const enr = this.enr.toENR();
+    this.logger.debug(`DiscV5 random node query complete`, {
+      kadBefore,
+      kadAfter,
+      enrIp: enr.ip ?? 'none',
+      enrUdp: enr.udp ?? 'none',
+      enrTcp: enr.tcp ?? 'none',
+    });
   }
 
   public getKadValues(): ENR[] {
@@ -244,7 +265,12 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
   private async onEnrAdded(enr: ENR) {
     const multiAddrTcp = await enr.getFullMultiaddr('tcp');
     const multiAddrUdp = await enr.getFullMultiaddr('udp');
-    this.logger.debug(`Added ENR ${enr.encodeTxt()}`, { multiAddrTcp, multiAddrUdp, nodeId: enr.nodeId });
+    this.logger.info(`DiscV5 ENR added (peer discovered via DHT)`, {
+      nodeId: enr.nodeId,
+      multiAddrTcp: multiAddrTcp?.toString() ?? 'none',
+      multiAddrUdp: multiAddrUdp?.toString() ?? 'none',
+      isBootnode: this.isOurBootnode(enr),
+    });
     this.onDiscovered(enr);
   }
 
@@ -277,7 +303,10 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
     // Check the peer is an aztec peer
     const value = enr.kvs.get(AZTEC_ENR_KEY);
     if (!value) {
-      this.logger.debug(`Peer node ${enr.nodeId} does not have aztec key in ENR`);
+      this.logger.info(`Discovered peer has no aztec key in ENR, ignoring`, {
+        nodeId: enr.nodeId,
+        enrIp: enr.ip ?? 'none',
+      });
       return false;
     }
 
@@ -289,7 +318,9 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
       return true;
     } catch (err: any) {
       if (err.name === 'ComponentsVersionsError') {
-        this.logger.debug(`Peer node ${enr.nodeId} has incorrect version: ${err.message}`, {
+        this.logger.info(`Discovered peer has wrong version, ignoring`, {
+          nodeId: enr.nodeId,
+          enrIp: enr.ip ?? 'none',
           compressedVersion,
           expected: this.versions,
         });

yarn-project/p2p/src/services/discv5/discv5_service.test.ts

@@ -144,6 +144,37 @@ describe('Discv5Service', () => {
     await stopNodes(...nodes);
   });
 
+  it('should correct a wrong initial IP via PONG votes when enrUpdate is forced on', async () => {
+    const extraNodes = 3;
+    const nodes: DiscV5Service[] = [];
+
+    // Simulate the scenario where getPublicIp() returned a wrong IP at startup (e.g. NAT egress IP).
+    // With enrUpdate forced on, PONG votes from peers should correct the ENR to 127.0.0.1.
+    const node = await createNode({
+      p2pIp: '1.2.3.4',
+      config: { enrUpdate: true, addrVotesToUpdateEnr: 1, pingInterval: 200 },
+    });
+    await node.start();
+    nodes.push(node);
+
+    expect(node.getEnr().ip).toEqual('1.2.3.4');
+
+    for (let i = 1; i < extraNodes; i++) {
+      const n = await createNode({ config: { pingInterval: 200 } });
+      await n.start();
+      nodes.push(n);
+    }
+
+    // Wait for the ENR IP to be corrected by PONG votes
+    await runDiscoveryUntil(nodes, () => node.getEnr().ip !== '1.2.3.4');
+
+    // ENR should now reflect the real IP (127.0.0.1) as reported by peers
+    expect(node.getEnr().ip).toEqual('127.0.0.1');
+    expect(node.getEnr().tcp).toEqual(node.getEnr().udp);
+
+    await stopNodes(...nodes);
+  });
+
   it('should refuse to connect to a bootstrap node with wrong chain id', async () => {
     const node1 = await createNode({ l1ChainId: 13, bootstrapNodeEnrVersionCheck: true });
     const node2 = await createNode({ l1ChainId: 14, bootstrapNodeEnrVersionCheck: false });