From 4d3c00213f917c1b6d3c4d2860a03367025494b9 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 8 Apr 2026 13:10:53 +0000 Subject: [PATCH 01/10] feat(p2p): automatically detect changes in announce P2P IP --- yarn-project/p2p/src/client/factory.ts | 2 +- .../p2p/src/services/discv5/discV5_service.ts | 7 +++- .../services/discv5/discv5_service.test.ts | 31 ++++++++++++++ .../p2p/src/services/libp2p/libp2p_service.ts | 41 ++++++++++++++++--- yarn-project/p2p/src/services/service.ts | 3 ++ .../p2p/src/test-helpers/mock-pubsub.ts | 7 ++++ .../p2p/src/test-helpers/reqresp-nodes.ts | 4 ++ yarn-project/p2p/src/util.ts | 21 ++++------ 8 files changed, 93 insertions(+), 23 deletions(-) diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index dbe16e6debe4..3c6c70b16ad7 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -57,7 +57,7 @@ export async function createP2PClient( telemetry: TelemetryClient = getTelemetryClient(), deps: P2PClientDeps = {}, ) { - const config = await configureP2PClientAddresses({ + const config = configureP2PClientAddresses({ ...inputConfig, dataStoreMapSizeKb: inputConfig.p2pStoreMapSizeKb ?? inputConfig.dataStoreMapSizeKb, }); diff --git a/yarn-project/p2p/src/services/discv5/discV5_service.ts b/yarn-project/p2p/src/services/discv5/discV5_service.ts index de288a63092a..251e62ef4097 100644 --- a/yarn-project/p2p/src/services/discv5/discV5_service.ts +++ b/yarn-project/p2p/src/services/discv5/discV5_service.ts @@ -96,7 +96,8 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService lookupTimeout: 2000, requestTimeout: 2000, allowUnverifiedSessions: true, - enrUpdate: !p2pIp ? true : false, // If no p2p IP is set, enrUpdate can automatically resolve it + enrUpdate: config.queryForIp || !p2pIp, + pingInterval: config.queryForIp ? 10_000 : 300_000, ...configOverrides.config, }, metricsRegistry, @@ -127,11 +128,13 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService } private onMultiaddrUpdated(m: Multiaddr) { + const address = m.nodeAddress().address; // We want to update our tcp port to match the udp port // p2pBroadcastPort is optional on config, however it is set to default within the p2p client factory - const multiAddrTcp = multiaddr(convertToMultiaddr(m.nodeAddress().address, this.config.p2pBroadcastPort!, 'tcp')); + const multiAddrTcp = multiaddr(convertToMultiaddr(address, this.config.p2pBroadcastPort!, 'tcp')); this.enr.setLocationMultiaddr(multiAddrTcp); this.logger.info('Multiaddr updated', { multiaddr: multiAddrTcp.toString() }); + this.emit('ip:changed', address); } public async start(): Promise { diff --git a/yarn-project/p2p/src/services/discv5/discv5_service.test.ts b/yarn-project/p2p/src/services/discv5/discv5_service.test.ts index fe4528c27ec2..d942d58d8407 100644 --- a/yarn-project/p2p/src/services/discv5/discv5_service.test.ts +++ b/yarn-project/p2p/src/services/discv5/discv5_service.test.ts @@ -144,6 +144,37 @@ describe('Discv5Service', () => { await stopNodes(...nodes); }); + it('should correct a wrong initial IP via PONG votes when enrUpdate is forced on', async () => { + const extraNodes = 3; + const nodes: DiscV5Service[] = []; + + // Simulate the scenario where getPublicIp() returned a wrong IP at startup (e.g. NAT egress IP). + // With enrUpdate forced on, PONG votes from peers should correct the ENR to 127.0.0.1. + const node = await createNode({ + p2pIp: '1.2.3.4', + config: { enrUpdate: true, addrVotesToUpdateEnr: 1, pingInterval: 200 }, + }); + await node.start(); + nodes.push(node); + + expect(node.getEnr().ip).toEqual('1.2.3.4'); + + for (let i = 1; i < extraNodes; i++) { + const n = await createNode({ config: { pingInterval: 200 } }); + await n.start(); + nodes.push(n); + } + + // Wait for the ENR IP to be corrected by PONG votes + await runDiscoveryUntil(nodes, () => node.getEnr().ip !== '1.2.3.4'); + + // ENR should now reflect the real IP (127.0.0.1) as reported by peers + expect(node.getEnr().ip).toEqual('127.0.0.1'); + expect(node.getEnr().tcp).toEqual(node.getEnr().udp); + + await stopNodes(...nodes); + }); + it('should refuse to connect to a bootstrap node with wrong chain id', async () => { const node1 = await createNode({ l1ChainId: 13, bootstrapNodeEnrVersionCheck: true }); const node2 = await createNode({ l1ChainId: 14, bootstrapNodeEnrVersionCheck: false }); diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts index 23a76ef69f47..8c09aabea4bc 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts @@ -51,9 +51,10 @@ import { yamux } from '@chainsafe/libp2p-yamux'; import { bootstrap } from '@libp2p/bootstrap'; import { identify } from '@libp2p/identify'; import { type Message, type MultiaddrConnection, type PeerId, TopicValidatorResult } from '@libp2p/interface'; -import type { ConnectionManager } from '@libp2p/interface-internal'; +import type { AddressManager, ConnectionManager } from '@libp2p/interface-internal'; import { mplex } from '@libp2p/mplex'; import { tcp } from '@libp2p/tcp'; +import { multiaddr } from '@multiformats/multiaddr'; import { ENR } from '@nethermindeth/enr'; import { createLibp2p } from 'libp2p'; @@ -182,6 +183,9 @@ export class LibP2PService extends WithTracer implements P2PService { private gossipSubEventHandler: (e: CustomEvent) => void; + private ipChangedHandler?: (ip: string) => void; + private discoveredP2pIp?: string; + private instrumentation: P2PInstrumentation; private telemetry: TelemetryClient; @@ -452,8 +456,9 @@ export class LibP2PService extends WithTracer implements P2PService { topics: topicScoreParams, }), }) as (components: GossipSubComponents) => GossipSub, - components: (components: { connectionManager: ConnectionManager }) => ({ + components: (components: { connectionManager: ConnectionManager; addressManager: AddressManager }) => ({ connectionManager: components.connectionManager, + addressManager: components.addressManager, }), }, logger: createLibp2pComponentLogger(logger.module, logger.getBindings()), @@ -510,12 +515,11 @@ export class LibP2PService extends WithTracer implements P2PService { throw new Error('P2P service already started'); } - // Get listen & announce addresses for logging const { p2pIp, p2pPort } = this.config; - if (!p2pIp) { - throw new Error('Announce address not provided.'); + if (!p2pIp && !this.config.queryForIp) { + throw new Error('Announce address not provided and queryForIp is not enabled.'); } - const announceTcpMultiaddr = convertToMultiaddr(p2pIp, p2pPort, 'tcp'); + const announceTcpMultiaddr = p2pIp ? convertToMultiaddr(p2pIp, p2pPort, 'tcp') : undefined; // Create request response protocol handlers const txHandler = reqRespTxHandler(this.mempools); @@ -569,6 +573,26 @@ export class LibP2PService extends WithTracer implements P2PService { if (!this.config.p2pDiscoveryDisabled) { await this.peerDiscoveryService.start(); } + + // Bridge discv5 IP changes to libp2p's AddressManager so peers see the updated address + if (this.config.queryForIp) { + this.ipChangedHandler = (ip: string) => { + const addressManager = this.node.services.components.addressManager; + const newAddr = multiaddr(convertToMultiaddr(ip, this.config.p2pPort, 'tcp')); + + if (this.discoveredP2pIp) { + const oldAddr = multiaddr(convertToMultiaddr(this.discoveredP2pIp, this.config.p2pPort, 'tcp')); + addressManager.removeObservedAddr(oldAddr); + } + + addressManager.addObservedAddr(newAddr); + addressManager.confirmObservedAddr(newAddr); + this.discoveredP2pIp = ip; + this.logger.info('Public IP discovered via discv5', { ip }); + }; + this.peerDiscoveryService.on('ip:changed', this.ipChangedHandler); + } + this.discoveryRunningPromise = new RunningPromise( async () => { await this.peerManager.heartbeat(); @@ -594,6 +618,11 @@ export class LibP2PService extends WithTracer implements P2PService { // Remove gossip sub listener this.node.services.pubsub.removeEventListener(GossipSubEvent.MESSAGE, this.gossipSubEventHandler); + if (this.ipChangedHandler) { + this.peerDiscoveryService.removeListener('ip:changed', this.ipChangedHandler); + this.ipChangedHandler = undefined; + } + // Stop peer manager this.logger.debug('Stopping peer manager...'); await this.peerManager.stop(); diff --git a/yarn-project/p2p/src/services/service.ts b/yarn-project/p2p/src/services/service.ts index 3151973b31a0..f7f5e09267d2 100644 --- a/yarn-project/p2p/src/services/service.ts +++ b/yarn-project/p2p/src/services/service.ts @@ -201,6 +201,9 @@ export interface PeerDiscoveryService extends EventEmitter { on(event: 'peer:discovered', listener: (enr: ENR) => void): this; emit(event: 'peer:discovered', enr: ENR): boolean; + on(event: 'ip:changed', listener: (ip: string) => void): this; + emit(event: 'ip:changed', ip: string): boolean; + getStatus(): PeerDiscoveryState; getEnr(): ENR | undefined; diff --git a/yarn-project/p2p/src/test-helpers/mock-pubsub.ts b/yarn-project/p2p/src/test-helpers/mock-pubsub.ts index cf48654e0aff..8551378222e4 100644 --- a/yarn-project/p2p/src/test-helpers/mock-pubsub.ts +++ b/yarn-project/p2p/src/test-helpers/mock-pubsub.ts @@ -212,6 +212,13 @@ export class MockPubSub implements PubSubLibp2p { get services() { return { pubsub: this.gossipSub, + components: { + addressManager: { + addObservedAddr: () => {}, + confirmObservedAddr: () => {}, + removeObservedAddr: () => {}, + }, + }, }; } diff --git a/yarn-project/p2p/src/test-helpers/reqresp-nodes.ts b/yarn-project/p2p/src/test-helpers/reqresp-nodes.ts index 72f9145ab1fb..b74d0304217e 100644 --- a/yarn-project/p2p/src/test-helpers/reqresp-nodes.ts +++ b/yarn-project/p2p/src/test-helpers/reqresp-nodes.ts @@ -76,6 +76,10 @@ export async function createLibp2pNode( identify: identify({ protocolPrefix: 'aztec', }), + components: (components: { connectionManager: any; addressManager: any }) => ({ + connectionManager: components.connectionManager, + addressManager: components.addressManager, + }), }, }; diff --git a/yarn-project/p2p/src/util.ts b/yarn-project/p2p/src/util.ts index 37bba2f5f0b9..4003c0402a7d 100644 --- a/yarn-project/p2p/src/util.ts +++ b/yarn-project/p2p/src/util.ts @@ -7,7 +7,7 @@ import type { GossipSub } from '@chainsafe/libp2p-gossipsub'; import { generateKeyPair, marshalPrivateKey, unmarshalPrivateKey } from '@libp2p/crypto/keys'; import type { Identify } from '@libp2p/identify'; import type { PeerId, PrivateKey } from '@libp2p/interface'; -import type { ConnectionManager } from '@libp2p/interface-internal'; +import type { AddressManager, ConnectionManager } from '@libp2p/interface-internal'; import { createFromPrivKey } from '@libp2p/peer-id-factory'; import { resolve } from 'dns/promises'; import { promises as fs } from 'fs'; @@ -31,6 +31,9 @@ export interface PubSubLibp2p extends Pick & { score: Pick }; + components: { + addressManager: Pick; + }; }; } @@ -39,6 +42,7 @@ export type FullLibp2p = Libp2p<{ pubsub: GossipSub; components: { connectionManager: ConnectionManager; + addressManager: AddressManager; }; }>; @@ -102,26 +106,15 @@ function addressToMultiAddressType(address: string): 'ip4' | 'ip6' | 'dns' { } } -export async function configureP2PClientAddresses( - _config: P2PConfig & DataStoreConfig, -): Promise { +export function configureP2PClientAddresses(_config: P2PConfig & DataStoreConfig): P2PConfig & DataStoreConfig { const config = { ..._config }; - const { p2pIp, queryForIp, p2pBroadcastPort, p2pPort } = config; + const { p2pBroadcastPort, p2pPort } = config; // If no broadcast port is provided, use the given p2p port as the broadcast port if (!p2pBroadcastPort) { config.p2pBroadcastPort = p2pPort; } - // check if no announce IP was provided - if (!p2pIp) { - if (queryForIp) { - const publicIp = await getPublicIp(); - config.p2pIp = publicIp; - } - } - // TODO(md): guard against setting a local ip address as the announce ip - return config; } From b44ec2751220ba898cac2d8e00748b95b2b0fb41 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 8 Apr 2026 14:18:50 +0000 Subject: [PATCH 02/10] allow custom image deployment --- .github/workflows/deploy-network.yml | 48 +++++++++++++++------------ .github/workflows/deploy-next-net.yml | 4 +-- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index f99adb2d1e8c..7aede837ab59 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -10,11 +10,11 @@ on: required: true type: string semver: - description: "Semver version (e.g., 2.3.4)" - required: true + description: "Semver version (e.g., 2.3.4). Required unless docker_image is provided." + required: false type: string - docker_image_tag: - description: "Full docker image tag (optional, defaults to semver)" + docker_image: + description: "Full docker image (e.g., aztecprotocol/aztec:2.3.4 or myregistry/aztec:custom). Defaults to aztecprotocol/aztec:" required: false type: string ref: @@ -46,11 +46,11 @@ on: - testnet - mainnet semver: - description: "Semver version (e.g., 2.3.4)" - required: true + description: "Semver version (e.g., 2.3.4). Required unless docker_image is provided." + required: false type: string - docker_image_tag: - description: "Full docker image tag (optional, defaults to semver)" + docker_image: + description: "Full docker image (e.g., aztecprotocol/aztec:2.3.4 or myregistry/aztec:custom). Defaults to aztecprotocol/aztec:" required: false type: string namespace: @@ -68,7 +68,7 @@ on: type: string concurrency: - group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.semver }}-${{ github.ref || github.ref_name }} + group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.docker_image || inputs.semver }}-${{ github.ref || github.ref_name }} cancel-in-progress: true jobs: @@ -112,16 +112,22 @@ jobs: exit 1 fi - # Validate semver format - if ! echo "${{ inputs.semver }}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-.*)?$'; then - echo "Error: Invalid semver format '${{ inputs.semver }}'. Expected format: X.Y.Z or X.Y.Z-suffix" + # Need at least one of semver or docker_image + if [[ -z "${{ inputs.semver }}" && -z "${{ inputs.docker_image }}" ]]; then + echo "Error: Either semver or docker_image must be provided" exit 1 fi - # Extract major version for v2 check - major_version="${{ inputs.semver }}" - major_version="${major_version%%.*}" - echo "MAJOR_VERSION=$major_version" >> $GITHUB_ENV + # Validate semver format if provided + if [[ -n "${{ inputs.semver }}" ]]; then + if ! echo "${{ inputs.semver }}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-.*)?$'; then + echo "Error: Invalid semver format '${{ inputs.semver }}'. Expected format: X.Y.Z or X.Y.Z-suffix" + exit 1 + fi + major_version="${{ inputs.semver }}" + major_version="${major_version%%.*}" + echo "MAJOR_VERSION=$major_version" >> $GITHUB_ENV + fi - name: Store the GCP key in a file env: @@ -166,12 +172,12 @@ jobs: RUN_ID: ${{ github.run_id }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} GOOGLE_APPLICATION_CREDENTIALS: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} - REF_NAME: "v${{ inputs.semver }}" + REF_NAME: ${{ inputs.semver && format('v{0}', inputs.semver) || '' }} GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} NAMESPACE: ${{ inputs.namespace }} - AZTEC_DOCKER_IMAGE: "aztecprotocol/aztec:${{ inputs.docker_image_tag || inputs.semver }}" + AZTEC_DOCKER_IMAGE: ${{ inputs.docker_image || format('aztecprotocol/aztec:{0}', inputs.semver) }} CREATE_ROLLUP_CONTRACTS: ${{ inputs.deploy_contracts == true && 'true' || '' }} - PROVER_AGENT_DOCKER_IMAGE: "aztecprotocol/aztec-prover-agent:${{ inputs.docker_image_tag || inputs.semver }}" + PROVER_AGENT_DOCKER_IMAGE: ${{ inputs.semver && format('aztecprotocol/aztec-prover-agent:{0}', inputs.semver) || inputs.docker_image }} run: | echo "Deploying network: ${{ inputs.network }}" echo "Using image: $AZTEC_DOCKER_IMAGE" @@ -200,7 +206,7 @@ jobs: echo "| Item | Value |" echo "|------|-------|" echo "| Network | \`${{ inputs.network }}\` |" - echo "| Semver | \`${{ inputs.semver }}\` |" + echo "| Image | \`${{ inputs.docker_image || format('aztecprotocol/aztec:{0}', inputs.semver) }}\` |" echo "| Ref | \`${{ steps.checkout-ref.outputs.ref }}\` |" if [[ -n "${{ inputs.source_tag }}" ]]; then echo "| Source Tag | [\`${{ inputs.source_tag }}\`](https://github.com/${{ github.repository }}/releases/tag/${{ inputs.source_tag }}) |" @@ -216,7 +222,7 @@ jobs: read -r -d '' data <" + "text": "Deploy Network workflow FAILED for *${{ inputs.network }}* (image ${{ inputs.docker_image || format('aztecprotocol/aztec:{0}', inputs.semver) }}): " } EOF curl -X POST https://slack.com/api/chat.postMessage \ diff --git a/.github/workflows/deploy-next-net.yml b/.github/workflows/deploy-next-net.yml index 60a7329b4ade..9cdacbc36f84 100644 --- a/.github/workflows/deploy-next-net.yml +++ b/.github/workflows/deploy-next-net.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: image_tag: - description: 'Docker image tag (e.g., 2.3.4, 3.0.0-nightly.20251004-amd64, or leave empty for latest nightly)' + description: "Docker image tag (e.g., 2.3.4, 3.0.0-nightly.20251004-amd64, or leave empty for latest nightly)" required: false type: string @@ -67,6 +67,6 @@ jobs: with: network: next-net semver: ${{ needs.get-image-tag.outputs.semver }} - docker_image_tag: ${{ needs.get-image-tag.outputs.tag }} + docker_image: "aztecprotocol/aztec:${{ needs.get-image-tag.outputs.tag }}" ref: ${{ github.ref }} secrets: inherit From ec30936528476a6bfc00504eacff08a17085abe5 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 8 Apr 2026 14:52:40 +0000 Subject: [PATCH 03/10] try init with getPublicIp --- spartan/environments/next-net.env | 2 ++ yarn-project/p2p/src/client/factory.ts | 2 +- yarn-project/p2p/src/util.ts | 12 ++++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/spartan/environments/next-net.env b/spartan/environments/next-net.env index 5d46d171f9d6..243589a11426 100644 --- a/spartan/environments/next-net.env +++ b/spartan/environments/next-net.env @@ -29,6 +29,8 @@ L1_TX_FAILED_STORE=gs://aztec-develop/next-net/failed-l1-txs TEST_ACCOUNTS=true SPONSORED_FPC=true +LOG_LEVEL=debug + SEQ_MIN_TX_PER_BLOCK=1 SEQ_MAX_TX_PER_CHECKPOINT=7 diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index 3c6c70b16ad7..dbe16e6debe4 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -57,7 +57,7 @@ export async function createP2PClient( telemetry: TelemetryClient = getTelemetryClient(), deps: P2PClientDeps = {}, ) { - const config = configureP2PClientAddresses({ + const config = await configureP2PClientAddresses({ ...inputConfig, dataStoreMapSizeKb: inputConfig.p2pStoreMapSizeKb ?? inputConfig.dataStoreMapSizeKb, }); diff --git a/yarn-project/p2p/src/util.ts b/yarn-project/p2p/src/util.ts index 4003c0402a7d..d57f18a1b697 100644 --- a/yarn-project/p2p/src/util.ts +++ b/yarn-project/p2p/src/util.ts @@ -106,15 +106,23 @@ function addressToMultiAddressType(address: string): 'ip4' | 'ip6' | 'dns' { } } -export function configureP2PClientAddresses(_config: P2PConfig & DataStoreConfig): P2PConfig & DataStoreConfig { +export async function configureP2PClientAddresses( + _config: P2PConfig & DataStoreConfig, +): Promise { const config = { ..._config }; - const { p2pBroadcastPort, p2pPort } = config; + const { p2pIp, queryForIp, p2pBroadcastPort, p2pPort } = config; // If no broadcast port is provided, use the given p2p port as the broadcast port if (!p2pBroadcastPort) { config.p2pBroadcastPort = p2pPort; } + // Resolve the initial public IP so the ENR and announce address are set at startup. + // If queryForIp is enabled, discv5 will also track IP changes at runtime via enrUpdate. + if (!p2pIp && queryForIp) { + config.p2pIp = await getPublicIp(); + } + return config; } From 263a7a4708762993ee8f294b67e1ddf775a234e8 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 8 Apr 2026 15:25:53 +0000 Subject: [PATCH 04/10] 60s --- yarn-project/p2p/src/services/discv5/discV5_service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/p2p/src/services/discv5/discV5_service.ts b/yarn-project/p2p/src/services/discv5/discV5_service.ts index 251e62ef4097..7de74dc6249d 100644 --- a/yarn-project/p2p/src/services/discv5/discV5_service.ts +++ b/yarn-project/p2p/src/services/discv5/discV5_service.ts @@ -97,7 +97,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService requestTimeout: 2000, allowUnverifiedSessions: true, enrUpdate: config.queryForIp || !p2pIp, - pingInterval: config.queryForIp ? 10_000 : 300_000, + pingInterval: config.queryForIp ? 60_000 : 300_000, ...configOverrides.config, }, metricsRegistry, From 72127a73c1b4b55f562aca2c75a50fa92323145b Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 9 Apr 2026 13:08:05 +0000 Subject: [PATCH 05/10] feat(p2p): detect and track announce IP changes at runtime Keep getPublicIp() at startup so the ENR has a valid IP from the start (required for discv5 routing table insertion), and enable discv5 enrUpdate so PONG votes can correct the IP at runtime if it changes. Also improve P2P observability: log KAD table state in peer manager heartbeats, log ENR additions with multiaddrs, and have the bootnode explicitly addEnr on discovery to fix routing table gaps. --- .github/workflows/deploy-network.yml | 48 ++++++++----------- yarn-project/p2p/src/bootstrap/bootstrap.ts | 8 ++++ yarn-project/p2p/src/client/factory.ts | 9 ++++ .../p2p/src/services/discv5/discV5_service.ts | 38 +++++++++++++-- .../peer-manager/peer_manager.test.ts | 1 + .../src/services/peer-manager/peer_manager.ts | 3 ++ 6 files changed, 75 insertions(+), 32 deletions(-) diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index 7aede837ab59..f99adb2d1e8c 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -10,11 +10,11 @@ on: required: true type: string semver: - description: "Semver version (e.g., 2.3.4). Required unless docker_image is provided." - required: false + description: "Semver version (e.g., 2.3.4)" + required: true type: string - docker_image: - description: "Full docker image (e.g., aztecprotocol/aztec:2.3.4 or myregistry/aztec:custom). Defaults to aztecprotocol/aztec:" + docker_image_tag: + description: "Full docker image tag (optional, defaults to semver)" required: false type: string ref: @@ -46,11 +46,11 @@ on: - testnet - mainnet semver: - description: "Semver version (e.g., 2.3.4). Required unless docker_image is provided." - required: false + description: "Semver version (e.g., 2.3.4)" + required: true type: string - docker_image: - description: "Full docker image (e.g., aztecprotocol/aztec:2.3.4 or myregistry/aztec:custom). Defaults to aztecprotocol/aztec:" + docker_image_tag: + description: "Full docker image tag (optional, defaults to semver)" required: false type: string namespace: @@ -68,7 +68,7 @@ on: type: string concurrency: - group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.docker_image || inputs.semver }}-${{ github.ref || github.ref_name }} + group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.semver }}-${{ github.ref || github.ref_name }} cancel-in-progress: true jobs: @@ -112,22 +112,16 @@ jobs: exit 1 fi - # Need at least one of semver or docker_image - if [[ -z "${{ inputs.semver }}" && -z "${{ inputs.docker_image }}" ]]; then - echo "Error: Either semver or docker_image must be provided" + # Validate semver format + if ! echo "${{ inputs.semver }}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-.*)?$'; then + echo "Error: Invalid semver format '${{ inputs.semver }}'. Expected format: X.Y.Z or X.Y.Z-suffix" exit 1 fi - # Validate semver format if provided - if [[ -n "${{ inputs.semver }}" ]]; then - if ! echo "${{ inputs.semver }}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-.*)?$'; then - echo "Error: Invalid semver format '${{ inputs.semver }}'. Expected format: X.Y.Z or X.Y.Z-suffix" - exit 1 - fi - major_version="${{ inputs.semver }}" - major_version="${major_version%%.*}" - echo "MAJOR_VERSION=$major_version" >> $GITHUB_ENV - fi + # Extract major version for v2 check + major_version="${{ inputs.semver }}" + major_version="${major_version%%.*}" + echo "MAJOR_VERSION=$major_version" >> $GITHUB_ENV - name: Store the GCP key in a file env: @@ -172,12 +166,12 @@ jobs: RUN_ID: ${{ github.run_id }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} GOOGLE_APPLICATION_CREDENTIALS: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} - REF_NAME: ${{ inputs.semver && format('v{0}', inputs.semver) || '' }} + REF_NAME: "v${{ inputs.semver }}" GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} NAMESPACE: ${{ inputs.namespace }} - AZTEC_DOCKER_IMAGE: ${{ inputs.docker_image || format('aztecprotocol/aztec:{0}', inputs.semver) }} + AZTEC_DOCKER_IMAGE: "aztecprotocol/aztec:${{ inputs.docker_image_tag || inputs.semver }}" CREATE_ROLLUP_CONTRACTS: ${{ inputs.deploy_contracts == true && 'true' || '' }} - PROVER_AGENT_DOCKER_IMAGE: ${{ inputs.semver && format('aztecprotocol/aztec-prover-agent:{0}', inputs.semver) || inputs.docker_image }} + PROVER_AGENT_DOCKER_IMAGE: "aztecprotocol/aztec-prover-agent:${{ inputs.docker_image_tag || inputs.semver }}" run: | echo "Deploying network: ${{ inputs.network }}" echo "Using image: $AZTEC_DOCKER_IMAGE" @@ -206,7 +200,7 @@ jobs: echo "| Item | Value |" echo "|------|-------|" echo "| Network | \`${{ inputs.network }}\` |" - echo "| Image | \`${{ inputs.docker_image || format('aztecprotocol/aztec:{0}', inputs.semver) }}\` |" + echo "| Semver | \`${{ inputs.semver }}\` |" echo "| Ref | \`${{ steps.checkout-ref.outputs.ref }}\` |" if [[ -n "${{ inputs.source_tag }}" ]]; then echo "| Source Tag | [\`${{ inputs.source_tag }}\`](https://github.com/${{ github.repository }}/releases/tag/${{ inputs.source_tag }}) |" @@ -222,7 +216,7 @@ jobs: read -r -d '' data <" + "text": "Deploy Network workflow FAILED for *${{ inputs.network }}* (version ${{ inputs.semver }}): " } EOF curl -X POST https://slack.com/api/chat.postMessage \ diff --git a/yarn-project/p2p/src/bootstrap/bootstrap.ts b/yarn-project/p2p/src/bootstrap/bootstrap.ts index f0e78d14e9d7..86ed966e6544 100644 --- a/yarn-project/p2p/src/bootstrap/bootstrap.ts +++ b/yarn-project/p2p/src/bootstrap/bootstrap.ts @@ -86,6 +86,14 @@ export class BootstrapNode implements P2PBootstrapApi { this.node.on('discovered', async (enr: SignableENR) => { const addr = await enr.getFullMultiaddr('udp'); this.logger.verbose(`Discovered new peer`, { enr: enr.encodeTxt(), addr: addr?.toString() }); + // discv5's discovered() only updates routing table entries that already exist. Nodes that + // established a session with an empty-IP ENR are never inserted, so even after their ENR + // gains a valid socket address the routing table stays empty and FINDNODE always returns 0 + // peers. Calling addEnr() here does an insertOrUpdate regardless of prior state, fixing + // the routing table so these nodes become discoverable to other peers. + if (addr) { + this.node.addEnr(enr); + } }); try { diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index dbe16e6debe4..ac9838aae311 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -64,6 +64,15 @@ export async function createP2PClient( const logger = deps.logger ?? createLogger('p2p'); + logger.info('P2P client address config resolved', { + p2pIp: config.p2pIp ?? 'not set', + queryForIp: config.queryForIp, + p2pPort: config.p2pPort, + p2pBroadcastPort: config.p2pBroadcastPort, + listenAddress: config.listenAddress, + bootstrapNodeCount: config.bootstrapNodes.length, + }); + if (config.bootstrapNodes.length === 0) { logger.warn( 'No bootstrap nodes have been provided. Set the BOOTSTRAP_NODES environment variable in order to join the P2P network', diff --git a/yarn-project/p2p/src/services/discv5/discV5_service.ts b/yarn-project/p2p/src/services/discv5/discV5_service.ts index 7de74dc6249d..3560a22bac0d 100644 --- a/yarn-project/p2p/src/services/discv5/discV5_service.ts +++ b/yarn-project/p2p/src/services/discv5/discV5_service.ts @@ -53,6 +53,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService configOverrides: Partial = {}, ) { super(); + const { p2pIp, p2pPort, p2pBroadcastPort, bootstrapNodes, trustedPeers, privatePeers } = config; this.bootstrapNodeEnrs = bootstrapNodes.map(x => ENR.decodeTxt(x)); @@ -97,7 +98,8 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService requestTimeout: 2000, allowUnverifiedSessions: true, enrUpdate: config.queryForIp || !p2pIp, - pingInterval: config.queryForIp ? 60_000 : 300_000, + pingInterval: config.queryForIp ? 10_000 : 300_000, + addrVotesToUpdateEnr: config.queryForIp ? 1 : 10, ...configOverrides.config, }, metricsRegistry, @@ -132,8 +134,14 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService // We want to update our tcp port to match the udp port // p2pBroadcastPort is optional on config, however it is set to default within the p2p client factory const multiAddrTcp = multiaddr(convertToMultiaddr(address, this.config.p2pBroadcastPort!, 'tcp')); + const prevIp = this.enr.toENR().ip ?? 'none'; this.enr.setLocationMultiaddr(multiAddrTcp); - this.logger.info('Multiaddr updated', { multiaddr: multiAddrTcp.toString() }); + this.logger.info('Multiaddr updated via discv5 PONG vote', { + prevIp, + newIp: address, + multiaddr: multiAddrTcp.toString(), + }); + this.emit('ip:changed', address); } @@ -204,11 +212,21 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService await sleep(delayBeforeStart - msSinceStart); } + const kadBefore = this.discv5.kadValues().length; try { await this.discv5.findRandomNode(); } catch (err) { this.logger.error(`Error running discV5 random node query: ${err}`); } + const kadAfter = this.discv5.kadValues().length; + const enr = this.enr.toENR(); + this.logger.debug(`DiscV5 random node query complete`, { + kadBefore, + kadAfter, + enrIp: enr.ip ?? 'none', + enrUdp: enr.udp ?? 'none', + enrTcp: enr.tcp ?? 'none', + }); } public getKadValues(): ENR[] { @@ -247,7 +265,12 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService private async onEnrAdded(enr: ENR) { const multiAddrTcp = await enr.getFullMultiaddr('tcp'); const multiAddrUdp = await enr.getFullMultiaddr('udp'); - this.logger.debug(`Added ENR ${enr.encodeTxt()}`, { multiAddrTcp, multiAddrUdp, nodeId: enr.nodeId }); + this.logger.info(`DiscV5 ENR added (peer discovered via DHT)`, { + nodeId: enr.nodeId, + multiAddrTcp: multiAddrTcp?.toString() ?? 'none', + multiAddrUdp: multiAddrUdp?.toString() ?? 'none', + isBootnode: this.isOurBootnode(enr), + }); this.onDiscovered(enr); } @@ -280,7 +303,10 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService // Check the peer is an aztec peer const value = enr.kvs.get(AZTEC_ENR_KEY); if (!value) { - this.logger.debug(`Peer node ${enr.nodeId} does not have aztec key in ENR`); + this.logger.info(`Discovered peer has no aztec key in ENR, ignoring`, { + nodeId: enr.nodeId, + enrIp: enr.ip ?? 'none', + }); return false; } @@ -292,7 +318,9 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService return true; } catch (err: any) { if (err.name === 'ComponentsVersionsError') { - this.logger.debug(`Peer node ${enr.nodeId} has incorrect version: ${err.message}`, { + this.logger.info(`Discovered peer has wrong version, ignoring`, { + nodeId: enr.nodeId, + enrIp: enr.ip ?? 'none', compressedVersion, expected: this.versions, }); diff --git a/yarn-project/p2p/src/services/peer-manager/peer_manager.test.ts b/yarn-project/p2p/src/services/peer-manager/peer_manager.test.ts index f108e28bc3f9..ae04181fecfb 100644 --- a/yarn-project/p2p/src/services/peer-manager/peer_manager.test.ts +++ b/yarn-project/p2p/src/services/peer-manager/peer_manager.test.ts @@ -43,6 +43,7 @@ describe('PeerManager', () => { off: jest.fn(), isBootstrapPeer: jest.fn().mockReturnValue(false), runRandomNodesQuery: jest.fn(), + getKadValues: jest.fn().mockReturnValue([]), }; const mockEpochCache = mock(); diff --git a/yarn-project/p2p/src/services/peer-manager/peer_manager.ts b/yarn-project/p2p/src/services/peer-manager/peer_manager.ts index 470dae309aeb..db9f89f06063 100644 --- a/yarn-project/p2p/src/services/peer-manager/peer_manager.ts +++ b/yarn-project/p2p/src/services/peer-manager/peer_manager.ts @@ -522,11 +522,14 @@ export class PeerManager implements PeerManagerInterface { const peersToConnect = this.config.maxPeerCount - healthyConnections.length - protectedPeerCount; const logLevel = this.heartbeatCounter % this.displayPeerCountsPeerHeartbeat === 0 ? 'info' : 'debug'; + const kadValues = this.peerDiscoveryService.getKadValues(); this.logger[logLevel](`Connected to ${healthyConnections.length + this.trustedPeers.size} peers`, { discoveredConnections: healthyConnections.length, protectedConnections: protectedPeerCount, maxPeerCount: this.config.maxPeerCount, cachedPeers: this.cachedPeers.size, + discv5KadPeers: kadValues.length, + discv5KadAddrs: kadValues.slice(0, 5).map(e => `${e.ip ?? 'no-ip'}:${e.udp ?? 'no-udp'}`), ...this.peerScoring.getStats(), }); From 7fff6de29bd98faf4fd71e9f84ea631f3bd13e14 Mon Sep 17 00:00:00 2001 From: spypsy Date: Thu, 9 Apr 2026 15:51:48 +0000 Subject: [PATCH 06/10] allow external docker img again --- .github/workflows/deploy-network.yml | 67 +++++++++++++++++----------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index 046c44b425ef..c32f35a3b69e 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -9,12 +9,12 @@ on: description: "Network to deploy (e.g., staging-public, testnet, next-net)" required: true type: string - semver: - description: "Semver version (e.g., 2.3.4)" - required: true + aztec_docker_image: + description: "Full Aztec docker image (e.g., aztecprotocol/aztec:2.3.4). If not set, constructed from semver." + required: false type: string - docker_image_tag: - description: "Full docker image tag (optional, defaults to semver)" + semver: + description: "Semver version (e.g., 2.3.4). Used to construct docker image if aztec_docker_image is not set." required: false type: string ref: @@ -49,12 +49,12 @@ on: - staging-public - testnet - mainnet - semver: - description: "Semver version (e.g., 2.3.4)" - required: true + aztec_docker_image: + description: "Full Aztec docker image (e.g., aztecprotocol/aztec:2.3.4). If not set, constructed from semver." + required: false type: string - docker_image_tag: - description: "Full docker image tag (optional, defaults to semver)" + semver: + description: "Semver version (e.g., 2.3.4). Used to construct docker image if aztec_docker_image is not set." required: false type: string namespace: @@ -76,7 +76,7 @@ on: type: string concurrency: - group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.semver }}-${{ github.ref || github.ref_name }} + group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.aztec_docker_image || inputs.semver }}-${{ github.ref || github.ref_name }} cancel-in-progress: true jobs: @@ -120,16 +120,31 @@ jobs: exit 1 fi - # Validate semver format - if ! echo "${{ inputs.semver }}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-.*)?$'; then - echo "Error: Invalid semver format '${{ inputs.semver }}'. Expected format: X.Y.Z or X.Y.Z-suffix" + # Require at least one of aztec_docker_image or semver + if [[ -z "${{ inputs.aztec_docker_image }}" && -z "${{ inputs.semver }}" ]]; then + echo "Error: Either 'aztec_docker_image' or 'semver' must be provided" exit 1 fi - # Extract major version for v2 check - major_version="${{ inputs.semver }}" - major_version="${major_version%%.*}" - echo "MAJOR_VERSION=$major_version" >> $GITHUB_ENV + # Validate semver format if provided + if [[ -n "${{ inputs.semver }}" ]]; then + if ! echo "${{ inputs.semver }}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-.*)?$'; then + echo "Error: Invalid semver format '${{ inputs.semver }}'. Expected format: X.Y.Z or X.Y.Z-suffix" + exit 1 + fi + fi + + # Resolve the docker image + if [[ -n "${{ inputs.aztec_docker_image }}" ]]; then + AZTEC_DOCKER_IMAGE="${{ inputs.aztec_docker_image }}" + else + AZTEC_DOCKER_IMAGE="aztecprotocol/aztec:${{ inputs.semver }}" + fi + echo "AZTEC_DOCKER_IMAGE=$AZTEC_DOCKER_IMAGE" >> $GITHUB_ENV + + # Derive prover agent image by replacing /aztec: with /aztec-prover-agent: + PROVER_AGENT_DOCKER_IMAGE="${AZTEC_DOCKER_IMAGE/\/aztec:/\/aztec-prover-agent:}" + echo "PROVER_AGENT_DOCKER_IMAGE=$PROVER_AGENT_DOCKER_IMAGE" >> $GITHUB_ENV - name: Store the GCP key in a file env: @@ -174,12 +189,12 @@ jobs: RUN_ID: ${{ github.run_id }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} GOOGLE_APPLICATION_CREDENTIALS: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} - REF_NAME: "v${{ inputs.semver }}" + REF_NAME: ${{ inputs.semver && format('v{0}', inputs.semver) || '' }} GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} NAMESPACE: ${{ inputs.namespace }} - AZTEC_DOCKER_IMAGE: "aztecprotocol/aztec:${{ inputs.docker_image_tag || inputs.semver }}" + AZTEC_DOCKER_IMAGE: ${{ env.AZTEC_DOCKER_IMAGE }} CREATE_ROLLUP_CONTRACTS: ${{ inputs.deploy_contracts == true && 'true' || '' }} - PROVER_AGENT_DOCKER_IMAGE: "aztecprotocol/aztec-prover-agent:${{ inputs.docker_image_tag || inputs.semver }}" + PROVER_AGENT_DOCKER_IMAGE: ${{ env.PROVER_AGENT_DOCKER_IMAGE }} VALIDATOR_HA_DOCKER_IMAGE: ${{ inputs.ha_docker_image || '' }} run: | echo "Deploying network: ${{ inputs.network }}" @@ -209,7 +224,7 @@ jobs: echo "| Item | Value |" echo "|------|-------|" echo "| Network | \`${{ inputs.network }}\` |" - echo "| Semver | \`${{ inputs.semver }}\` |" + echo "| Docker Image | \`${{ env.AZTEC_DOCKER_IMAGE }}\` |" echo "| Ref | \`${{ steps.checkout-ref.outputs.ref }}\` |" if [[ -n "${{ inputs.source_tag }}" ]]; then echo "| Source Tag | [\`${{ inputs.source_tag }}\`](https://github.com/${{ github.repository }}/releases/tag/${{ inputs.source_tag }}) |" @@ -229,7 +244,7 @@ jobs: CHANNEL="#alerts-${{ inputs.network }}" RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - TEXT="Deploy Network workflow FAILED for *${{ inputs.network }}* (version ${{ inputs.semver }}): <${RUN_URL}|View Run> (🤖)" + TEXT="Deploy Network workflow FAILED for *${{ inputs.network }}* (image ${{ env.AZTEC_DOCKER_IMAGE }}): <${RUN_URL}|View Run> (🤖)" # Post to Slack and capture timestamp for permalink RESP=$(curl -sS -X POST https://slack.com/api/chat.postMessage \ @@ -247,11 +262,11 @@ jobs: fi # Dispatch ClaudeBox to investigate the failure - PROMPT="Deployment of ${{ inputs.network }} (version ${{ inputs.semver }}) failed. \ + PROMPT="Deployment of ${{ inputs.network }} (image ${{ env.AZTEC_DOCKER_IMAGE }}) failed. \ Follow .claude/claudebox/deploy-investigation.md to investigate. \ GitHub Actions run: ${RUN_URL}. \ - Network: ${{ inputs.network }}. Version: ${{ inputs.semver }}. \ - Docker image: ${{ inputs.docker_image_tag || inputs.semver }}. \ + Network: ${{ inputs.network }}. \ + Docker image: ${{ env.AZTEC_DOCKER_IMAGE }}. \ Git ref: ${{ steps.checkout-ref.outputs.ref }}. \ Namespace: ${{ inputs.namespace || inputs.network }}. \ Deploy contracts: ${{ inputs.deploy_contracts }}." From 9509a30a16c73f0e153d5c6c3041a8cdba0e26aa Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 10 Apr 2026 11:10:32 +0000 Subject: [PATCH 07/10] fix prover agent img --- .github/workflows/deploy-network.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index c32f35a3b69e..28dc83cda749 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -142,9 +142,11 @@ jobs: fi echo "AZTEC_DOCKER_IMAGE=$AZTEC_DOCKER_IMAGE" >> $GITHUB_ENV - # Derive prover agent image by replacing /aztec: with /aztec-prover-agent: - PROVER_AGENT_DOCKER_IMAGE="${AZTEC_DOCKER_IMAGE/\/aztec:/\/aztec-prover-agent:}" - echo "PROVER_AGENT_DOCKER_IMAGE=$PROVER_AGENT_DOCKER_IMAGE" >> $GITHUB_ENV + # Only use the separate prover-agent image for official semver builds; + # for custom images, let the deploy script fall back to AZTEC_DOCKER_IMAGE + if [[ -n "${{ inputs.semver }}" ]]; then + echo "PROVER_AGENT_DOCKER_IMAGE=aztecprotocol/aztec-prover-agent:${{ inputs.semver }}" >> $GITHUB_ENV + fi - name: Store the GCP key in a file env: @@ -194,7 +196,7 @@ jobs: NAMESPACE: ${{ inputs.namespace }} AZTEC_DOCKER_IMAGE: ${{ env.AZTEC_DOCKER_IMAGE }} CREATE_ROLLUP_CONTRACTS: ${{ inputs.deploy_contracts == true && 'true' || '' }} - PROVER_AGENT_DOCKER_IMAGE: ${{ env.PROVER_AGENT_DOCKER_IMAGE }} + PROVER_AGENT_DOCKER_IMAGE: ${{ env.PROVER_AGENT_DOCKER_IMAGE || env.AZTEC_DOCKER_IMAGE }} VALIDATOR_HA_DOCKER_IMAGE: ${{ inputs.ha_docker_image || '' }} run: | echo "Deploying network: ${{ inputs.network }}" From 508b6a7b4e23567a01c5b8d4f0906440e3c4f9df Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 14 Apr 2026 10:52:38 +0000 Subject: [PATCH 08/10] ip discovery URL fallbacks --- yarn-project/p2p/src/util.ts | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/yarn-project/p2p/src/util.ts b/yarn-project/p2p/src/util.ts index d57f18a1b697..5e16b6176e70 100644 --- a/yarn-project/p2p/src/util.ts +++ b/yarn-project/p2p/src/util.ts @@ -19,6 +19,13 @@ import type { P2PConfig } from './config.js'; const PEER_ID_DATA_DIR_FILE = 'p2p-private-key'; +const PUBLIC_IP_SERVICES = [ + 'https://api.ipify.org/', + 'https://checkip.amazonaws.com/', + 'https://ifconfig.me/ip', + 'https://icanhazip.com/', +]; + export interface PubSubLibp2p extends Pick { services: { pubsub: Pick< @@ -64,16 +71,24 @@ export function convertToMultiaddr(address: string, port: number, protocol: 'tcp } /** - * Queries the public IP address of the machine. + * Queries the public IP address of the machine, trying multiple services in order. */ export async function getPublicIp(): Promise { - const resp = await fetch('https://checkip.amazonaws.com/'); - const text = await resp.text(); - const address = text.trim(); - if (!isValidIpAddress(address)) { - throw new Error(`Received invalid IP address from checkip service: ${address}`); + const errors: string[] = []; + for (const url of PUBLIC_IP_SERVICES) { + try { + const resp = await fetch(url, { signal: AbortSignal.timeout(5000) }); + const text = await resp.text(); + const address = text.trim(); + if (isValidIpAddress(address)) { + return address; + } + errors.push(`${url}: invalid IP "${address}"`); + } catch (err: any) { + errors.push(`${url}: ${err.message ?? err}`); + } } - return address; + throw new Error(`Failed to determine public IP from all services:\n${errors.join('\n')}`); } export function isValidIpAddress(address: string): boolean { From 8ab0ba45a7ee36b8a5dceabca806e228bd8e430c Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 14 Apr 2026 11:42:05 +0000 Subject: [PATCH 09/10] nicer var order --- .github/workflows/deploy-network.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index 28dc83cda749..60aa8a20d411 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -9,14 +9,14 @@ on: description: "Network to deploy (e.g., staging-public, testnet, next-net)" required: true type: string - aztec_docker_image: - description: "Full Aztec docker image (e.g., aztecprotocol/aztec:2.3.4). If not set, constructed from semver." - required: false - type: string semver: description: "Semver version (e.g., 2.3.4). Used to construct docker image if aztec_docker_image is not set." required: false type: string + aztec_docker_image: + description: "Full Aztec docker image (e.g., aztecprotocol/aztec:2.3.4). If not set, constructed from semver." + required: false + type: string ref: description: "Git ref to checkout" required: false @@ -49,14 +49,14 @@ on: - staging-public - testnet - mainnet - aztec_docker_image: - description: "Full Aztec docker image (e.g., aztecprotocol/aztec:2.3.4). If not set, constructed from semver." - required: false - type: string semver: description: "Semver version (e.g., 2.3.4). Used to construct docker image if aztec_docker_image is not set." required: false type: string + aztec_docker_image: + description: "Full Aztec docker image (e.g., aztecprotocol/aztec:2.3.4). If not set, constructed from semver." + required: false + type: string namespace: description: "Kubernetes namespace override (optional, defaults to env file value)" required: false From f5f0d41b206d66f044aaf5c8b71afa8c7adfe3b1 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 14 Apr 2026 14:12:57 +0000 Subject: [PATCH 10/10] test updates --- yarn-project/p2p/src/client/factory.ts | 9 -- .../p2p/src/services/discv5/discV5_service.ts | 37 +----- .../services/discv5/discv5_service.test.ts | 10 +- .../services/libp2p/libp2p_service.test.ts | 108 +++++++++++++++++- .../src/services/peer-manager/peer_manager.ts | 4 +- 5 files changed, 119 insertions(+), 49 deletions(-) diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index ac9838aae311..dbe16e6debe4 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -64,15 +64,6 @@ export async function createP2PClient( const logger = deps.logger ?? createLogger('p2p'); - logger.info('P2P client address config resolved', { - p2pIp: config.p2pIp ?? 'not set', - queryForIp: config.queryForIp, - p2pPort: config.p2pPort, - p2pBroadcastPort: config.p2pBroadcastPort, - listenAddress: config.listenAddress, - bootstrapNodeCount: config.bootstrapNodes.length, - }); - if (config.bootstrapNodes.length === 0) { logger.warn( 'No bootstrap nodes have been provided. Set the BOOTSTRAP_NODES environment variable in order to join the P2P network', diff --git a/yarn-project/p2p/src/services/discv5/discV5_service.ts b/yarn-project/p2p/src/services/discv5/discV5_service.ts index 3560a22bac0d..6765619579c4 100644 --- a/yarn-project/p2p/src/services/discv5/discV5_service.ts +++ b/yarn-project/p2p/src/services/discv5/discV5_service.ts @@ -99,7 +99,6 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService allowUnverifiedSessions: true, enrUpdate: config.queryForIp || !p2pIp, pingInterval: config.queryForIp ? 10_000 : 300_000, - addrVotesToUpdateEnr: config.queryForIp ? 1 : 10, ...configOverrides.config, }, metricsRegistry, @@ -131,17 +130,9 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService private onMultiaddrUpdated(m: Multiaddr) { const address = m.nodeAddress().address; - // We want to update our tcp port to match the udp port - // p2pBroadcastPort is optional on config, however it is set to default within the p2p client factory const multiAddrTcp = multiaddr(convertToMultiaddr(address, this.config.p2pBroadcastPort!, 'tcp')); - const prevIp = this.enr.toENR().ip ?? 'none'; this.enr.setLocationMultiaddr(multiAddrTcp); - this.logger.info('Multiaddr updated via discv5 PONG vote', { - prevIp, - newIp: address, - multiaddr: multiAddrTcp.toString(), - }); - + this.logger.info('Multiaddr updated', { multiaddr: multiAddrTcp.toString() }); this.emit('ip:changed', address); } @@ -212,21 +203,11 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService await sleep(delayBeforeStart - msSinceStart); } - const kadBefore = this.discv5.kadValues().length; try { await this.discv5.findRandomNode(); } catch (err) { this.logger.error(`Error running discV5 random node query: ${err}`); } - const kadAfter = this.discv5.kadValues().length; - const enr = this.enr.toENR(); - this.logger.debug(`DiscV5 random node query complete`, { - kadBefore, - kadAfter, - enrIp: enr.ip ?? 'none', - enrUdp: enr.udp ?? 'none', - enrTcp: enr.tcp ?? 'none', - }); } public getKadValues(): ENR[] { @@ -265,12 +246,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService private async onEnrAdded(enr: ENR) { const multiAddrTcp = await enr.getFullMultiaddr('tcp'); const multiAddrUdp = await enr.getFullMultiaddr('udp'); - this.logger.info(`DiscV5 ENR added (peer discovered via DHT)`, { - nodeId: enr.nodeId, - multiAddrTcp: multiAddrTcp?.toString() ?? 'none', - multiAddrUdp: multiAddrUdp?.toString() ?? 'none', - isBootnode: this.isOurBootnode(enr), - }); + this.logger.debug(`Added ENR ${enr.encodeTxt()}`, { multiAddrTcp, multiAddrUdp, nodeId: enr.nodeId }); this.onDiscovered(enr); } @@ -303,10 +279,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService // Check the peer is an aztec peer const value = enr.kvs.get(AZTEC_ENR_KEY); if (!value) { - this.logger.info(`Discovered peer has no aztec key in ENR, ignoring`, { - nodeId: enr.nodeId, - enrIp: enr.ip ?? 'none', - }); + this.logger.debug(`Peer node ${enr.nodeId} does not have aztec key in ENR`); return false; } @@ -318,9 +291,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService return true; } catch (err: any) { if (err.name === 'ComponentsVersionsError') { - this.logger.info(`Discovered peer has wrong version, ignoring`, { - nodeId: enr.nodeId, - enrIp: enr.ip ?? 'none', + this.logger.debug(`Peer node ${enr.nodeId} has incorrect version: ${err.message}`, { compressedVersion, expected: this.versions, }); diff --git a/yarn-project/p2p/src/services/discv5/discv5_service.test.ts b/yarn-project/p2p/src/services/discv5/discv5_service.test.ts index d942d58d8407..f3fec17d4fe0 100644 --- a/yarn-project/p2p/src/services/discv5/discv5_service.test.ts +++ b/yarn-project/p2p/src/services/discv5/discv5_service.test.ts @@ -144,7 +144,7 @@ describe('Discv5Service', () => { await stopNodes(...nodes); }); - it('should correct a wrong initial IP via PONG votes when enrUpdate is forced on', async () => { + it('should correct a wrong initial IP via PONG votes and emit ip:changed', async () => { const extraNodes = 3; const nodes: DiscV5Service[] = []; @@ -157,6 +157,10 @@ describe('Discv5Service', () => { await node.start(); nodes.push(node); + // Track ip:changed events (these are what libp2p_service bridges to its AddressManager) + const ipChanges: string[] = []; + node.on('ip:changed', (ip: string) => ipChanges.push(ip)); + expect(node.getEnr().ip).toEqual('1.2.3.4'); for (let i = 1; i < extraNodes; i++) { @@ -172,6 +176,10 @@ describe('Discv5Service', () => { expect(node.getEnr().ip).toEqual('127.0.0.1'); expect(node.getEnr().tcp).toEqual(node.getEnr().udp); + // ip:changed should have fired with the corrected IP + expect(ipChanges.length).toBeGreaterThanOrEqual(1); + expect(ipChanges[ipChanges.length - 1]).toEqual('127.0.0.1'); + await stopNodes(...nodes); }); diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts index 4d5ee8f0957a..4ae7dba359a3 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts @@ -24,6 +24,7 @@ import { ServerWorldStateSynchronizer } from '@aztec/world-state'; import { afterEach, beforeEach, describe, expect, it, jest } from '@jest/globals'; import type { Message, PeerId } from '@libp2p/interface'; import { TopicValidatorResult } from '@libp2p/interface'; +import EventEmitter from 'events'; import { type MockProxy, mock } from 'jest-mock-extended'; import { type P2PConfig, p2pConfigMappings } from '../../config.js'; @@ -991,6 +992,86 @@ describe('LibP2PService', () => { expect(allNodesCheckpointReceivedCallback).toHaveBeenCalledWith(expect.any(Object), expect.anything()); }); }); + + describe('ip:changed bridge to AddressManager', () => { + let peerDiscoveryEmitter: PeerDiscoveryService; + let addObservedAddr: jest.Mock; + let confirmObservedAddr: jest.Mock; + let removeObservedAddr: jest.Mock; + let ipNode: MockProxy; + + beforeEach(() => { + peerDiscoveryEmitter = new EventEmitter() as PeerDiscoveryService; + peerDiscoveryEmitter.start = jest.fn<() => Promise>().mockResolvedValue(undefined); + peerDiscoveryEmitter.stop = jest.fn<() => Promise>().mockResolvedValue(undefined); + peerDiscoveryEmitter.getKadValues = jest.fn<() => any[]>().mockReturnValue([]); + peerDiscoveryEmitter.runRandomNodesQuery = jest.fn<() => Promise>().mockResolvedValue(undefined); + peerDiscoveryEmitter.isBootstrapPeer = jest.fn<() => boolean>().mockReturnValue(false); + peerDiscoveryEmitter.getStatus = jest.fn().mockReturnValue('running') as any; + peerDiscoveryEmitter.getEnr = jest.fn().mockReturnValue(undefined) as any; + peerDiscoveryEmitter.bootstrapNodeEnrs = []; + + addObservedAddr = jest.fn(); + confirmObservedAddr = jest.fn(); + removeObservedAddr = jest.fn(); + + ipNode = mock(); + ipNode.services = { + pubsub: { + reportMessageValidationResult: jest.fn(), + subscribe: jest.fn(), + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + getTopics: jest.fn().mockReturnValue([]), + }, + components: { + addressManager: { addObservedAddr, confirmObservedAddr, removeObservedAddr }, + }, + } as any; + ipNode.start = jest.fn<() => Promise>().mockResolvedValue(undefined) as any; + }); + + it('should update libp2p AddressManager when discv5 emits ip:changed', async () => { + const ipService = createTestLibP2PService({ + peerManager: mock(), + node: ipNode, + configOverrides: { queryForIp: true, p2pIp: '10.0.0.1', p2pPort: 40400 }, + peerDiscoveryService: peerDiscoveryEmitter, + }); + + await ipService.start(); + + // Emit first IP change + peerDiscoveryEmitter.emit('ip:changed', '1.2.3.4'); + expect(addObservedAddr).toHaveBeenCalledTimes(1); + expect(confirmObservedAddr).toHaveBeenCalledTimes(1); + expect(removeObservedAddr).not.toHaveBeenCalled(); + + // Emit second IP change - should remove the old address + peerDiscoveryEmitter.emit('ip:changed', '5.6.7.8'); + expect(addObservedAddr).toHaveBeenCalledTimes(2); + expect(confirmObservedAddr).toHaveBeenCalledTimes(2); + expect(removeObservedAddr).toHaveBeenCalledTimes(1); + + await ipService.stop(); + }); + + it('should not wire the bridge when queryForIp is false', async () => { + const ipService = createTestLibP2PService({ + peerManager: mock(), + node: ipNode, + configOverrides: { queryForIp: false, p2pIp: '10.0.0.1', p2pPort: 40400 }, + peerDiscoveryService: peerDiscoveryEmitter, + }); + + await ipService.start(); + + peerDiscoveryEmitter.emit('ip:changed', '1.2.3.4'); + expect(addObservedAddr).not.toHaveBeenCalled(); + + await ipService.stop(); + }); + }); }); /** Mock type for tx objects used in block txs validation tests. */ @@ -1014,6 +1095,8 @@ interface CreateTestLibP2PServiceOptions { attestationPool?: AttestationPool; txPool?: MockProxy; epochCache?: MockProxy; + configOverrides?: Partial; + peerDiscoveryService?: PeerDiscoveryService; } /** @@ -1042,6 +1125,8 @@ class TestLibP2PService extends LibP2PService { /** Exposed epoch cache for test configuration. */ public testEpochCache: MockProxy; + public mockPeerDiscoveryService: PeerDiscoveryService; + constructor( node: PubSubLibp2p, peerManager: PeerManagerInterface, @@ -1050,6 +1135,8 @@ class TestLibP2PService extends LibP2PService { epochCache: MockProxy, telemetry: TelemetryClient, logger: Logger, + configOverrides?: Partial, + peerDiscoveryService?: PeerDiscoveryService, ) { // Create minimal mock dependencies for the base class const mockConfig: P2PConfig = { @@ -1062,9 +1149,10 @@ class TestLibP2PService extends LibP2PService { l1Contracts: { rollupAddress: EthAddress.random(), }, + ...configOverrides, }; - const mockPeerDiscoveryService = mock(); + const resolvedPeerDiscoveryService = peerDiscoveryService ?? mock(); const mockReqResp = mock(); const mockWorldStateSynchronizer = mock(); const mockProofVerifier = mock({ @@ -1074,7 +1162,7 @@ class TestLibP2PService extends LibP2PService { super( mockConfig, node, - mockPeerDiscoveryService, + resolvedPeerDiscoveryService, mockReqResp, peerManager, mempools, @@ -1086,6 +1174,8 @@ class TestLibP2PService extends LibP2PService { logger, ); + this.mockPeerDiscoveryService = resolvedPeerDiscoveryService; + this.testEpochCache = epochCache; this.validateRequestedTxMock = jest.fn(() => Promise.resolve()); this.stubValidator = { @@ -1178,6 +1268,8 @@ function createTestLibP2PService(options: CreateTestLibP2PServiceOptions): TestL attestationPool = new AttestationPool(openTmpStore(true)), txPool = mock(), epochCache = mock(), + configOverrides, + peerDiscoveryService, } = options; epochCache.getL1Constants.mockReturnValue({ @@ -1191,7 +1283,17 @@ function createTestLibP2PService(options: CreateTestLibP2PServiceOptions): TestL const telemetry = getTelemetryClient(); const logger = createLogger('p2p:test'); - return new TestLibP2PService(node, peerManager, mempools, archiver, epochCache, telemetry, logger); + return new TestLibP2PService( + node, + peerManager, + mempools, + archiver, + epochCache, + telemetry, + logger, + configOverrides, + peerDiscoveryService, + ); } /** Creates a TestLibP2PService instance with real attestation pool and mocked tx pool. */ diff --git a/yarn-project/p2p/src/services/peer-manager/peer_manager.ts b/yarn-project/p2p/src/services/peer-manager/peer_manager.ts index db9f89f06063..7bf8a90e3791 100644 --- a/yarn-project/p2p/src/services/peer-manager/peer_manager.ts +++ b/yarn-project/p2p/src/services/peer-manager/peer_manager.ts @@ -522,14 +522,12 @@ export class PeerManager implements PeerManagerInterface { const peersToConnect = this.config.maxPeerCount - healthyConnections.length - protectedPeerCount; const logLevel = this.heartbeatCounter % this.displayPeerCountsPeerHeartbeat === 0 ? 'info' : 'debug'; - const kadValues = this.peerDiscoveryService.getKadValues(); this.logger[logLevel](`Connected to ${healthyConnections.length + this.trustedPeers.size} peers`, { discoveredConnections: healthyConnections.length, protectedConnections: protectedPeerCount, maxPeerCount: this.config.maxPeerCount, cachedPeers: this.cachedPeers.size, - discv5KadPeers: kadValues.length, - discv5KadAddrs: kadValues.slice(0, 5).map(e => `${e.ip ?? 'no-ip'}:${e.udp ?? 'no-udp'}`), + discv5KadPeers: this.peerDiscoveryService.getKadValues().length, ...this.peerScoring.getStats(), });