From 262640cbede76ff07dc7baff37f7613d9dfface8 Mon Sep 17 00:00:00 2001 From: benesjan <13470840+benesjan@users.noreply.github.com> Date: Mon, 22 Dec 2025 16:38:01 +0000 Subject: [PATCH] refactor: more robust tagging index sync as recipient Fixes #17775 In this PR I implement a new log sync algorithm we've come up with in Buenos Aires that should be fully robust against any kind of private log losses. The algorithm is explained in the body of `loadPrivateLogsForSenderRecipientPair` function. That function should be the entrypoint when reviewing this PR (if checking the utility functions first you would not have enough context). Unfortunately this PR introduces a performance regression that is tracked by [this issue](https://linear.app/aztec-labs/issue/F-229/improve-log-sync-performance). I am fairly certain that the regression is caused by an unreasonable number of requests performed by the `loadLogsForRange` function - that will be tackled in a followup PR. In this PR the algorithm is not yet integrated into the system. That is done in a [PR up the stack](https://github.com/AztecProtocol/aztec-packages/pull/19125). The directory structure is not yet final - I wanted to keep this PR contained in one place to not have conflicts with Martin's PR. So please ignore that for now (will move stuff around in a final pass). My plan is as follows: 1. Merge this PR, 2. fix the regression by modifying the `getLogsByTag` API such that it gives me all the info I need (now it doesn't give me block timestamp), 3. once that PR is merged most likely wait for Martin's refactor to be merged and then rebase and polish my integrating new log sync PR, 4. move some files around to make it all cuter. ## Update on regression of time in e2e tests Unfortunately realized that this has caused a regression and log sync is now significantly slower. Did 2 runs of the `e2e_l1_with_wall_time` test and on `next` the results are 337 and 334 seconds but with the new code it's 661 and 658 seconds. Will work on dropping that down before merging that "integrating new log sync" PR. --- .../sender_tagging_data_provider.test.ts | 6 +- .../sender_tagging_data_provider.ts | 6 +- ...ate_logs_for_sender_recipient_pair.test.ts | 242 ++++++++++++ ..._private_logs_for_sender_recipient_pair.ts | 129 +++++++ .../new_recipient_tagging_data_provider.ts | 53 +++ .../utils/find_highest_indexes.test.ts | 116 ++++++ .../utils/find_highest_indexes.ts | 34 ++ .../utils/load_logs_for_range.test.ts | 355 ++++++++++++++++++ .../utils/load_logs_for_range.ts | 79 ++++ .../sync/sync_sender_tagging_indexes.test.ts | 4 +- .../sync/sync_sender_tagging_indexes.ts | 6 +- 11 files changed, 1019 insertions(+), 11 deletions(-) create mode 100644 yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts create mode 100644 yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts create mode 100644 yarn-project/pxe/src/tagging/recipient_sync/new_recipient_tagging_data_provider.ts create mode 100644 yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.test.ts create mode 100644 yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.ts create mode 100644 yarn-project/pxe/src/tagging/recipient_sync/utils/load_logs_for_range.test.ts create mode 100644 yarn-project/pxe/src/tagging/recipient_sync/utils/load_logs_for_range.ts diff --git a/yarn-project/pxe/src/storage/tagging_data_provider/sender_tagging_data_provider.test.ts b/yarn-project/pxe/src/storage/tagging_data_provider/sender_tagging_data_provider.test.ts index 1159b5a8515e..e7b86f9f9f3f 100644 --- a/yarn-project/pxe/src/storage/tagging_data_provider/sender_tagging_data_provider.test.ts +++ b/yarn-project/pxe/src/storage/tagging_data_provider/sender_tagging_data_provider.test.ts @@ -3,7 +3,7 @@ import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { DirectionalAppTaggingSecret, type PreTag } from '@aztec/stdlib/logs'; import { TxHash } from '@aztec/stdlib/tx'; -import { WINDOW_LEN } from '../../tagging/sync/sync_sender_tagging_indexes.js'; +import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/sync/sync_sender_tagging_indexes.js'; import { SenderTaggingDataProvider } from './sender_tagging_data_provider.js'; describe('SenderTaggingDataProvider', () => { @@ -147,7 +147,7 @@ describe('SenderTaggingDataProvider', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); const finalizedIndex = 10; - const indexBeyondWindow = finalizedIndex + WINDOW_LEN + 1; + const indexBeyondWindow = finalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; // First store and finalize an index await taggingDataProvider.storePendingIndexes([{ secret: secret1, index: finalizedIndex }], txHash1); @@ -165,7 +165,7 @@ describe('SenderTaggingDataProvider', () => { const txHash1 = TxHash.random(); const txHash2 = TxHash.random(); const finalizedIndex = 10; - const indexAtBoundary = finalizedIndex + WINDOW_LEN; + const indexAtBoundary = finalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; // First store and finalize an index await taggingDataProvider.storePendingIndexes([{ secret: secret1, index: finalizedIndex }], txHash1); diff --git a/yarn-project/pxe/src/storage/tagging_data_provider/sender_tagging_data_provider.ts b/yarn-project/pxe/src/storage/tagging_data_provider/sender_tagging_data_provider.ts index 180a7b710681..01d61f170c74 100644 --- a/yarn-project/pxe/src/storage/tagging_data_provider/sender_tagging_data_provider.ts +++ b/yarn-project/pxe/src/storage/tagging_data_provider/sender_tagging_data_provider.ts @@ -3,7 +3,7 @@ import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store'; import type { DirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs'; import { TxHash } from '@aztec/stdlib/tx'; -import { WINDOW_LEN as SENDER_TAGGING_INDEXES_SYNC_WINDOW_LEN } from '../../tagging/sync/sync_sender_tagging_indexes.js'; +import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../../tagging/sync/sync_sender_tagging_indexes.js'; /** * Data provider of tagging data used when syncing the sender tagging indexes. The recipient counterpart of this class @@ -68,10 +68,10 @@ export class SenderTaggingDataProvider { // First we check that for any secret the highest used index in tx is not further than window length from // the highest finalized index. const finalizedIndex = (await this.getLastFinalizedIndex(secret)) ?? 0; - if (index > finalizedIndex + SENDER_TAGGING_INDEXES_SYNC_WINDOW_LEN) { + if (index > finalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) { throw new Error( `Highest used index ${index} is further than window length from the highest finalized index ${finalizedIndex}. - Tagging window length ${SENDER_TAGGING_INDEXES_SYNC_WINDOW_LEN} is configured too low. Contact the Aztec team + Tagging window length ${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN} is configured too low. Contact the Aztec team to increase it!`, ); } diff --git a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts new file mode 100644 index 000000000000..ede10640f48f --- /dev/null +++ b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts @@ -0,0 +1,242 @@ +import { MAX_INCLUDE_BY_TIMESTAMP_DURATION } from '@aztec/constants'; +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; +import { L2BlockHash } from '@aztec/stdlib/block'; +import type { AztecNode } from '@aztec/stdlib/interfaces/server'; +import { DirectionalAppTaggingSecret, PrivateLog, TxScopedL2Log } from '@aztec/stdlib/logs'; +import { makeBlockHeader } from '@aztec/stdlib/testing'; +import { TxHash } from '@aztec/stdlib/tx'; + +import { type MockProxy, mock } from 'jest-mock-extended'; + +import { SiloedTag } from '../siloed_tag.js'; +import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../sync/sync_sender_tagging_indexes.js'; +import { Tag } from '../tag.js'; +import { loadPrivateLogsForSenderRecipientPair } from './load_private_logs_for_sender_recipient_pair.js'; +import { NewRecipientTaggingDataProvider } from './new_recipient_tagging_data_provider.js'; + +// In this test suite we don't care about the anchor block behavior as that is sufficiently tested by +// the loadLogsForRange test suite, so we use a high block number to ensure it occurs after all logs. +const NON_INTERFERING_ANCHOR_BLOCK_NUMBER = BlockNumber(100); + +describe('loadPrivateLogsForSenderRecipientPair', () => { + let secret: DirectionalAppTaggingSecret; + let app: AztecAddress; + + let aztecNode: MockProxy; + let taggingDataProvider: NewRecipientTaggingDataProvider; + + const currentTimestamp = BigInt(Math.floor(Date.now() / 1000)); + + async function computeSiloedTagForIndex(index: number) { + const tag = await Tag.compute({ secret, index }); + return SiloedTag.compute(tag, app); + } + + function makeLog(blockHash: Fr, blockNumber: number, tag: Fr) { + return new TxScopedL2Log( + TxHash.random(), + 0, + 0, + BlockNumber(blockNumber), + L2BlockHash.fromField(blockHash), + PrivateLog.random(tag), + ); + } + + beforeAll(async () => { + secret = DirectionalAppTaggingSecret.fromString(Fr.random().toString()); + app = await AztecAddress.random(); + aztecNode = mock(); + }); + + beforeEach(async () => { + aztecNode.getLogsByTags.mockReset(); + aztecNode.getBlockHeaderByHash.mockReset(); + aztecNode.getL2Tips.mockReset(); + aztecNode.getBlockHeader.mockReset(); + taggingDataProvider = new NewRecipientTaggingDataProvider(await openTmpStore('test')); + }); + + it('returns empty array when no logs found', async () => { + aztecNode.getL2Tips.mockResolvedValue({ + finalized: { number: BlockNumber(10) }, + } as any); + + aztecNode.getBlockHeader.mockResolvedValue(makeBlockHeader(0, { timestamp: currentTimestamp })); + + // no logs found for any tag + aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => { + return Promise.resolve(tags.map((_tag: Fr) => [])); + }); + + const logs = await loadPrivateLogsForSenderRecipientPair( + secret, + app, + aztecNode, + taggingDataProvider, + NON_INTERFERING_ANCHOR_BLOCK_NUMBER, + ); + + expect(logs).toHaveLength(0); + expect(await taggingDataProvider.getHighestAgedIndex(secret)).toBeUndefined(); + expect(await taggingDataProvider.getHighestFinalizedIndex(secret)).toBeUndefined(); + }); + + it('loads log and updates highest finalized index but not highest aged index', async () => { + const finalizedBlockNumber = 10; + + const logBlockTimestamp = currentTimestamp - 5000n; // not aged + const logIndex = 5; + const logTag = await computeSiloedTagForIndex(logIndex); + const logBlockHeader = makeBlockHeader(0, { timestamp: logBlockTimestamp }); + + aztecNode.getL2Tips.mockResolvedValue({ + finalized: { number: BlockNumber(finalizedBlockNumber) }, + } as any); + + aztecNode.getBlockHeader.mockResolvedValue(makeBlockHeader(0, { timestamp: currentTimestamp })); + + // The log is finalized + aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => { + return Promise.all( + tags.map(async (t: Fr) => + t.equals(logTag.value) ? [makeLog(await logBlockHeader.hash(), finalizedBlockNumber, logTag.value)] : [], + ), + ); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + if (hash.equals(await logBlockHeader.hash())) { + return logBlockHeader; + } + return undefined; + }); + + const logs = await loadPrivateLogsForSenderRecipientPair( + secret, + app, + aztecNode, + taggingDataProvider, + NON_INTERFERING_ANCHOR_BLOCK_NUMBER, + ); + + expect(logs).toHaveLength(1); + expect(await taggingDataProvider.getHighestFinalizedIndex(secret)).toBe(logIndex); + expect(await taggingDataProvider.getHighestAgedIndex(secret)).toBeUndefined(); + }); + + it('loads log and updates both highest aged and highest finalized indexes', async () => { + const finalizedBlockNumber = 10; + + const logBlockTimestamp = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) - 1000n; // aged + const logIndex = 7; + const logTag = await computeSiloedTagForIndex(logIndex); + const logBlockHeader = makeBlockHeader(0, { timestamp: logBlockTimestamp }); + + aztecNode.getL2Tips.mockResolvedValue({ + finalized: { number: BlockNumber(finalizedBlockNumber) }, + } as any); + + aztecNode.getBlockHeader.mockResolvedValue(makeBlockHeader(0, { timestamp: currentTimestamp })); + + // The log is finalized + aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => { + return Promise.all( + tags.map(async (t: Fr) => + t.equals(logTag.value) ? [makeLog(await logBlockHeader.hash(), finalizedBlockNumber, logTag.value)] : [], + ), + ); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + if (hash.equals(await logBlockHeader.hash())) { + return logBlockHeader; + } + return undefined; + }); + + const logs = await loadPrivateLogsForSenderRecipientPair( + secret, + app, + aztecNode, + taggingDataProvider, + NON_INTERFERING_ANCHOR_BLOCK_NUMBER, + ); + + expect(logs).toHaveLength(1); + expect(await taggingDataProvider.getHighestAgedIndex(secret)).toBe(logIndex); + expect(await taggingDataProvider.getHighestFinalizedIndex(secret)).toBe(logIndex); + }); + + it('logs at boundaries are properly loaded, window and highest indexes advance as expected', async () => { + const finalizedBlockNumber = 10; + + const log1BlockTimestamp = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) - 1000n; // Aged + const log2BlockTimestamp = currentTimestamp - 5000n; // Not aged + const highestAgedIndex = 3; + const highestFinalizedIndex = 5; + const log1Index = highestAgedIndex + 1; // At the beginning of the range + const log2Index = highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; // At the window boundary + const log1Tag = await computeSiloedTagForIndex(log1Index); + const log2Tag = await computeSiloedTagForIndex(log2Index); + const log1BlockHeader = makeBlockHeader(0, { timestamp: log1BlockTimestamp }); + const log2BlockHeader = makeBlockHeader(1, { timestamp: log2BlockTimestamp }); + + // Set existing highest aged index and highest finalized index + await taggingDataProvider.updateHighestAgedIndex(secret, highestAgedIndex); + await taggingDataProvider.updateHighestFinalizedIndex(secret, highestFinalizedIndex); + + aztecNode.getL2Tips.mockResolvedValue({ + finalized: { number: BlockNumber(finalizedBlockNumber) }, + } as any); + + aztecNode.getBlockHeader.mockResolvedValue(makeBlockHeader(0, { timestamp: currentTimestamp })); + + // We record the number of queried tags to be able to verify that the window was moved forward correctly. + let numQueriedTags = 0; + + aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => { + numQueriedTags += tags.length; + return Promise.all( + tags.map(async (t: Fr) => { + if (t.equals(log1Tag.value)) { + return [makeLog(await log1BlockHeader.hash(), finalizedBlockNumber, log1Tag.value)]; + } else if (t.equals(log2Tag.value)) { + return [makeLog(await log2BlockHeader.hash(), finalizedBlockNumber, log2Tag.value)]; + } + return []; + }), + ); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + if (hash.equals(await log1BlockHeader.hash())) { + return log1BlockHeader; + } else if (hash.equals(await log2BlockHeader.hash())) { + return log2BlockHeader; + } + return undefined; + }); + + const logs = await loadPrivateLogsForSenderRecipientPair( + secret, + app, + aztecNode, + taggingDataProvider, + NON_INTERFERING_ANCHOR_BLOCK_NUMBER, + ); + + // Verify that both logs at the boundaries of the range were found and processed + expect(logs).toHaveLength(2); + expect(await taggingDataProvider.getHighestFinalizedIndex(secret)).toBe(log2Index); + expect(await taggingDataProvider.getHighestAgedIndex(secret)).toBe(log1Index); + + // Verify that the window was moved forward correctly + // Total range queried: from (highestAgedIndex + 1) to (log2Index + WINDOW_LEN + 1) exclusive + const expectedNumQueriedTags = log2Index + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN - highestAgedIndex; + expect(numQueriedTags).toBe(expectedNumQueriedTags); + }); +}); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts new file mode 100644 index 000000000000..0433cf083787 --- /dev/null +++ b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts @@ -0,0 +1,129 @@ +import type { BlockNumber } from '@aztec/foundation/branded-types'; +import type { AztecAddress } from '@aztec/stdlib/aztec-address'; +import type { AztecNode } from '@aztec/stdlib/interfaces/client'; +import type { DirectionalAppTaggingSecret, TxScopedL2Log } from '@aztec/stdlib/logs'; + +import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../sync/sync_sender_tagging_indexes.js'; +import type { NewRecipientTaggingDataProvider } from './new_recipient_tagging_data_provider.js'; +import { findHighestIndexes } from './utils/find_highest_indexes.js'; +import { loadLogsForRange } from './utils/load_logs_for_range.js'; + +/** + * Loads private logs for `app` and sender-recipient pair defined by `secret` and updates the highest aged and + * finalized indexes in the db. At most load logs from blocks up to and including `anchorBlockNumber`. + * + * @dev This function can be safely executed "in parallel" for other sender-recipient pairs because the data in + * in the tagging data provider is indexed by the secret and hence completely disjoint. + */ +export async function loadPrivateLogsForSenderRecipientPair( + secret: DirectionalAppTaggingSecret, + app: AztecAddress, + aztecNode: AztecNode, + taggingDataProvider: NewRecipientTaggingDataProvider, + anchorBlockNumber: BlockNumber, +): Promise { + // # Explanation of how the algorithm works + // When we perform the sync we will look at logs that correspond to the tagging index range + // (highestAgedIndex, highestFinalizedIndex + WINDOW_LEN] + // + // highestAgedIndex is the highest index that was used in a tx that is included in a block at least + // `MAX_INCLUDE_BY_TIMESTAMP_DURATION` seconds ago. + // highestFinalizedIndex is the highest index that was used in a tx that is included in a finalized block. + // + // "(" denotes an open end of the range - the index is not included in the range. + // "]" denotes a closed end of the range - the index is included in the range. + // + // ## Explanation of highestAgedIndex + // + // highestAgedIndex is chosen such that for all tagging indexes `i <= highestAgedIndex` we know that no new logs can + // ever appear. + // + // This relies on the "maximum inclusion timestamp" rule enforced by the kernel and rollup circuits: + // - a transaction's maximum inclusion timestamp is at most `MAX_INCLUDE_BY_TIMESTAMP_DURATION` seconds after + // the timestamp of its anchor block; and + // - a rollup only includes transactions whose inclusion timestamp is >= the L2 block's timestamp. + // + // Suppose some device used index `I` in a transaction anchored to block `B_N` at time `N`, and that block is now at + // least `MAX_INCLUDE_BY_TIMESTAMP_DURATION` seconds in the past. Then there is no possibility of any *other* device + // trying to use an index <= `I` while anchoring to a *newer* block than `B_N` because if we were anchoring to + // a newer block than `B_N` then we would already have seen the log with index `I` and hence the device would have + // chosen a larger index. + // If that *other* device would anchor to a block older than `B_N` then that tx could never be included in a block + // because it would already have been expired. + // + // Therefore, once we see that index `I` has been used in a block that is at least `MAX_INCLUDE_BY_TIMESTAMP_DURATION` + // seconds old, we can safely stop syncing logs for all indexes <= `I` and set highestAgedIndex = `I`. + // + // ## Explanation of the upper bound `highestFinalizedIndex + WINDOW_LEN` + // + // When a sender chooses a tagging index, they will select an index that is at most `WINDOW_LEN` greater than + // the highest finalized index. If that index was already used, they will throw an error. For this reason we + // don't have to look further than `highestFinalizedIndex + WINDOW_LEN`. + + let finalizedBlockNumber: number, currentTimestamp: bigint; + { + const [l2Tips, latestBlockHeader] = await Promise.all([aztecNode.getL2Tips(), aztecNode.getBlockHeader('latest')]); + + if (!latestBlockHeader) { + throw new Error('Node failed to return latest block header when syncing logs'); + } + + [finalizedBlockNumber, currentTimestamp] = [l2Tips.finalized.number, latestBlockHeader.globalVariables.timestamp]; + } + + let start: number, end: number; + { + const currentHighestAgedIndex = await taggingDataProvider.getHighestAgedIndex(secret); + const currentHighestFinalizedIndex = await taggingDataProvider.getHighestFinalizedIndex(secret); + + // We don't want to include the highest aged index so we start from `currentHighestAgedIndex + 1` (or 0 if not set) + start = currentHighestAgedIndex === undefined ? 0 : currentHighestAgedIndex + 1; + + // The highest index a sender can choose is "highest finalized index + window length" but given that + // `loadLogsForRange` expects an exclusive `end` we add 1. + end = (currentHighestFinalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; + } + + const logs: TxScopedL2Log[] = []; + + while (true) { + // Get private logs with their block timestamps and corresponding tagging indexes + const logsWithTimestampsAndIndexes = await loadLogsForRange(secret, app, aztecNode, start, end, anchorBlockNumber); + + if (logsWithTimestampsAndIndexes.length === 0) { + break; + } + + logs.push(...logsWithTimestampsAndIndexes.map(({ log }) => log)); + + const { highestAgedIndex, highestFinalizedIndex } = findHighestIndexes( + logsWithTimestampsAndIndexes, + currentTimestamp, + finalizedBlockNumber, + ); + + // Store updates in data provider and update local variables + if (highestAgedIndex !== undefined) { + await taggingDataProvider.updateHighestAgedIndex(secret, highestAgedIndex); + } + + if (highestFinalizedIndex === undefined) { + // We have not found a new highest finalized index, so there is no need to move the window forward. + break; + } + + if (highestAgedIndex !== undefined && highestAgedIndex > highestFinalizedIndex) { + // This is just a sanity check as this should never happen. + throw new Error('Highest aged index lower than highest finalized index invariant violated'); + } + + await taggingDataProvider.updateHighestFinalizedIndex(secret, highestFinalizedIndex); + + // For the next iteration we want to look only at indexes for which we have not attempted to load logs yet while + // ensuring that we do not look further than WINDOW_LEN ahead of the highest finalized index. + start = end; + end = highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; // `end` is exclusive so we add 1. + } + + return logs; +} diff --git a/yarn-project/pxe/src/tagging/recipient_sync/new_recipient_tagging_data_provider.ts b/yarn-project/pxe/src/tagging/recipient_sync/new_recipient_tagging_data_provider.ts new file mode 100644 index 000000000000..fb14342e0368 --- /dev/null +++ b/yarn-project/pxe/src/tagging/recipient_sync/new_recipient_tagging_data_provider.ts @@ -0,0 +1,53 @@ +import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store'; +import type { DirectionalAppTaggingSecret } from '@aztec/stdlib/logs'; + +/** + * Data provider of tagging data used when syncing the logs as a recipient. The sender counterpart of this class + * is called SenderTaggingDataProvider. We have the providers separate for the sender and recipient because + * the algorithms are completely disjoint and there is not data reuse between the two. + * + * @dev Chain reorgs do not need to be handled here because both the finalized and aged indexes refer to finalized + * blocks, which by definition cannot be affected by reorgs. + * + * TODO(benesjan): Rename as to RecipientTaggingDataProvider and relocate once the old tagging sync is purged. + */ +export class NewRecipientTaggingDataProvider { + #store: AztecAsyncKVStore; + + #highestAgedIndex: AztecAsyncMap; + #highestFinalizedIndex: AztecAsyncMap; + + constructor(store: AztecAsyncKVStore) { + this.#store = store; + + this.#highestAgedIndex = this.#store.openMap('highest_aged_index'); + this.#highestFinalizedIndex = this.#store.openMap('highest_finalized_index'); + } + + getHighestAgedIndex(secret: DirectionalAppTaggingSecret): Promise { + return this.#highestAgedIndex.getAsync(secret.toString()); + } + + async updateHighestAgedIndex(secret: DirectionalAppTaggingSecret, index: number): Promise { + const currentIndex = await this.#highestAgedIndex.getAsync(secret.toString()); + if (currentIndex !== undefined && index <= currentIndex) { + // Log sync should never set a lower highest aged index. + throw new Error(`New highest aged index (${index}) must be higher than the current one (${currentIndex})`); + } + await this.#highestAgedIndex.set(secret.toString(), index); + } + + getHighestFinalizedIndex(secret: DirectionalAppTaggingSecret): Promise { + return this.#highestFinalizedIndex.getAsync(secret.toString()); + } + + async updateHighestFinalizedIndex(secret: DirectionalAppTaggingSecret, index: number): Promise { + const currentIndex = await this.#highestFinalizedIndex.getAsync(secret.toString()); + if (currentIndex !== undefined && index < currentIndex) { + // Log sync should never set a lower highest finalized index but it can happen that it would try to set the same + // one because we are loading logs from highest aged index + 1 and not from the highest finalized index. + throw new Error(`New highest finalized index (${index}) must be higher than the current one (${currentIndex})`); + } + await this.#highestFinalizedIndex.set(secret.toString(), index); + } +} diff --git a/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.test.ts new file mode 100644 index 000000000000..47761743522d --- /dev/null +++ b/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.test.ts @@ -0,0 +1,116 @@ +import { MAX_INCLUDE_BY_TIMESTAMP_DURATION } from '@aztec/constants'; +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { L2BlockHash } from '@aztec/stdlib/block'; +import { PrivateLog, TxScopedL2Log } from '@aztec/stdlib/logs'; +import { TxHash } from '@aztec/stdlib/tx'; + +import { findHighestIndexes } from './find_highest_indexes.js'; + +describe('findHighestIndexes', () => { + const currentTimestamp = BigInt(Math.floor(Date.now() / 1000)); + + function makeLog(blockNumber: number): TxScopedL2Log { + return new TxScopedL2Log( + TxHash.random(), + 0, + 0, + BlockNumber(blockNumber), + L2BlockHash.random(), + PrivateLog.random(), + ); + } + + it('returns undefined for highestAgedIndex when no logs are at least 24 hours old', () => { + const finalizedBlockNumber = 10; + const blockTimestamp = currentTimestamp - 1n; // not aged + const log = makeLog(5); + + const result = findHighestIndexes( + [{ log, blockTimestamp, taggingIndex: 3 }], + currentTimestamp, + finalizedBlockNumber, + ); + + expect(result.highestAgedIndex).toBeUndefined(); + expect(result.highestFinalizedIndex).toBe(3); + }); + + it('returns undefined for highestFinalizedIndex when no logs are in finalized blocks', () => { + const finalizedBlockNumber = 5; + const blockTimestamp = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION); + const log = makeLog(10); // block 10 > finalizedBlockNumber 5 + + const result = findHighestIndexes( + [{ log, blockTimestamp, taggingIndex: 3 }], + currentTimestamp, + finalizedBlockNumber, + ); + + expect(result.highestAgedIndex).toBe(3); + expect(result.highestFinalizedIndex).toBeUndefined(); + }); + + it('selects the highest index from multiple aged logs', () => { + const finalizedBlockNumber = 10; + const blockTimestamp1 = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) - 1000n; // aged + const blockTimestamp2 = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) - 500n; // aged + const log1 = makeLog(5); + const log2 = makeLog(6); + + const result = findHighestIndexes( + [ + { log: log1, blockTimestamp: blockTimestamp1, taggingIndex: 2 }, + { log: log2, blockTimestamp: blockTimestamp2, taggingIndex: 5 }, + { log: log1, blockTimestamp: blockTimestamp1, taggingIndex: 3 }, + ], + currentTimestamp, + finalizedBlockNumber, + ); + + expect(result.highestAgedIndex).toBe(5); + expect(result.highestFinalizedIndex).toBe(5); + }); + + it('selects the highest index from multiple finalized logs', () => { + const finalizedBlockNumber = 10; + const blockTimestamp = currentTimestamp - 500n; // 500 seconds ago - not aged + const log1 = makeLog(5); + const log2 = makeLog(8); + const log3 = makeLog(10); // At finalized block number + + const result = findHighestIndexes( + [ + { log: log1, blockTimestamp, taggingIndex: 2 }, + { log: log2, blockTimestamp, taggingIndex: 7 }, + { log: log3, blockTimestamp, taggingIndex: 1 }, + ], + currentTimestamp, + finalizedBlockNumber, + ); + + expect(result.highestAgedIndex).toBeUndefined(); + expect(result.highestFinalizedIndex).toBe(7); + }); + + it('handles mixed scenarios with multiple logs of different types', () => { + const finalizedBlockNumber = 10; + const veryOldTimestamp = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) * 2n - 1000n; // Aged + const oldTimestamp = currentTimestamp - BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) - 1000n; // Aged + const recentTimestamp = currentTimestamp - 5000n; // Not aged + + const logs = [ + { log: makeLog(5), blockTimestamp: veryOldTimestamp, taggingIndex: 1 }, // Aged, finalized + { log: makeLog(8), blockTimestamp: oldTimestamp, taggingIndex: 5 }, // Aged, finalized + { log: makeLog(10), blockTimestamp: recentTimestamp, taggingIndex: 8 }, // Not aged, finalized + { log: makeLog(15), blockTimestamp: oldTimestamp, taggingIndex: 12 }, // Aged, not finalized + { log: makeLog(20), blockTimestamp: recentTimestamp, taggingIndex: 15 }, // Not aged, not finalized + ]; + + const result = findHighestIndexes(logs, currentTimestamp, finalizedBlockNumber); + + // Highest aged: index 12 (from block 15, which is aged but not finalized) + expect(result.highestAgedIndex).toBe(12); + // Highest finalized: index 8 (from block 10, which is finalized but not aged) + expect(result.highestFinalizedIndex).toBe(8); + }); +}); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.ts b/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.ts new file mode 100644 index 000000000000..4bb8ee2f317c --- /dev/null +++ b/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.ts @@ -0,0 +1,34 @@ +import { MAX_INCLUDE_BY_TIMESTAMP_DURATION } from '@aztec/constants'; +import type { TxScopedL2Log } from '@aztec/stdlib/logs'; + +/** + * Finds the highest aged and the highest finalized tagging indexes. + */ +export function findHighestIndexes( + logsWithTimestampsAndIndexes: Array<{ log: TxScopedL2Log; blockTimestamp: bigint; taggingIndex: number }>, + currentTimestamp: bigint, + finalizedBlockNumber: number, +): { highestAgedIndex: number | undefined; highestFinalizedIndex: number | undefined } { + let highestAgedIndex = undefined; + let highestFinalizedIndex = undefined; + + for (const { log, blockTimestamp, taggingIndex } of logsWithTimestampsAndIndexes) { + const ageInSeconds = currentTimestamp - blockTimestamp; + + if ( + ageInSeconds >= BigInt(MAX_INCLUDE_BY_TIMESTAMP_DURATION) && + (highestAgedIndex === undefined || taggingIndex > highestAgedIndex) + ) { + highestAgedIndex = taggingIndex; + } + + if ( + log.blockNumber <= finalizedBlockNumber && + (highestFinalizedIndex === undefined || taggingIndex > highestFinalizedIndex) + ) { + highestFinalizedIndex = taggingIndex; + } + } + + return { highestAgedIndex, highestFinalizedIndex }; +} diff --git a/yarn-project/pxe/src/tagging/recipient_sync/utils/load_logs_for_range.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/utils/load_logs_for_range.test.ts new file mode 100644 index 000000000000..6881ea403f20 --- /dev/null +++ b/yarn-project/pxe/src/tagging/recipient_sync/utils/load_logs_for_range.test.ts @@ -0,0 +1,355 @@ +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; +import { L2BlockHash } from '@aztec/stdlib/block'; +import type { AztecNode } from '@aztec/stdlib/interfaces/server'; +import { DirectionalAppTaggingSecret, PrivateLog, PublicLog, TxScopedL2Log } from '@aztec/stdlib/logs'; +import { makeBlockHeader } from '@aztec/stdlib/testing'; +import { TxHash } from '@aztec/stdlib/tx'; + +import { type MockProxy, mock } from 'jest-mock-extended'; + +import { SiloedTag } from '../../siloed_tag.js'; +import { Tag } from '../../tag.js'; +import { loadLogsForRange } from './load_logs_for_range.js'; + +// In tests where the anchor block behavior is not under examination, we use a high block number to ensure it occurs +// after all logs. +const NON_INTERFERING_ANCHOR_BLOCK_NUMBER = BlockNumber(100); + +describe('loadLogsForRange', () => { + // App contract address and secret to be used on the input of the loadLogsForRange function. + let secret: DirectionalAppTaggingSecret; + let app: AztecAddress; + + let aztecNode: MockProxy; + + async function computeSiloedTagForIndex(index: number) { + const tag = await Tag.compute({ secret, index }); + return SiloedTag.compute(tag, app); + } + + function makeLog(txHash: TxHash, blockHash: Fr, blockNumber: number, tag: SiloedTag) { + return new TxScopedL2Log( + txHash, + 0, + 0, + BlockNumber(blockNumber), + L2BlockHash.fromField(blockHash), + PrivateLog.random(tag.value), + ); + } + + beforeAll(async () => { + secret = DirectionalAppTaggingSecret.fromString(Fr.random().toString()); + app = await AztecAddress.random(); + aztecNode = mock(); + }); + + beforeEach(() => { + aztecNode.getLogsByTags.mockReset(); + aztecNode.getBlockHeaderByHash.mockReset(); + }); + + it('returns empty array when no logs found for the given window', async () => { + aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => { + // No log found for any tag + return Promise.resolve(tags.map((_tag: Fr) => [])); + }); + + expect(await loadLogsForRange(secret, app, aztecNode, 0, 10, NON_INTERFERING_ANCHOR_BLOCK_NUMBER)).toHaveLength(0); + }); + + it('only returns private logs', async () => { + const txHash = TxHash.random(); + const blockNumber = 5; + const index = 3; + const timestamp = 1000n; + const tag = await computeSiloedTagForIndex(index); + const blockHeader = makeBlockHeader(0, { timestamp }); + + aztecNode.getLogsByTags.mockImplementation(async (tags: Fr[]) => { + const blockHash = await blockHeader.hash(); + const privateLog = makeLog(txHash, blockHash, blockNumber, tag); + const publicLog = new TxScopedL2Log( + TxHash.random(), + 0, + 0, + BlockNumber(blockNumber), + L2BlockHash.fromField(blockHash), + await PublicLog.random(), + ); + return tags.map((t: Fr) => (t.equals(tag.value) ? [privateLog, publicLog] : [])); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + if (hash.equals(await blockHeader.hash())) { + return blockHeader; + } + return undefined; + }); + + const result = await loadLogsForRange(secret, app, aztecNode, 0, 10, NON_INTERFERING_ANCHOR_BLOCK_NUMBER); + + expect(result).toHaveLength(1); + expect(result[0].log.txHash.equals(txHash)).toBe(true); + expect(result[0].log.isFromPublic).toBe(false); + }); + + it('handles multiple logs at different indexes', async () => { + const txHash1 = TxHash.random(); + const txHash2 = TxHash.random(); + const blockNumber1 = 5; + const blockNumber2 = 6; + const index1 = 2; + const index2 = 7; + const timestamp1 = 1000n; + const timestamp2 = 2000n; + const tag1 = await computeSiloedTagForIndex(index1); + const tag2 = await computeSiloedTagForIndex(index2); + const blockHeader1 = makeBlockHeader(0, { timestamp: timestamp1 }); + const blockHeader2 = makeBlockHeader(1, { timestamp: timestamp2 }); + + aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => { + return Promise.all( + tags.map(async (t: Fr) => { + if (t.equals(tag1.value)) { + return [makeLog(txHash1, await blockHeader1.hash(), blockNumber1, tag1)]; + } else if (t.equals(tag2.value)) { + return [makeLog(txHash2, await blockHeader2.hash(), blockNumber2, tag2)]; + } + return []; + }), + ); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + if (hash.equals(await blockHeader1.hash())) { + return blockHeader1; + } else if (hash.equals(await blockHeader2.hash())) { + return blockHeader2; + } + return undefined; + }); + + const result = await loadLogsForRange(secret, app, aztecNode, 0, 10, NON_INTERFERING_ANCHOR_BLOCK_NUMBER); + + expect(result).toHaveLength(2); + const resultByIndex = result.sort((a, b) => a.taggingIndex - b.taggingIndex); + expect(resultByIndex[0].taggingIndex).toBe(index1); + expect(resultByIndex[0].blockTimestamp).toBe(timestamp1); + expect(resultByIndex[0].log.txHash.equals(txHash1)).toBe(true); + expect(resultByIndex[1].taggingIndex).toBe(index2); + expect(resultByIndex[1].blockTimestamp).toBe(timestamp2); + expect(resultByIndex[1].log.txHash.equals(txHash2)).toBe(true); + }); + + it('handles multiple logs at the same index', async () => { + const txHash1 = TxHash.random(); + const txHash2 = TxHash.random(); + const blockNumber1 = 5; + const blockNumber2 = 6; + const index = 4; + const timestamp1 = 1000n; + const timestamp2 = 2000n; + const tag = await computeSiloedTagForIndex(index); + const blockHeader1 = makeBlockHeader(0, { timestamp: timestamp1 }); + const blockHeader2 = makeBlockHeader(1, { timestamp: timestamp2 }); + + aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => { + return Promise.all( + tags.map(async (t: Fr) => + t.equals(tag.value) + ? [ + makeLog(txHash1, await blockHeader1.hash(), blockNumber1, tag), + makeLog(txHash2, await blockHeader2.hash(), blockNumber2, tag), + ] + : [], + ), + ); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + if (hash.equals(await blockHeader1.hash())) { + return blockHeader1; + } else if (hash.equals(await blockHeader2.hash())) { + return blockHeader2; + } + return undefined; + }); + + const result = await loadLogsForRange(secret, app, aztecNode, 0, 10, NON_INTERFERING_ANCHOR_BLOCK_NUMBER); + + expect(result).toHaveLength(2); + expect(result[0].taggingIndex).toBe(index); + expect(result[1].taggingIndex).toBe(index); + const txHashes = result.map(r => r.log.txHash.toString()); + expect(txHashes).toContain(txHash1.toString()); + expect(txHashes).toContain(txHash2.toString()); + }); + + it('handles multiple logs in the same block', async () => { + const txHash1 = TxHash.random(); + const txHash2 = TxHash.random(); + const blockNumber = 5; + const index1 = 2; + const index2 = 3; + const timestamp = 1000n; + const tag1 = await computeSiloedTagForIndex(index1); + const tag2 = await computeSiloedTagForIndex(index2); + const blockHeader = makeBlockHeader(0, { timestamp }); + + aztecNode.getLogsByTags.mockImplementation(async (tags: Fr[]) => { + const blockHash = await blockHeader.hash(); + return tags.map((t: Fr) => { + if (t.equals(tag1.value)) { + return [makeLog(txHash1, blockHash, blockNumber, tag1)]; + } else if (t.equals(tag2.value)) { + return [makeLog(txHash2, blockHash, blockNumber, tag2)]; + } + return []; + }); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + if (hash.equals(await blockHeader.hash())) { + return blockHeader; + } + return undefined; + }); + + const result = await loadLogsForRange(secret, app, aztecNode, 0, 10, NON_INTERFERING_ANCHOR_BLOCK_NUMBER); + + expect(result).toHaveLength(2); + // Should only fetch block header once for the same block + expect(aztecNode.getBlockHeaderByHash).toHaveBeenCalledTimes(1); + const resultByIndex = result.sort((a, b) => a.taggingIndex - b.taggingIndex); + expect(resultByIndex[0].taggingIndex).toBe(index1); + expect(resultByIndex[0].blockTimestamp).toBe(timestamp); + expect(resultByIndex[1].taggingIndex).toBe(index2); + expect(resultByIndex[1].blockTimestamp).toBe(timestamp); + }); + + it('respects start (inclusive) and end (exclusive) boundaries', async () => { + const start = 5; + const end = 10; + + const txHashAtStart = TxHash.random(); + const txHashAtEnd = TxHash.random(); + const timestamp = 1000n; + const tagAtStart = await computeSiloedTagForIndex(start); + const tagAtEnd = await computeSiloedTagForIndex(end); + const blockHeader = makeBlockHeader(0, { timestamp }); + + aztecNode.getLogsByTags.mockImplementation(async (tags: Fr[]) => { + const blockHash = await blockHeader.hash(); + return tags.map((t: Fr) => { + if (t.equals(tagAtStart.value)) { + return [makeLog(txHashAtStart, blockHash, 5, tagAtStart)]; + } else if (t.equals(tagAtEnd.value)) { + return [makeLog(txHashAtEnd, blockHash, 6, tagAtEnd)]; + } + return []; + }); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + if (hash.equals(await blockHeader.hash())) { + return blockHeader; + } + return undefined; + }); + + const result = await loadLogsForRange(secret, app, aztecNode, start, end, NON_INTERFERING_ANCHOR_BLOCK_NUMBER); + + // Should only include log at start (inclusive), not at end (exclusive) + expect(result).toHaveLength(1); + expect(result[0].taggingIndex).toBe(start); + expect(result[0].log.txHash.equals(txHashAtStart)).toBe(true); + }); + + it('ignores logs from reorged blocks', async () => { + const txHashReorged = TxHash.random(); + const txHashValid = TxHash.random(); + const blockHashReorged = L2BlockHash.random(); + const blockNumberReorged = 5; + const blockNumberValid = 6; + const index1 = 3; + const index2 = 4; + const timestamp = 2000n; + const tag1 = await computeSiloedTagForIndex(index1); + const tag2 = await computeSiloedTagForIndex(index2); + const blockHeaderValid = makeBlockHeader(1, { timestamp }); + + aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => { + return Promise.all( + tags.map(async (t: Fr) => { + if (t.equals(tag1.value)) { + return [makeLog(txHashReorged, Fr.fromBuffer(blockHashReorged.toBuffer()), blockNumberReorged, tag1)]; + } else if (t.equals(tag2.value)) { + return [makeLog(txHashValid, await blockHeaderValid.hash(), blockNumberValid, tag2)]; + } + return []; + }), + ); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + // Block header for reorged block is not found (returns undefined) + if (hash.equals(Fr.fromBuffer(blockHashReorged.toBuffer()))) { + return undefined; + } else if (hash.equals(await blockHeaderValid.hash())) { + return blockHeaderValid; + } + return undefined; + }); + + const result = await loadLogsForRange(secret, app, aztecNode, 0, 10, NON_INTERFERING_ANCHOR_BLOCK_NUMBER); + + // Should only include the log from the valid block, ignoring the log from the reorged block + expect(result).toHaveLength(1); + expect(result[0].log.txHash.equals(txHashValid)).toBe(true); + expect(result[0].taggingIndex).toBe(index2); + }); + + it('filters out logs from blocks after anchor block', async () => { + const anchorBlockNumber = 10; + + const index = 3; + const timestamp = 1000n; + const tag = await computeSiloedTagForIndex(index); + const blockHeaderBefore = makeBlockHeader(0, { timestamp }); + const blockHeaderAtAnchor = makeBlockHeader(1, { timestamp }); + const blockHeaderAfter = makeBlockHeader(2, { timestamp }); + + aztecNode.getLogsByTags.mockImplementation((tags: Fr[]) => { + return Promise.all( + tags.map(async (t: Fr) => + t.equals(tag.value) + ? [ + makeLog(TxHash.random(), await blockHeaderBefore.hash(), anchorBlockNumber - 1, tag), + makeLog(TxHash.random(), await blockHeaderAtAnchor.hash(), anchorBlockNumber, tag), + makeLog(TxHash.random(), await blockHeaderAfter.hash(), anchorBlockNumber + 1, tag), + ] + : [], + ), + ); + }); + + aztecNode.getBlockHeaderByHash.mockImplementation(async (hash: Fr) => { + if (hash.equals(await blockHeaderBefore.hash())) { + return blockHeaderBefore; + } else if (hash.equals(await blockHeaderAtAnchor.hash())) { + return blockHeaderAtAnchor; + } else if (hash.equals(await blockHeaderAfter.hash())) { + return blockHeaderAfter; + } + return undefined; + }); + + const result = await loadLogsForRange(secret, app, aztecNode, 0, 10, BlockNumber(anchorBlockNumber)); + + // Should only include logs from blocks at or before the anchor block number + expect(result).toHaveLength(2); + }); +}); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/utils/load_logs_for_range.ts b/yarn-project/pxe/src/tagging/recipient_sync/utils/load_logs_for_range.ts new file mode 100644 index 000000000000..74c5d703e625 --- /dev/null +++ b/yarn-project/pxe/src/tagging/recipient_sync/utils/load_logs_for_range.ts @@ -0,0 +1,79 @@ +import type { BlockNumber } from '@aztec/foundation/branded-types'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import type { AztecAddress } from '@aztec/stdlib/aztec-address'; +import type { AztecNode } from '@aztec/stdlib/interfaces/client'; +import type { DirectionalAppTaggingSecret, PreTag, TxScopedL2Log } from '@aztec/stdlib/logs'; + +import { SiloedTag } from '../../siloed_tag.js'; +import { Tag } from '../../tag.js'; + +/** + * Gets private logs with their corresponding block timestamps and tagging indexes for the given index range, `app` and + * `secret`. At most load logs from blocks up to and including `anchorBlockNumber`. `start` is inclusive and `end` is + * exclusive. + * + * TODO: Optimize Aztec Node API such that this function performs only a single call. + */ +export async function loadLogsForRange( + secret: DirectionalAppTaggingSecret, + app: AztecAddress, + aztecNode: AztecNode, + start: number, + end: number, + anchorBlockNumber: BlockNumber, +): Promise> { + // Derive tags for the window + const preTags: PreTag[] = Array(end - start) + .fill(0) + .map((_, i) => ({ secret, index: start + i })); + const siloedTags = await Promise.all(preTags.map(preTag => Tag.compute(preTag))).then(tags => + Promise.all(tags.map(tag => SiloedTag.compute(tag, app))), + ); + + // Get logs for these tags + const tagsAsFr = siloedTags.map(tag => tag.value); + const allLogs = await aztecNode.getLogsByTags(tagsAsFr); + + // Collect all private logs with their corresponding tagging indexes + const privateLogsWithIndexes: Array<{ log: TxScopedL2Log; taggingIndex: number }> = []; + for (let i = 0; i < allLogs.length; i++) { + const logs = allLogs[i]; + const taggingIndex = preTags[i].index; + for (const log of logs) { + if (!log.isFromPublic && log.blockNumber <= anchorBlockNumber) { + privateLogsWithIndexes.push({ log, taggingIndex }); + } + } + } + + // If no private logs were obtained, return an empty array + if (privateLogsWithIndexes.length === 0) { + return []; + } + + // Get unique block hashes + const uniqueBlockHashes = Array.from(new Set(privateLogsWithIndexes.map(({ log }) => log.blockHash.toBigInt()))).map( + hash => new Fr(hash), + ); + + // Get block headers for all unique block hashes + const blockHeaders = await Promise.all(uniqueBlockHashes.map(blockHash => aztecNode.getBlockHeaderByHash(blockHash))); + + // Return logs with their corresponding block timestamps and tagging indexes + const result: Array<{ log: TxScopedL2Log; blockTimestamp: bigint; taggingIndex: number }> = []; + for (const { log, taggingIndex } of privateLogsWithIndexes) { + // TODO: Unify types of blockHash on log and on block header so we don't need to do this ugly conversion. + const logBlockHash = log.blockHash.toBigInt(); + const logBlockHeader = blockHeaders[uniqueBlockHashes.findIndex(hash => hash.toBigInt() === logBlockHash)]; + if (!logBlockHeader) { + // If the block header for a log cannot be found, it indicates a reorg occurred between `getLogsByTags` and + // `getBlockHeaderByHash`. It is correct and safe to ignore such logs because they have been pruned from + // the chain. PXE block synchronizer will reset any state following the reorg block. + continue; + } + + result.push({ log, blockTimestamp: logBlockHeader.globalVariables.timestamp, taggingIndex }); + } + + return result; +} diff --git a/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.test.ts b/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.test.ts index 6f207bd6b7da..3a93bae46312 100644 --- a/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.test.ts @@ -11,7 +11,7 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js'; import { DirectionalAppTaggingSecret, SiloedTag, Tag } from '../index.js'; -import { WINDOW_LEN, syncSenderTaggingIndexes } from './sync_sender_tagging_indexes.js'; +import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, syncSenderTaggingIndexes } from './sync_sender_tagging_indexes.js'; describe('syncSenderTaggingIndexes', () => { // Contract address and secret to be used on the input of the syncSenderTaggingIndexes function. @@ -129,7 +129,7 @@ describe('syncSenderTaggingIndexes', () => { // Move finalized block into the future const newFinalizedBlockNumber = finalizedBlockNumberStep1 + 5; const newHighestFinalizedIndex = finalizedIndexStep1 + 4; - const newHighestUsedIndex = newHighestFinalizedIndex + WINDOW_LEN; + const newHighestUsedIndex = newHighestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; // Create tx hashes for new logs const newHighestFinalizedTxHash = TxHash.random(); diff --git a/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.ts b/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.ts index 9579f65caa73..758f81b03dcb 100644 --- a/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.ts +++ b/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.ts @@ -21,7 +21,7 @@ import { loadAndStoreNewTaggingIndexes } from './utils/load_and_store_new_taggin // - This value is below MAX_RPC_LEN (100) which is the limit for array parameters in the JSON RPC schema for // `getLogsByTags`. Any test that would perform sync over JSON RPC (not by having access to the Aztec node instance // directly) would error out if that maximum was hit (docs_examples.test.ts is an example of this). -export const WINDOW_LEN = 95; +export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 95; /** * Syncs tagging indexes. This function needs to be called whenever a private log is being sent. @@ -64,7 +64,7 @@ export async function syncSenderTaggingIndexes( const finalizedIndex = await taggingDataProvider.getLastFinalizedIndex(secret); let start = finalizedIndex === undefined ? 0 : finalizedIndex + 1; - let end = start + WINDOW_LEN; + let end = start + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; let previousFinalizedIndex = finalizedIndex; let newFinalizedIndex = undefined; @@ -101,7 +101,7 @@ export async function syncSenderTaggingIndexes( const previousEnd = end; // Add 1 because `end` is exclusive and the known finalized index is not included in the window. - end = newFinalizedIndex! + WINDOW_LEN + 1; + end = newFinalizedIndex! + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; start = previousEnd; previousFinalizedIndex = newFinalizedIndex; } else {