feat: integrating new log sync

yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts

@@ -11,15 +11,13 @@ import { type MockProxy, mock } from 'jest-mock-extended';
 
 import { AnchorBlockDataProvider } from '../storage/anchor_block_data_provider/anchor_block_data_provider.js';
 import { NoteDataProvider } from '../storage/note_data_provider/note_data_provider.js';
-import { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js';
 import { BlockSynchronizer } from './block_synchronizer.js';
 
 describe('BlockSynchronizer', () => {
   let synchronizer: BlockSynchronizer;
   let tipsStore: L2TipsKVStore;
   let anchorBlockDataProvider: AnchorBlockDataProvider;
   let noteDataProvider: NoteDataProvider;
-  let recipientTaggingDataProvider: RecipientTaggingDataProvider;
   let aztecNode: MockProxy<AztecNode>;
   let blockStream: MockProxy<L2BlockStream>;
 
@@ -36,14 +34,7 @@ describe('BlockSynchronizer', () => {
     tipsStore = new L2TipsKVStore(store, 'pxe');
     anchorBlockDataProvider = new AnchorBlockDataProvider(store);
     noteDataProvider = await NoteDataProvider.create(store);
-    recipientTaggingDataProvider = new RecipientTaggingDataProvider(store);
-    synchronizer = new TestSynchronizer(
-      aztecNode,
-      anchorBlockDataProvider,
-      noteDataProvider,
-      recipientTaggingDataProvider,
-      tipsStore,
-    );
+    synchronizer = new TestSynchronizer(aztecNode, anchorBlockDataProvider, noteDataProvider, tipsStore);
   });
 
   it('sets header from latest block', async () => {
@@ -58,9 +49,6 @@ describe('BlockSynchronizer', () => {
     const rollbackNotesAndNullifiers = jest
       .spyOn(noteDataProvider, 'rollbackNotesAndNullifiers')
       .mockImplementation(() => Promise.resolve());
-    const resetNoteSyncData = jest
-      .spyOn(recipientTaggingDataProvider, 'resetNoteSyncData')
-      .mockImplementation(() => Promise.resolve());
     aztecNode.getBlockHeader.mockImplementation(async blockNumber =>
       (await L2Block.random(BlockNumber(blockNumber as number))).getBlockHeader(),
     );
@@ -72,6 +60,5 @@ describe('BlockSynchronizer', () => {
     await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', block: { number: BlockNumber(3), hash: '0x3' } });
 
     expect(rollbackNotesAndNullifiers).toHaveBeenCalledWith(3, 4);
-    expect(resetNoteSyncData).toHaveBeenCalled();
   });
 });

yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts

@@ -7,7 +7,6 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/client';
 import type { PXEConfig } from '../config/index.js';
 import type { AnchorBlockDataProvider } from '../storage/anchor_block_data_provider/anchor_block_data_provider.js';
 import type { NoteDataProvider } from '../storage/note_data_provider/note_data_provider.js';
-import type { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js';
 
 /**
  * The BlockSynchronizer class orchestrates synchronization between PXE and Aztec node, maintaining an up-to-date
@@ -23,7 +22,6 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler {
     private node: AztecNode,
     private anchorBlockDataProvider: AnchorBlockDataProvider,
     private noteDataProvider: NoteDataProvider,
-    private recipientTaggingDataProvider: RecipientTaggingDataProvider,
     private l2TipsStore: L2TipsKVStore,
     config: Partial<Pick<PXEConfig, 'l2BlockBatchSize'>> = {},
     loggerOrSuffix?: string | Logger,
@@ -64,12 +62,6 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler {
         // We first unnullify and then remove so that unnullified notes that were created after the block number end up deleted.
         const lastSynchedBlockNumber = (await this.anchorBlockDataProvider.getBlockHeader()).getBlockNumber();
         await this.noteDataProvider.rollbackNotesAndNullifiers(event.block.number, lastSynchedBlockNumber);
-        // Remove all note tagging indexes to force a full resync. This is suboptimal, but unless we track the
-        // block number in which each index is used it's all we can do.
-        // Note: This is now unnecessary for the sender tagging data provider because the new algorithm handles reorgs.
-        // TODO(#17775): Once this issue is implemented we will have the index-block number mapping, so we can
-        // implement this more intelligently.
-        await this.recipientTaggingDataProvider.resetNoteSyncData();
         // Update the header to the last block.
         const newHeader = await this.node.getBlockHeader(event.block.number);
         if (!newHeader) {

yarn-project/pxe/src/contract_function_simulator/contract_function_simulator.ts

@@ -77,8 +77,9 @@ import type { CapsuleDataProvider } from '../storage/capsule_data_provider/capsu
 import type { ContractDataProvider } from '../storage/contract_data_provider/contract_data_provider.js';
 import type { NoteDataProvider } from '../storage/note_data_provider/note_data_provider.js';
 import type { PrivateEventDataProvider } from '../storage/private_event_data_provider/private_event_data_provider.js';
-import type { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js';
 import type { SenderTaggingDataProvider } from '../storage/tagging_data_provider/sender_tagging_data_provider.js';
+import type { SendersDataProvider } from '../storage/tagging_data_provider/senders_data_provider.js';
+import type { RecipientTaggingDataProvider } from '../tagging/recipient_sync/recipient_tagging_data_provider.js';
 import { ExecutionNoteCache } from './execution_note_cache.js';
 import { ExecutionTaggingIndexCache } from './execution_tagging_index_cache.js';
 import { HashedValuesCache } from './hashed_values_cache.js';
@@ -103,6 +104,7 @@ export class ContractFunctionSimulator {
     private anchorBlockDataProvider: AnchorBlockDataProvider,
     private senderTaggingDataProvider: SenderTaggingDataProvider,
     private recipientTaggingDataProvider: RecipientTaggingDataProvider,
+    private sendersDataProvider: SendersDataProvider,
     private capsuleDataProvider: CapsuleDataProvider,
     private privateEventDataProvider: PrivateEventDataProvider,
     private simulator: CircuitSimulator,
@@ -183,6 +185,7 @@ export class ContractFunctionSimulator {
       this.anchorBlockDataProvider,
       this.senderTaggingDataProvider,
       this.recipientTaggingDataProvider,
+      this.sendersDataProvider,
       this.capsuleDataProvider,
       this.privateEventDataProvider,
       0, // totalPublicArgsCount
@@ -278,8 +281,8 @@ export class ContractFunctionSimulator {
       this.addressDataProvider,
       this.aztecNode,
       this.anchorBlockDataProvider,
-      this.senderTaggingDataProvider,
       this.recipientTaggingDataProvider,
+      this.sendersDataProvider,
       this.capsuleDataProvider,
       this.privateEventDataProvider,
       undefined,

yarn-project/pxe/src/contract_function_simulator/oracle/oracle_version_is_checked.test.ts

@@ -18,8 +18,9 @@ import type { CapsuleDataProvider } from '../../storage/capsule_data_provider/ca
 import type { ContractDataProvider } from '../../storage/contract_data_provider/contract_data_provider.js';
 import type { NoteDataProvider } from '../../storage/note_data_provider/note_data_provider.js';
 import type { PrivateEventDataProvider } from '../../storage/private_event_data_provider/private_event_data_provider.js';
-import type { RecipientTaggingDataProvider } from '../../storage/tagging_data_provider/recipient_tagging_data_provider.js';
 import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js';
+import type { SendersDataProvider } from '../../storage/tagging_data_provider/senders_data_provider.js';
+import type { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js';
 import { ContractFunctionSimulator } from '../contract_function_simulator.js';
 import { UtilityExecutionOracle } from './utility_execution_oracle.js';
 
@@ -34,6 +35,7 @@ describe('Oracle Version Check test suite', () => {
   let anchorBlockDataProvider: ReturnType<typeof mock<AnchorBlockDataProvider>>;
   let senderTaggingDataProvider: ReturnType<typeof mock<SenderTaggingDataProvider>>;
   let recipientTaggingDataProvider: ReturnType<typeof mock<RecipientTaggingDataProvider>>;
+  let sendersDataProvider: ReturnType<typeof mock<SendersDataProvider>>;
   let capsuleDataProvider: ReturnType<typeof mock<CapsuleDataProvider>>;
   let privateEventDataProvider: ReturnType<typeof mock<PrivateEventDataProvider>>;
   let acirSimulator: ContractFunctionSimulator;
@@ -52,6 +54,7 @@ describe('Oracle Version Check test suite', () => {
     anchorBlockDataProvider = mock<AnchorBlockDataProvider>();
     senderTaggingDataProvider = mock<SenderTaggingDataProvider>();
     recipientTaggingDataProvider = mock<RecipientTaggingDataProvider>();
+    sendersDataProvider = mock<SendersDataProvider>();
     capsuleDataProvider = mock<CapsuleDataProvider>();
     privateEventDataProvider = mock<PrivateEventDataProvider>();
     utilityAssertCompatibleOracleVersionSpy = jest.spyOn(
@@ -69,10 +72,7 @@ describe('Oracle Version Check test suite', () => {
     senderTaggingDataProvider.getLastUsedIndex.mockResolvedValue(undefined);
     senderTaggingDataProvider.getTxHashesOfPendingIndexes.mockResolvedValue([]);
     senderTaggingDataProvider.storePendingIndexes.mockResolvedValue();
-    recipientTaggingDataProvider.getSenderAddresses.mockResolvedValue([]);
-    recipientTaggingDataProvider.getLastUsedIndexes.mockImplementation(secrets =>
-      Promise.resolve(secrets.map(() => undefined)),
-    );
+
     noteDataProvider.getNotes.mockResolvedValue([]);
     keyStore.getAccounts.mockResolvedValue([]);
 
@@ -100,6 +100,7 @@ describe('Oracle Version Check test suite', () => {
       anchorBlockDataProvider,
       senderTaggingDataProvider,
       recipientTaggingDataProvider,
+      sendersDataProvider,
       capsuleDataProvider,
       privateEventDataProvider,
       simulator,

yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts

@@ -70,8 +70,9 @@ import type { CapsuleDataProvider } from '../../storage/capsule_data_provider/ca
 import type { ContractDataProvider } from '../../storage/contract_data_provider/contract_data_provider.js';
 import type { NoteDataProvider } from '../../storage/note_data_provider/note_data_provider.js';
 import type { PrivateEventDataProvider } from '../../storage/private_event_data_provider/private_event_data_provider.js';
-import type { RecipientTaggingDataProvider } from '../../storage/tagging_data_provider/recipient_tagging_data_provider.js';
 import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js';
+import type { SendersDataProvider } from '../../storage/tagging_data_provider/senders_data_provider.js';
+import type { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js';
 import { ContractFunctionSimulator } from '../contract_function_simulator.js';
 
 jest.setTimeout(60_000);
@@ -116,12 +117,12 @@ describe('Private Execution test suite', () => {
   let keyStore: MockProxy<KeyStore>;
   let senderTaggingDataProvider: MockProxy<SenderTaggingDataProvider>;
   let recipientTaggingDataProvider: MockProxy<RecipientTaggingDataProvider>;
+  let sendersDataProvider: MockProxy<SendersDataProvider>;
   let aztecNode: MockProxy<AztecNode>;
   let anchorBlockDataProvider: MockProxy<AnchorBlockDataProvider>;
   let capsuleDataProvider: MockProxy<CapsuleDataProvider>;
   let privateEventDataProvider: MockProxy<PrivateEventDataProvider>;
   let acirSimulator: ContractFunctionSimulator;
-
   let anchorBlockHeader = BlockHeader.empty();
   let logger: Logger;
 
@@ -310,6 +311,7 @@ describe('Private Execution test suite', () => {
     anchorBlockDataProvider = mock<AnchorBlockDataProvider>();
     capsuleDataProvider = mock<CapsuleDataProvider>();
     privateEventDataProvider = mock<PrivateEventDataProvider>();
+    sendersDataProvider = mock<SendersDataProvider>();
     contracts = {};
     anchorBlockHeader = makeBlockHeader();
     anchorBlockDataProvider.getBlockHeader.mockImplementation(() => Promise.resolve(anchorBlockHeader));
@@ -320,15 +322,25 @@ describe('Private Execution test suite', () => {
     senderTaggingDataProvider.getLastUsedIndex.mockResolvedValue(undefined);
     senderTaggingDataProvider.getTxHashesOfPendingIndexes.mockResolvedValue([]);
     senderTaggingDataProvider.storePendingIndexes.mockResolvedValue();
-    recipientTaggingDataProvider.getSenderAddresses.mockResolvedValue([]);
-    recipientTaggingDataProvider.getLastUsedIndexes.mockImplementation(secrets =>
-      Promise.resolve(new Array(secrets?.length ?? 0).fill(undefined)),
-    );
+
+    // Mock senders data provider methods
+    sendersDataProvider.getSenderAddresses.mockResolvedValue([]);
 
     // Mock aztec node methods - the return array needs to have the same length as the number of tags
     // on the input.
     aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => Promise.resolve(tags.map(() => [])));
 
+    // Mock getL2Tips and getBlockHeader for loadPrivateLogsForSenderRecipientPair
+    aztecNode.getL2Tips.mockResolvedValue({
+      finalized: { number: anchorBlockHeader.globalVariables.blockNumber },
+    } as any);
+    aztecNode.getBlockHeader.mockImplementation((blockNumber: BlockNumber | 'latest') => {
+      if (blockNumber === 'latest') {
+        return Promise.resolve(anchorBlockHeader);
+      }
+      return Promise.resolve(anchorBlockHeader);
+    });
+
     // TODO: refactor. Maybe it's worth stubbing a key store
     // and cleaning up the mess that is setting up keys.
     // Also: having owner, recipient, and sender for tags
@@ -434,6 +446,7 @@ describe('Private Execution test suite', () => {
       anchorBlockDataProvider,
       senderTaggingDataProvider,
       recipientTaggingDataProvider,
+      sendersDataProvider,
       capsuleDataProvider,
       privateEventDataProvider,
       simulator,

yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts

@@ -36,8 +36,9 @@ import type { CapsuleDataProvider } from '../../storage/capsule_data_provider/ca
 import type { ContractDataProvider } from '../../storage/contract_data_provider/contract_data_provider.js';
 import type { NoteDataProvider } from '../../storage/note_data_provider/note_data_provider.js';
 import type { PrivateEventDataProvider } from '../../storage/private_event_data_provider/private_event_data_provider.js';
-import type { RecipientTaggingDataProvider } from '../../storage/tagging_data_provider/recipient_tagging_data_provider.js';
 import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js';
+import type { SendersDataProvider } from '../../storage/tagging_data_provider/senders_data_provider.js';
+import type { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js';
 import { syncSenderTaggingIndexes } from '../../tagging/sync/sync_sender_tagging_indexes.js';
 import type { ExecutionNoteCache } from '../execution_note_cache.js';
 import { ExecutionTaggingIndexCache } from '../execution_tagging_index_cache.js';
@@ -95,8 +96,9 @@ export class PrivateExecutionOracle extends UtilityExecutionOracle implements IP
     addressDataProvider: AddressDataProvider,
     aztecNode: AztecNode,
     anchorBlockDataProvider: AnchorBlockDataProvider,
-    senderTaggingDataProvider: SenderTaggingDataProvider,
+    private readonly senderTaggingDataProvider: SenderTaggingDataProvider,
     recipientTaggingDataProvider: RecipientTaggingDataProvider,
+    sendersDataProvider: SendersDataProvider,
     capsuleDataProvider: CapsuleDataProvider,
     privateEventDataProvider: PrivateEventDataProvider,
     private totalPublicCalldataCount: number = 0,
@@ -117,8 +119,8 @@ export class PrivateExecutionOracle extends UtilityExecutionOracle implements IP
       addressDataProvider,
       aztecNode,
       anchorBlockDataProvider,
-      senderTaggingDataProvider,
       recipientTaggingDataProvider,
+      sendersDataProvider,
       capsuleDataProvider,
       privateEventDataProvider,
       log,
@@ -580,6 +582,7 @@ export class PrivateExecutionOracle extends UtilityExecutionOracle implements IP
       this.anchorBlockDataProvider,
       this.senderTaggingDataProvider,
       this.recipientTaggingDataProvider,
+      this.sendersDataProvider,
       this.capsuleDataProvider,
       this.privateEventDataProvider,
       this.totalPublicCalldataCount,