diff --git a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts index 9f8783e6881d..d471834578c5 100644 --- a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts +++ b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts @@ -26,6 +26,8 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import type { BlockSynchronizerConfig } from '../config/index.js'; import type { ContractSyncService } from '../contract_sync/contract_sync_service.js'; import { AnchorBlockStore } from '../storage/anchor_block_store/anchor_block_store.js'; +import { FactStore } from '../storage/fact_store/fact_store.js'; +import { FactCollectionKey, FactCollectionTypeKey } from '../storage/fact_store/fact_store_keys.js'; import { NoteStore } from '../storage/note_store/note_store.js'; import { PrivateEventStore } from '../storage/private_event_store/private_event_store.js'; import { BlockSynchronizer } from './block_synchronizer.js'; @@ -43,6 +45,7 @@ describe('BlockSynchronizer', () => { let anchorBlockStore: AnchorBlockStore; let noteStore: NoteStore; let privateEventStore: PrivateEventStore; + let factStore: FactStore; let aztecNode: MockProxy; let getBlock: NodeGetBlockMock; let blockStream: MockProxy; @@ -61,6 +64,7 @@ describe('BlockSynchronizer', () => { anchorBlockStore, noteStore, privateEventStore, + factStore, tipsStore, contractSyncService, config, @@ -77,6 +81,18 @@ describe('BlockSynchronizer', () => { number: block.number, }); + // The L2BlockId (number + hash) for a block, as carried by block-stream events. + const blockId = async (block: L2Block): Promise => + makeL2BlockId(block.number, (await block.hash()).toString()); + + // Configures the node to serve `block` by hash. + const serveBlock = async (block: L2Block) => { + const response = await blockResponse(block); + getBlock.mockImplementation(param => + Promise.resolve(param instanceof BlockHash && param.equals(response.hash) ? response : undefined), + ); + }; + // Builds a note DAO anchored to the given block id (caller adds it to the store and commits). const noteAt = (contract: AztecAddress, block: L2BlockId): Promise => NoteDao.random({ contractAddress: contract, l2BlockNumber: block.number, l2BlockHash: block.hash }); @@ -109,6 +125,7 @@ describe('BlockSynchronizer', () => { anchorBlockStore = new AnchorBlockStore(store); noteStore = new NoteStore(store); privateEventStore = new PrivateEventStore(store); + factStore = new FactStore(store); contractSyncService = mock(); synchronizer = createSynchronizer(); }); @@ -150,10 +167,7 @@ describe('BlockSynchronizer', () => { it('updates anchor block on a reorg', async () => { const reorgBlock = await L2Block.random(BlockNumber(3)); - const reorgResponse = await blockResponse(reorgBlock); - getBlock.mockImplementation(param => - Promise.resolve(param instanceof BlockHash && param.equals(reorgResponse.hash) ? reorgResponse : undefined), - ); + await serveBlock(reorgBlock); // Anchor sits above the prune target so the prune guard lets the rollback through. const anchorBlock = await L2Block.random(BlockNumber(4)); @@ -161,7 +175,7 @@ describe('BlockSynchronizer', () => { await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', - block: makeL2BlockId(reorgBlock.number, reorgResponse.hash.toString()), + block: await blockId(reorgBlock), checkpointed: { block: makeL2BlockId(BlockNumber.ZERO, GENESIS_BLOCK_HEADER_HASH.toString()), checkpoint: makeL2CheckpointId(CheckpointNumber.ZERO, GENESIS_CHECKPOINT_HEADER_HASH.toString()), @@ -229,12 +243,11 @@ describe('BlockSynchronizer', () => { // Block 3 is the fork point (a real block the node still serves); 4 and 5 are on the abandoned fork. const forkBlock = await L2Block.random(BlockNumber(3)); - const block3 = makeL2BlockId(forkBlock.number, (await forkBlock.hash()).toString()); const block4 = makeL2BlockId(BlockNumber(4), Fr.random().toString()); const block5 = makeL2BlockId(BlockNumber(5), Fr.random().toString()); // Seed a note at each block, anchored to that block's id. - const noteAt3 = await noteAt(contract, block3); + const noteAt3 = await noteAt(contract, await blockId(forkBlock)); const noteAt4 = await noteAt(contract, block4); const noteAt5 = await noteAt(contract, block5); await noteStore.addNotes([noteAt3, noteAt4, noteAt5], scope, 'note-job'); @@ -244,7 +257,7 @@ describe('BlockSynchronizer', () => { const eventIdAt3 = Fr.random(); const eventIdAt4 = Fr.random(); const eventIdAt5 = Fr.random(); - await storeEvent(contract, scope, eventIdAt3, block3); + await storeEvent(contract, scope, eventIdAt3, await blockId(forkBlock)); await storeEvent(contract, scope, eventIdAt4, block4); await storeEvent(contract, scope, eventIdAt5, block5); await privateEventStore.commit('event-job'); @@ -254,15 +267,12 @@ describe('BlockSynchronizer', () => { await anchorBlockStore.setHeader(anchorBlock5.header); // The node serves the fork-point block; it becomes the new anchor after the prune. - const forkResponse = await blockResponse(forkBlock); - getBlock.mockImplementation(param => - Promise.resolve(param instanceof BlockHash && param.equals(forkResponse.hash) ? forkResponse : undefined), - ); + await serveBlock(forkBlock); // Prune back to block 3 (orphaning blocks 4 and 5). await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', - block: block3, + block: await blockId(forkBlock), checkpointed: { block: makeL2BlockId(BlockNumber.ZERO, GENESIS_BLOCK_HEADER_HASH.toString()), checkpoint: makeL2CheckpointId(CheckpointNumber.ZERO, GENESIS_CHECKPOINT_HEADER_HASH.toString()), @@ -315,17 +325,177 @@ describe('BlockSynchronizer', () => { expect(await privateEventStore.eventIdsAtBlock(9)).toEqual([eventId9.toString()]); }); + it('chain-pruned retracts facts at pruned block heights or above, dropping collections left empty', async () => { + const jobId = 'fact-job'; + + // Block 5 will be the fork point: the prune keeps it and abandons only blocks strictly above it. + const lastSurvivingBlock = await L2Block.random(BlockNumber(5)); + + const contractAddress = await AztecAddress.random(); + const scope = await AztecAddress.random(); + const factCollectionTypeId = Fr.random(); + const typeKey = FactCollectionTypeKey.from({ contractAddress, scope, factCollectionTypeId }); + + // A collection whose only fact is retractable and anchored to the fork point (block 5): the fork point is kept, + // so the fact and its collection must survive. + const survivingCollectionId = Fr.random(); + const survivingCollectionKey = FactCollectionKey.from({ + contractAddress, + scope, + factCollectionTypeId, + factCollectionId: survivingCollectionId, + }); + await factStore.recordFact( + survivingCollectionKey, + Fr.random(), + [Fr.random()], + { blockNumber: lastSurvivingBlock.number, blockHash: (await lastSurvivingBlock.hash()).toFr() }, + jobId, + ); + + // A collection whose only fact is retractable and originates just above the fork (block 6): the prune deletes the + // fact, and the now-empty collection disappears entirely. + const retractedCollectionId = Fr.random(); + const retractedCollectionKey = FactCollectionKey.from({ + contractAddress, + scope, + factCollectionTypeId, + factCollectionId: retractedCollectionId, + }); + await factStore.recordFact( + retractedCollectionKey, + Fr.random(), + [Fr.random()], + { blockNumber: lastSurvivingBlock.number + 1, blockHash: Fr.random() }, + jobId, + ); + + await store.transactionAsync(() => factStore.commit(jobId)); + + // Both collections must be present before the prune. + expect(await factStore.getFactCollectionsByType(typeKey, jobId)).toHaveLength(2); + // Release the read job so the prune's rollback is not blocked by an in-flight job. + await factStore.discardStaged(jobId); + + // Some blocks later... + const anchorBlock10 = await L2Block.random(BlockNumber(10)); + await anchorBlockStore.setHeader(anchorBlock10.header); + + // The node serves the fork-point block (number 5), so it becomes the new anchor after the prune. + await serveBlock(lastSurvivingBlock); + + // Prune back to block 5, dropping block 6 where the retracted fact originates. + await synchronizer.handleBlockStreamEvent({ + type: 'chain-pruned', + block: await blockId(lastSurvivingBlock), + checkpointed: { + block: makeL2BlockId(BlockNumber.ZERO, GENESIS_BLOCK_HEADER_HASH.toString()), + checkpoint: makeL2CheckpointId(CheckpointNumber.ZERO, GENESIS_CHECKPOINT_HEADER_HASH.toString()), + }, + proven: { + block: makeL2BlockId(BlockNumber.ZERO, GENESIS_BLOCK_HEADER_HASH.toString()), + checkpoint: makeL2CheckpointId(CheckpointNumber.ZERO, GENESIS_CHECKPOINT_HEADER_HASH.toString()), + }, + }); + + // Only the fork-point collection survives. The one whose sole fact originated above the fork is gone. + const collections = await factStore.getFactCollectionsByType(typeKey, jobId); + expect(collections).toHaveLength(1); + expect(collections[0].key.factCollectionId.equals(survivingCollectionId)).toBe(true); + expect(await factStore.getFactCollection(retractedCollectionKey, jobId)).toBeUndefined(); + expect((await factStore.getFactCollection(survivingCollectionKey, jobId))!.facts).toHaveLength(1); + }); + + it('chain-pruned keeps a collection and its facts up to the fork point, deleting only those above it', async () => { + const jobId = 'fact-job'; + + // Block 5 is the fork point: the prune keeps it and abandons only blocks strictly above it. + const lastSurvivingBlock = await L2Block.random(BlockNumber(5)); + + const contractAddress = await AztecAddress.random(); + const scope = await AztecAddress.random(); + + const factCollectionTypeId = Fr.random(); + const factCollectionId = Fr.random(); + const retractedFactType = Fr.random(); + const forkPointFactType = Fr.random(); + const nonRetractableFactType = Fr.random(); + + const typeKey = FactCollectionTypeKey.from({ contractAddress, scope, factCollectionTypeId }); + const collectionKey = FactCollectionKey.from({ contractAddress, scope, factCollectionTypeId, factCollectionId }); + + // A collection carrying three facts: a non-retractable one, a retractable one anchored to the fork point (block + // 5), and a retractable one originating just above it (block 6). The prune must keep the collection, its + // non-retractable fact, and the fork-point fact, deleting only the orphaned fact. + await factStore.recordFact(collectionKey, nonRetractableFactType, [Fr.random()], undefined, jobId); + await factStore.recordFact( + collectionKey, + forkPointFactType, + [], + { blockNumber: lastSurvivingBlock.number, blockHash: (await lastSurvivingBlock.hash()).toFr() }, + jobId, + ); + await factStore.recordFact( + collectionKey, + retractedFactType, + [], + { blockNumber: lastSurvivingBlock.number + 1, blockHash: Fr.random() }, + jobId, + ); + await store.transactionAsync(() => factStore.commit(jobId)); + + // The collection and all three facts must be present before the prune. + expect(await factStore.getFactCollectionsByType(typeKey, jobId)).toHaveLength(1); + expect((await factStore.getFactCollection(collectionKey, jobId))!.facts).toHaveLength(3); + // Release the read job so the prune's rollback is not blocked by an in-flight job. + await factStore.discardStaged(jobId); + + // Some blocks later... + const anchorBlock10 = await L2Block.random(BlockNumber(10)); + await anchorBlockStore.setHeader(anchorBlock10.header); + + // The node serves the fork-point block, so it becomes the new anchor after the prune. + await serveBlock(lastSurvivingBlock); + + // Prune back to block 5, orphaning block 6 where the retractable fact originates. + await synchronizer.handleBlockStreamEvent({ + type: 'chain-pruned', + block: await blockId(lastSurvivingBlock), + checkpointed: { + block: makeL2BlockId(BlockNumber.ZERO, GENESIS_BLOCK_HEADER_HASH.toString()), + checkpoint: makeL2CheckpointId(CheckpointNumber.ZERO, GENESIS_CHECKPOINT_HEADER_HASH.toString()), + }, + proven: { + block: makeL2BlockId(BlockNumber.ZERO, GENESIS_BLOCK_HEADER_HASH.toString()), + checkpoint: makeL2CheckpointId(CheckpointNumber.ZERO, GENESIS_CHECKPOINT_HEADER_HASH.toString()), + }, + }); + + // The collection survives, keeping its non-retractable fact and the fork-point fact. Only the fact originating + // above the fork is gone. + const collections = await factStore.getFactCollectionsByType(typeKey, jobId); + expect(collections).toHaveLength(1); + expect(collections[0].key.factCollectionId.equals(factCollectionId)).toBe(true); + + const remainingFactTypes = (await factStore.getFactCollection(collectionKey, jobId))!.facts.map( + fact => fact.factTypeId, + ); + expect(remainingFactTypes).toHaveLength(2); + expect(remainingFactTypes.some(factType => factType.equals(nonRetractableFactType))).toBe(true); + expect(remainingFactTypes.some(factType => factType.equals(forkPointFactType))).toBe(true); + expect(remainingFactTypes.some(factType => factType.equals(retractedFactType))).toBe(false); + }); + it('notes below the fork survive and remain queryable after a prune', async () => { const contract = await AztecAddress.random(); const scope = await AztecAddress.random(); // Block 1 is the fork point (a real block the node still serves); 2 and 3 are on the abandoned fork. const forkBlock = await L2Block.random(BlockNumber(1)); - const block1 = makeL2BlockId(forkBlock.number, (await forkBlock.hash()).toString()); const block2 = makeL2BlockId(BlockNumber(2), Fr.random().toString()); const block3 = makeL2BlockId(BlockNumber(3), Fr.random().toString()); - const noteAt1 = await noteAt(contract, block1); + const noteAt1 = await noteAt(contract, await blockId(forkBlock)); const noteAt2 = await noteAt(contract, block2); const noteAt3 = await noteAt(contract, block3); await noteStore.addNotes([noteAt1, noteAt2, noteAt3], scope, 'note-job'); @@ -336,15 +506,12 @@ describe('BlockSynchronizer', () => { await anchorBlockStore.setHeader(anchorBlock3.header); // The node serves the fork-point block; it becomes the new anchor after the prune. - const forkResponse = await blockResponse(forkBlock); - getBlock.mockImplementation(param => - Promise.resolve(param instanceof BlockHash && param.equals(forkResponse.hash) ? forkResponse : undefined), - ); + await serveBlock(forkBlock); // Prune back to block 1 (orphaning blocks 2 and 3). await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', - block: block1, + block: await blockId(forkBlock), checkpointed: { block: makeL2BlockId(BlockNumber.ZERO, GENESIS_BLOCK_HEADER_HASH.toString()), checkpoint: makeL2CheckpointId(CheckpointNumber.ZERO, GENESIS_CHECKPOINT_HEADER_HASH.toString()), @@ -626,6 +793,7 @@ describe('BlockSynchronizer', () => { anchorBlockStore, noteStore, privateEventStore, + factStore, tipsStore, contractSyncService, { syncChainTip: 'proposed' }, diff --git a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts index 839344854f52..5675dee647ec 100644 --- a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts +++ b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts @@ -10,6 +10,7 @@ import type { BlockHeader } from '@aztec/stdlib/tx'; import type { BlockSynchronizerConfig } from '../config/index.js'; import type { ContractSyncService } from '../contract_sync/contract_sync_service.js'; import type { AnchorBlockStore } from '../storage/anchor_block_store/index.js'; +import type { FactStore } from '../storage/fact_store/fact_store.js'; import type { NoteStore } from '../storage/note_store/index.js'; import type { PrivateEventStore } from '../storage/private_event_store/private_event_store.js'; import { blockStreamSourceFromAztecNode } from './block_stream_source.js'; @@ -26,14 +27,15 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler { protected readonly blockStream: L2BlockStream; constructor( - private node: AztecNode, - private store: AztecAsyncKVStore, - private anchorBlockStore: AnchorBlockStore, - private noteStore: NoteStore, - private privateEventStore: PrivateEventStore, - private l2TipsStore: L2TipsKVStore, - private contractSyncService: ContractSyncService, - private config: Partial = {}, + private readonly node: AztecNode, + private readonly store: AztecAsyncKVStore, + private readonly anchorBlockStore: AnchorBlockStore, + private readonly noteStore: NoteStore, + private readonly privateEventStore: PrivateEventStore, + private readonly factStore: FactStore, + private readonly l2TipsStore: L2TipsKVStore, + private readonly contractSyncService: ContractSyncService, + private readonly config: Partial = {}, bindings?: LoggerBindings, ) { this.log = createLogger('pxe:block_synchronizer', bindings); @@ -152,6 +154,7 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler { await this.store.transactionAsync(async () => { await this.noteStore.rollback(event.block.number); await this.privateEventStore.rollback(event.block.number); + await this.factStore.rollback(event.block.number); await this.updateAnchorBlockHeader(newAnchorBlockHeader); }); break; diff --git a/yarn-project/pxe/src/pxe.ts b/yarn-project/pxe/src/pxe.ts index 056b411e477a..2a3c5b80d5f5 100644 --- a/yarn-project/pxe/src/pxe.ts +++ b/yarn-project/pxe/src/pxe.ts @@ -274,6 +274,7 @@ export class PXE { capsuleStore, keyStore, l2TipsStore, + factStore, } = openPxeStores(store, initialBlockHash); const contractSyncService = new ContractSyncService( node, @@ -289,6 +290,7 @@ export class PXE { anchorBlockStore, noteStore, privateEventStore, + factStore, l2TipsStore, contractSyncService, config, @@ -302,6 +304,7 @@ export class PXE { recipientTaggingStore, privateEventStore, noteStore, + factStore, contractSyncService, ]); diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/FactStore.json b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/FactStore.json new file mode 100644 index 000000000000..ea146489013e --- /dev/null +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/FactStore.json @@ -0,0 +1,40 @@ +{ + "facts": [ + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000aa:0x0000000000000000000000000000000000000000000000000000000000000003:0x00eebee6df09ad503384e049aea9fbf0c0db2f5af46673e550694c6564d973f7:6:0x0000000000000000000000000000000000000000000000000000000000000002", + "value": "00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000aa000000000000000000000000000000000000000000000000000000000000000300000001000000000000000000000000000000000000000000000000000000000000000501000000060000000000000000000000000000000000000000000000000000000000000002" + }, + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000001:0x00a802572c436574f713d43f1852859f4496d694de0e5b17b89cf983439e6143:none", + "value": "00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000bb000000000000000000000000000000000000000000000000000000000000000100000001000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000" + }, + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000002:0x00df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b811:5:0x0000000000000000000000000000000000000000000000000000000000000001", + "value": "00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000bb00000000000000000000000000000000000000000000000000000000000000020000000001000000050000000000000000000000000000000000000000000000000000000000000001" + } + ], + "facts_by_collection": [ + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000aa", + "value": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000aa:0x0000000000000000000000000000000000000000000000000000000000000003:0x00eebee6df09ad503384e049aea9fbf0c0db2f5af46673e550694c6564d973f7:6:0x0000000000000000000000000000000000000000000000000000000000000002" + }, + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb", + "value": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000001:0x00a802572c436574f713d43f1852859f4496d694de0e5b17b89cf983439e6143:none" + }, + { + "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb", + "value": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000002:0x00df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b811:5:0x0000000000000000000000000000000000000000000000000000000000000001" + } + ], + "facts_by_block": [ + { + "key": "num:5", + "value": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000bb:0x0000000000000000000000000000000000000000000000000000000000000002:0x00df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b811:5:0x0000000000000000000000000000000000000000000000000000000000000001" + }, + { + "key": "num:6", + "value": "utf8:0x0000000000000000000000000000000000000000000000000000000000000064:0x0000000000000000000000000000000000000000000000000000000000000001:0x0000000000000000000000000000000000000000000000000000000000000007:0x00000000000000000000000000000000000000000000000000000000000000aa:0x0000000000000000000000000000000000000000000000000000000000000003:0x00eebee6df09ad503384e049aea9fbf0c0db2f5af46673e550694c6564d973f7:6:0x0000000000000000000000000000000000000000000000000000000000000002" + } + ] +} diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json index 0f52df771e6c..61059318ea2e 100644 --- a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json @@ -1,5 +1,5 @@ { - "schemaVersion": 8, + "schemaVersion": 9, "stores": [ { "name": "address_book", @@ -37,6 +37,18 @@ "name": "events_by_contract_selector", "kind": "multimap" }, + { + "name": "facts", + "kind": "map" + }, + { + "name": "facts_by_block", + "kind": "multimap" + }, + { + "name": "facts_by_collection", + "kind": "multimap" + }, { "name": "header", "kind": "singleton" diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts b/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts index 9b9f037de3e8..c38580011bb1 100644 --- a/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts @@ -39,6 +39,8 @@ import { AddressStore } from '../address_store/address_store.js'; import { AnchorBlockStore } from '../anchor_block_store/index.js'; import { CapsuleStore } from '../capsule_store/capsule_store.js'; import { ContractStore } from '../contract_store/contract_store.js'; +import { FactStore } from '../fact_store/fact_store.js'; +import { FactCollectionKey } from '../fact_store/fact_store_keys.js'; import { NoteStore } from '../note_store/note_store.js'; import { PrivateEventStore } from '../private_event_store/private_event_store.js'; import { RecipientTaggingStore, SenderAddressBookStore, SenderTaggingStore } from '../tagging_store/index.js'; @@ -214,6 +216,40 @@ export const SCHEMA_TESTS: readonly SchemaTest[] = [ }), }, + { + name: 'FactStore', + writeToStore: async kvStore => { + const factStore = new FactStore(kvStore); + const jobId = 'fixture-job'; + const contract = AztecAddress.fromBigInt(100n); + const scope = AztecAddress.fromBigInt(1n); + const factCollectionTypeId = new Fr(7n); + const keyA = FactCollectionKey.from({ + contractAddress: contract, + scope, + factCollectionTypeId, + factCollectionId: new Fr(0xaan), + }); + const keyB = FactCollectionKey.from({ + contractAddress: contract, + scope, + factCollectionTypeId, + factCollectionId: new Fr(0xbbn), + }); + // A collection whose only fact is retractable (origin block 6): pruned on a reorg above block 6. + await factStore.recordFact(keyA, new Fr(3n), [new Fr(5n)], { blockNumber: 6, blockHash: new Fr(2n) }, jobId); + // A collection with a non-retractable and a retractable fact. + await factStore.recordFact(keyB, new Fr(1n), [new Fr(9n)], undefined, jobId); + await factStore.recordFact(keyB, new Fr(2n), [], { blockNumber: 5, blockHash: new Fr(1n) }, jobId); + await kvStore.transactionAsync(() => factStore.commit(jobId)); + }, + snapshotStore: async kvStore => ({ + facts: await snapshotMap(kvStore.openMap('facts')), + facts_by_collection: await snapshotMap(kvStore.openMultiMap('facts_by_collection')), + facts_by_block: await snapshotMap(kvStore.openMultiMap('facts_by_block')), + }), + }, + { name: 'KeyStore', writeToStore: async kvStore => { diff --git a/yarn-project/pxe/src/storage/fact_store/fact_store.test.ts b/yarn-project/pxe/src/storage/fact_store/fact_store.test.ts new file mode 100644 index 000000000000..a21ff297ed86 --- /dev/null +++ b/yarn-project/pxe/src/storage/fact_store/fact_store.test.ts @@ -0,0 +1,429 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import type { AztecAsyncKVStore } from '@aztec/kv-store'; +import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import { FactStore } from './fact_store.js'; +import { FactCollectionKey, FactCollectionTypeKey } from './fact_store_keys.js'; + +describe('FactStore', () => { + let contract: AztecAddress; + let scope: AztecAddress; + let scopeB: AztecAddress; + let factCollectionTypeId: Fr; + let factTypeA: Fr; + let factTypeB: Fr; + let collectionId1: Fr; + let collectionId2: Fr; + let collectionKey1: FactCollectionKey; + let collectionKey2: FactCollectionKey; + let collectionKey1ScopeB: FactCollectionKey; + let typeKey: FactCollectionTypeKey; + let typeKeyScopeB: FactCollectionTypeKey; + const JOB = 'fact-store-test-job'; + + let kv: AztecAsyncKVStore; + let store: FactStore; + + beforeEach(async () => { + contract = await AztecAddress.random(); + scope = await AztecAddress.random(); + scopeB = await AztecAddress.random(); + factCollectionTypeId = Fr.random(); + factTypeA = Fr.random(); + factTypeB = Fr.random(); + collectionId1 = Fr.random(); + collectionId2 = Fr.random(); + collectionKey1 = FactCollectionKey.from({ + contractAddress: contract, + scope, + factCollectionTypeId, + factCollectionId: collectionId1, + }); + collectionKey2 = FactCollectionKey.from({ + contractAddress: contract, + scope, + factCollectionTypeId, + factCollectionId: collectionId2, + }); + collectionKey1ScopeB = FactCollectionKey.from({ + contractAddress: contract, + scope: scopeB, + factCollectionTypeId, + factCollectionId: collectionId1, + }); + typeKey = FactCollectionTypeKey.from({ contractAddress: contract, scope, factCollectionTypeId }); + typeKeyScopeB = FactCollectionTypeKey.from({ contractAddress: contract, scope: scopeB, factCollectionTypeId }); + + kv = await openTmpStore('fact-store-test'); + store = new FactStore(kv); + }); + afterEach(async () => { + await kv.close(); + }); + + const collectionIdsOf = (collections: { key: FactCollectionKey }[]) => collections.map(c => c.key.factCollectionId); + const hexSet = (frs: Fr[]) => new Set(frs.map(f => f.toString())); + + describe('recording and reading', () => { + it('records facts and reads a collection back after commit (implicit collection creation)', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await store.recordFact(collectionKey1, factTypeB, [], { blockNumber: 5, blockHash: Fr.random() }, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const { facts } = (await store.getFactCollection(collectionKey1, JOB))!; + expect(hexSet(facts.map(f => f.factTypeId))).toEqual(hexSet([factTypeA, factTypeB])); + }); + + it('getFactCollection returns undefined when no collection exists', async () => { + expect(await store.getFactCollection(collectionKey1, JOB)).toBeUndefined(); + }); + + it('lists collections via getFactCollectionsByType', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await store.recordFact(collectionKey2, factTypeA, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const collections = await store.getFactCollectionsByType(typeKey, JOB); + expect(hexSet(collectionIdsOf(collections))).toEqual(hexSet([collectionId1, collectionId2])); + }); + + it('getFactCollectionsByType returns each collection complete with its facts', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await store.recordFact(collectionKey1, factTypeB, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const collections = await store.getFactCollectionsByType(typeKey, JOB); + expect(collections).toHaveLength(1); + expect(hexSet(collections[0].facts.map(f => f.factTypeId))).toEqual(hexSet([factTypeA, factTypeB])); + }); + }); + + describe('idempotency and dedup', () => { + it('dedups identical (collection, factType, payload, originBlock) fact records', async () => { + const payload = Fr.random(); + await store.recordFact(collectionKey1, factTypeA, [payload], undefined, JOB); + await store.recordFact(collectionKey1, factTypeA, [payload], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + expect((await store.getFactCollection(collectionKey1, JOB))!.facts).toHaveLength(1); + }); + + it('the same payload at a different origin block is a distinct fact', async () => { + const payload = Fr.random(); + await store.recordFact(collectionKey1, factTypeA, [payload], { blockNumber: 5, blockHash: Fr.random() }, JOB); + await store.recordFact(collectionKey1, factTypeA, [payload], { blockNumber: 10, blockHash: Fr.random() }, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const { facts } = (await store.getFactCollection(collectionKey1, JOB))!; + expect(facts).toHaveLength(2); + expect(new Set(facts.map(f => f.originBlock?.blockNumber))).toEqual(new Set([5, 10])); + }); + + it('re-recording an identical fact across jobs is a no-op', async () => { + const payload = Fr.random(); + await store.recordFact(collectionKey1, factTypeA, [payload], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const JOB2 = 'rerecord-job'; + await store.recordFact(collectionKey1, factTypeA, [payload], undefined, JOB2); + await kv.transactionAsync(() => store.commit(JOB2)); + + expect((await store.getFactCollection(collectionKey1, JOB))!.facts).toHaveLength(1); + }); + }); + + describe('scope isolation', () => { + it('a collection recorded under one scope is a different collection under another scope', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + expect(await store.getFactCollection(collectionKey1, JOB)).toBeDefined(); + expect(await store.getFactCollection(collectionKey1ScopeB, JOB)).toBeUndefined(); + expect(await store.getFactCollectionsByType(typeKeyScopeB, JOB)).toHaveLength(0); + }); + + it('the same (contract, type, id) under two scopes are independent collections', async () => { + const payload = Fr.random(); + const origin = { blockNumber: 5, blockHash: Fr.random() }; + await store.recordFact(collectionKey1, factTypeA, [payload], origin, JOB); + await store.recordFact(collectionKey1ScopeB, factTypeB, [payload], origin, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + expect((await store.getFactCollection(collectionKey1, JOB))!.facts.map(f => f.factTypeId)).toEqual([factTypeA]); + expect((await store.getFactCollection(collectionKey1ScopeB, JOB))!.facts.map(f => f.factTypeId)).toEqual([ + factTypeB, + ]); + }); + + it('getFactCollectionsByType only returns collections for the queried scope', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await store.recordFact(collectionKey1ScopeB, factTypeA, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + expect(collectionIdsOf(await store.getFactCollectionsByType(typeKey, JOB))).toEqual([collectionId1]); + expect(collectionIdsOf(await store.getFactCollectionsByType(typeKeyScopeB, JOB))).toEqual([collectionId1]); + }); + }); + + describe('read-your-writes', () => { + it("reflects a job's own staged facts before commit; other jobs do not see them", async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + + expect((await store.getFactCollection(collectionKey1, JOB))!.facts.map(f => f.factTypeId)).toEqual([factTypeA]); + expect(collectionIdsOf(await store.getFactCollectionsByType(typeKey, JOB))).toEqual([collectionId1]); + + expect(await store.getFactCollection(collectionKey1, 'other-job')).toBeUndefined(); + expect(await store.getFactCollectionsByType(typeKey, 'other-job')).toHaveLength(0); + }); + + it('staged facts combine with committed ones', async () => { + const payloads = Array.from({ length: 4 }, () => Fr.random()); + await store.recordFact(collectionKey1, factTypeA, [payloads[0]], undefined, JOB); + await store.recordFact(collectionKey1, factTypeA, [payloads[1]], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const JOB2 = 'staged-job'; + await store.recordFact(collectionKey1, factTypeA, [payloads[2]], undefined, JOB2); + await store.recordFact(collectionKey1, factTypeA, [payloads[3]], undefined, JOB2); + + const { facts } = (await store.getFactCollection(collectionKey1, JOB2))!; + expect(hexSet(facts.map(f => f.payload[0]))).toEqual(hexSet(payloads)); + }); + }); + + describe('deleteFactCollection', () => { + it('deletes the collection and leaves neighbouring collections untouched', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await store.recordFact(collectionKey2, factTypeA, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const DEL = 'delete-job'; + await store.deleteFactCollection(collectionKey1, DEL); + await kv.transactionAsync(() => store.commit(DEL)); + + expect(await store.getFactCollection(collectionKey1, JOB)).toBeUndefined(); + expect(collectionIdsOf(await store.getFactCollectionsByType(typeKey, JOB))).toEqual([collectionId2]); + }); + + it('only deletes the queried scope: the same (contract,type,id) under another scope survives', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await store.recordFact(collectionKey1ScopeB, factTypeB, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const DEL = 'delete-job'; + await store.deleteFactCollection(collectionKey1, DEL); + await kv.transactionAsync(() => store.commit(DEL)); + + expect(await store.getFactCollection(collectionKey1, JOB)).toBeUndefined(); + expect((await store.getFactCollection(collectionKey1ScopeB, JOB))!.facts.map(f => f.factTypeId)).toEqual([ + factTypeB, + ]); + }); + + it('is a no-op for a collection that does not exist', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const DEL = 'delete-job'; + await store.deleteFactCollection(collectionKey2, DEL); + await kv.transactionAsync(() => store.commit(DEL)); + + expect((await store.getFactCollection(collectionKey1, JOB))!.facts).toHaveLength(1); + }); + + it('hides a collection from its own job after a staged delete, even over committed facts', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const DEL = 'delete-job'; + await store.deleteFactCollection(collectionKey1, DEL); + + expect(await store.getFactCollection(collectionKey1, DEL)).toBeUndefined(); + expect(await store.getFactCollectionsByType(typeKey, DEL)).toHaveLength(0); + expect((await store.getFactCollection(collectionKey1, 'reader'))!.facts).toHaveLength(1); + }); + + it('a staged delete-then-record re-creates the collection within the same job', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const JOB2 = 'recreate-job'; + await store.deleteFactCollection(collectionKey1, JOB2); + await store.recordFact(collectionKey1, factTypeB, [Fr.random()], undefined, JOB2); + + const { facts } = (await store.getFactCollection(collectionKey1, JOB2))!; + expect(facts.map(f => f.factTypeId)).toEqual([factTypeB]); + + await kv.transactionAsync(() => store.commit(JOB2)); + expect((await store.getFactCollection(collectionKey1, 'reader'))!.facts.map(f => f.factTypeId)).toEqual([ + factTypeB, + ]); + }); + + it('a staged record-then-delete leaves the collection deleted', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await store.deleteFactCollection(collectionKey1, JOB); + + expect(await store.getFactCollection(collectionKey1, JOB)).toBeUndefined(); + + await kv.transactionAsync(() => store.commit(JOB)); + expect(await store.getFactCollection(collectionKey1, 'reader')).toBeUndefined(); + }); + }); + + describe('rollback and retraction', () => { + it('removes retractable facts above the target block and keeps non-retractable ones', async () => { + const nonRetractable = Fr.random(); + const retractable = Fr.random(); + await store.recordFact(collectionKey1, factTypeA, [nonRetractable], undefined, JOB); + await store.recordFact(collectionKey1, factTypeB, [retractable], { blockNumber: 6, blockHash: Fr.random() }, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + await kv.transactionAsync(() => store.rollback(5)); + + const { facts } = (await store.getFactCollection(collectionKey1, JOB))!; + expect(hexSet(facts.map(f => f.payload[0]))).toEqual(hexSet([nonRetractable])); + }); + + it('a collection left with no facts after retraction disappears', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], { blockNumber: 6, blockHash: Fr.random() }, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + await kv.transactionAsync(() => store.rollback(5)); + + expect(await store.getFactCollection(collectionKey1, JOB)).toBeUndefined(); + expect(await store.getFactCollectionsByType(typeKey, JOB)).toHaveLength(0); + }); + + it('the same payload at two origin blocks yields independent facts pruned per block', async () => { + const payload = Fr.random(); + await store.recordFact(collectionKey1, factTypeA, [payload], { blockNumber: 5, blockHash: Fr.random() }, JOB); + await store.recordFact(collectionKey1, factTypeA, [payload], { blockNumber: 10, blockHash: Fr.random() }, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + await kv.transactionAsync(() => store.rollback(7)); + expect((await store.getFactCollection(collectionKey1, JOB))!.facts.map(f => f.originBlock?.blockNumber)).toEqual([ + 5, + ]); + await store.discardStaged(JOB); + + await kv.transactionAsync(() => store.rollback(4)); + expect(await store.getFactCollection(collectionKey1, JOB)).toBeUndefined(); + }); + + it('rollback throws while a job has staged writes', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, 'uncommitted-job'); + await expect(kv.transactionAsync(() => store.rollback(0))).rejects.toThrow( + 'PXE fact store rollback is not allowed while jobs are running', + ); + await store.discardStaged('uncommitted-job'); + await expect(kv.transactionAsync(() => store.rollback(0))).resolves.not.toThrow(); + }); + + it('a job that has only read still blocks rollback until it is discarded', async () => { + await store.getFactCollection(collectionKey1, 'reader-job'); + await expect(kv.transactionAsync(() => store.rollback(0))).rejects.toThrow( + 'PXE fact store rollback is not allowed while jobs are running', + ); + await store.discardStaged('reader-job'); + await expect(kv.transactionAsync(() => store.rollback(0))).resolves.not.toThrow(); + }); + }); + + describe('isolation', () => { + it('collections under different contracts and types are isolated', async () => { + const contract2 = await AztecAddress.random(); + const type2 = Fr.random(); + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await store.recordFact( + FactCollectionKey.from({ + contractAddress: contract2, + scope, + factCollectionTypeId, + factCollectionId: collectionId1, + }), + factTypeA, + [Fr.random()], + undefined, + JOB, + ); + await store.recordFact( + FactCollectionKey.from({ + contractAddress: contract, + scope, + factCollectionTypeId: type2, + factCollectionId: collectionId1, + }), + factTypeA, + [Fr.random()], + undefined, + JOB, + ); + await kv.transactionAsync(() => store.commit(JOB)); + + expect(await store.getFactCollectionsByType(typeKey, JOB)).toHaveLength(1); + expect( + await store.getFactCollectionsByType( + FactCollectionTypeKey.from({ contractAddress: contract2, scope, factCollectionTypeId }), + JOB, + ), + ).toHaveLength(1); + expect( + await store.getFactCollectionsByType( + FactCollectionTypeKey.from({ contractAddress: contract, scope, factCollectionTypeId: type2 }), + JOB, + ), + ).toHaveLength(1); + }); + }); + + describe('cross-job behavior', () => { + it("commit persists only the given job's facts", async () => { + const payloads = [Fr.random(), Fr.random()]; + await store.recordFact(collectionKey1, factTypeA, [payloads[0]], undefined, JOB); + const JOB2 = 'second-job'; + await store.recordFact(collectionKey1, factTypeA, [payloads[1]], undefined, JOB2); + await kv.transactionAsync(() => store.commit(JOB)); + + expect((await store.getFactCollection(collectionKey1, 'reader'))!.facts.map(f => f.payload[0])).toEqual([ + payloads[0], + ]); + expect(hexSet((await store.getFactCollection(collectionKey1, JOB2))!.facts.map(f => f.payload[0]))).toEqual( + hexSet(payloads), + ); + + await kv.transactionAsync(() => store.commit(JOB2)); + expect(hexSet((await store.getFactCollection(collectionKey1, 'reader'))!.facts.map(f => f.payload[0]))).toEqual( + hexSet(payloads), + ); + }); + + it('discardStaged drops staged writes without touching committed state', async () => { + await store.recordFact(collectionKey1, factTypeA, [Fr.random()], undefined, JOB); + await kv.transactionAsync(() => store.commit(JOB)); + + const JOB2 = 'discarded-job'; + await store.recordFact(collectionKey2, factTypeA, [Fr.random()], undefined, JOB2); + await store.recordFact(collectionKey1, factTypeB, [Fr.random()], undefined, JOB2); + await store.discardStaged(JOB2); + + expect(collectionIdsOf(await store.getFactCollectionsByType(typeKey, JOB2))).toEqual([collectionId1]); + await kv.transactionAsync(() => store.commit(JOB2)); + expect((await store.getFactCollection(collectionKey1, 'reader'))!.facts).toHaveLength(1); + expect(await store.getFactCollection(collectionKey2, 'reader')).toBeUndefined(); + }); + + it('a fact recorded by two jobs racing to the same collection dedups on commit', async () => { + const payload = Fr.random(); + const JOB2 = 'racing-job'; + await store.recordFact(collectionKey1, factTypeA, [payload], undefined, JOB); + await store.recordFact(collectionKey1, factTypeA, [payload], undefined, JOB2); + + await kv.transactionAsync(() => store.commit(JOB)); + await expect(kv.transactionAsync(() => store.commit(JOB2))).resolves.not.toThrow(); + + expect((await store.getFactCollection(collectionKey1, 'reader'))!.facts).toHaveLength(1); + }); + }); +}); diff --git a/yarn-project/pxe/src/storage/fact_store/fact_store.ts b/yarn-project/pxe/src/storage/fact_store/fact_store.ts new file mode 100644 index 000000000000..b212c07f790f --- /dev/null +++ b/yarn-project/pxe/src/storage/fact_store/fact_store.ts @@ -0,0 +1,487 @@ +import type { Fr } from '@aztec/foundation/curves/bn254'; +import { createLogger } from '@aztec/foundation/log'; +import { Semaphore } from '@aztec/foundation/queue'; +import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap } from '@aztec/kv-store'; + +import type { StagedStore } from '../../job_coordinator/job_coordinator.js'; +import { FactCollectionKey, type FactCollectionTypeKey, type OriginBlock } from './fact_store_keys.js'; +import { type Fact, StoredFact, factKeyStrOf } from './stored_fact.js'; + +type JobId = string; +type BlockNum = number; +type FactBuffer = Buffer; +type FactCollectionTypeKeyStr = string; +type FactCollectionKeyStr = string; +type FactKeyStr = string; + +/** A fact collection as returned by the store. */ +export type FactCollection = { key: FactCollectionKey; facts: Fact[] }; + +/** Internal auxiliary type assembling a collection. */ +type CollectionWithFacts = { key: FactCollectionKey; facts: Map }; + +/** A pending operation for a job: record a fact, or delete a fact collection. */ +type StagedOp = { kind: 'recordFact'; fact: StoredFact } | { kind: 'deleteFactCollection'; key: FactCollectionKey }; + +/** + * Stores immutable facts grouped into collections, isolated by contract and scope. + * + * A fact collection is a contract-defined bag of facts identified by a {@link FactCollectionKey} (contract, scope, + * collection type, and id). A fact is a contract-defined immutable, typed datum in a collection. Collections are + * implicit: one comes into being when its first fact is recorded and ceases to exist once it has no facts left. + * + * What makes this store different to, for example, the `CapsuleStore`, is that it is designed to support use cases + * where resilience to reorgs is needed, via what we call _retractability_. + * + * Facts can be retractable or non-retractable. They are retractable if they are associated to an origin block. + * Retractable facts are removed from the store when their origin block is pruned (typically due to a reorg). + * Non-retractable facts survive reorgs: they must then be explicitly deleted, so as not to keep consuming resources + * (storage and compute) indefinitely. + * + * Fact collections are isolated by scope. + * + * This store is designed to enable Aztec.nr to implement complex workflows such as offchain reception or partial note + * processing by storing structured data that is guaranteed to exist conditionally to specific blocks being included in + * the chain, while leaving the complexity of ensuring said guarantees to PXE. + * + * A key design driver is that PXE knows nothing about the actual fact contents: it just manages enough metadata to + * provide the guarantees mentioned above. That way, concepts such as offchain delivery or partial notes are completely + * defined by Aztec.nr, opening the door to further extension without the need for ad-hoc PXE support. + * + * As with most other PXE stores, writes are staged per-job and flushed atomically on commit. + */ +export class FactStore implements StagedStore { + readonly storeName: string = 'fact'; + + #store: AztecAsyncKVStore; + + /** Primary index of fact records. */ + #facts: AztecAsyncMap; + + /** Index for per-collection fact enumeration and by-type collection discovery. */ + #factsByCollection: AztecAsyncMultiMap; + + /** Index for delete-on-prune of retractable facts (those with an origin block). */ + #factsByBlock: AztecAsyncMultiMap; + + /** Job uncommitted data */ + #opsForJob: Map; + + /** Per-job locks */ + #jobLocks: Map; + + logger = createLogger('fact_store'); + + constructor(store: AztecAsyncKVStore) { + this.#store = store; + this.#facts = store.openMap('facts'); + this.#factsByCollection = store.openMultiMap('facts_by_collection'); + this.#factsByBlock = store.openMultiMap('facts_by_block'); + this.#opsForJob = new Map(); + this.#jobLocks = new Map(); + } + + /** + * Records a fact in a collection. + * + * The collection is created implicitly on the first fact recorded for its key: recording into an existing collection + * just adds to it. + * + * If `originBlock === undefined`, the fact is non-retractable: it survives reorgs. A defined origin block makes the + * fact retractable: on a prune below its block, it will be deleted. + * + * Idempotent: re-recording an identical fact (same collection, fact type, payload, and origin block) is a no-op. + * The same payload tied to a different origin block is a distinct fact. + */ + recordFact( + factCollectionKey: FactCollectionKey, + factTypeId: Fr, + payload: Fr[], + originBlock: OriginBlock | undefined, + jobId: string, + ): Promise { + return this.#withJobLock(jobId, () => { + this.#stagedOpsFor(jobId).push({ + kind: 'recordFact', + fact: new StoredFact(factCollectionKey, factTypeId, payload, originBlock), + }); + return Promise.resolve(); + }); + } + + /** + * Deletes a fact collection: removes every fact under the (scope-qualified) collection key. + * + * Idempotent: deleting a collection that does not exist is a no-op. + */ + deleteFactCollection(factCollectionKey: FactCollectionKey, jobId: string): Promise { + return this.#withJobLock(jobId, () => { + this.#stagedOpsFor(jobId).push({ kind: 'deleteFactCollection', key: factCollectionKey }); + return Promise.resolve(); + }); + } + + /** + * Returns the fact collection for the (scope-qualified) key, or undefined if it has no facts. + */ + async getFactCollection(factCollectionKey: FactCollectionKey, jobId: string): Promise { + const collectionKey = factCollectionKey.toString(); + const committed = await this.#store.transactionAsync(() => this.#readCollectionsFromDb([factCollectionKey])); + + const collection = this.#foldStagedOps(committed, jobId).get(collectionKey); + if (!collection) { + return undefined; + } + const facts = [...collection.facts.values()]; + return facts.length > 0 ? { key: factCollectionKey, facts } : undefined; + } + + /** + * Returns every fact collection of the given type for the queried scope, each holding its facts. + */ + async getFactCollectionsByType( + factCollectionTypeKey: FactCollectionTypeKey, + jobId: string, + ): Promise { + const typeKey = factCollectionTypeKey.toString(); + const committed = await this.#readCollectionsFromDbByType(typeKey); + + return Array.from(this.#foldStagedOps(committed, jobId, typeKey).values()) + .map(collection => ({ key: collection.key, facts: [...collection.facts.values()] })) + .filter(collection => collection.facts.length > 0); + } + + /** + * Commits all staged operations for the given job to persistent storage. + * + * Must be called inside a transaction owned by the caller (JobCoordinator wraps all commits in a single + * transactionAsync, and IndexedDB does not support nested transactions). + * + * DO NOT call `#withJobLock` here: awaiting the lock creates a microtask boundary that causes IndexedDB to + * auto-commit the outer transaction. + */ + async commit(jobId: string): Promise { + for (const op of this.#stagedOpsFor(jobId)) { + switch (op.kind) { + case 'recordFact': + await this.#commitFact(op.fact); + break; + case 'deleteFactCollection': + await this.#deleteCollection(op.key.toString()); + break; + default: { + const _exhaustive: never = op; + throw new Error(`Unhandled FactStore staged op kind: ${JSON.stringify(_exhaustive)}`); + } + } + } + this.#clearJobData(jobId); + } + + /** Discards all staged operations for the given job without persisting them. */ + discardStaged(jobId: string): Promise { + this.#clearJobData(jobId); + return Promise.resolve(); + } + + /** + * Removes every retractable fact originating from blocks over height `toBlock`, across all scopes. + * + * Non-retractable facts are untouched. Must run inside a caller-owned transaction (because it needs to share the + * transaction with other stores and IndexedDB has no nested transactions). + * + * Throws if any job is in flight (has accessed the store and not yet committed or discarded), since rolling back + * mid-job could re-introduce records originating from deleted blocks or change state underneath a job's view. + */ + async rollback(toBlock: BlockNum): Promise { + if (this.#opsForJob.size > 0) { + throw new Error('PXE fact store rollback is not allowed while jobs are running'); + } + + const removedFacts = await this.#retractFacts(toBlock); + + this.logger.verbose('rolled back fact store', { removedFacts, toBlock }); + } + + /** + * Deletes retractable facts originating above `toBlock`, returning the number of by-block entries scanned. + * Retraction is scope-agnostic: a reorg invalidates a fact regardless of which scope's collection it belongs to. + * + * Requires to be run in a transactionAsync context. + */ + async #retractFacts(toBlock: BlockNum): Promise { + // Snapshot the orphaned fact keys before mutating so we never delete from the cursor we are iterating, kicking off + // each fact-body read during the scan so a DB request stays pending across the cursor-to-delete boundary (a drained + // cursor with no read in flight would let the transaction auto-commit before the deletes). + const factReads = new Map>(); + for await (const [, factKey] of this.#factsByBlock.entriesAsync({ start: toBlock + 1 })) { + factReads.set(factKey, this.#facts.getAsync(factKey)); + } + await Promise.all( + Array.from(factReads, async ([factKey, read]) => { + const buf = await read; + if (!buf) { + // The by-block index just yielded this factKey, so a missing primary record means the indexes are out of + // sync. Log as this should be unreachable unless there is a bug. + this.logger.warn('Skipping retraction of a fact missing from the primary store: by-block index is stale', { + factKey, + }); + return; + } + await this.#deleteFact(factKey, StoredFact.fromBuffer(buf)); + }), + ); + return factReads.size; + } + + /** + * Reads the given collections (their facts) by key into a Map keyed by collection key, skipping + * keys with no committed facts. + * + * Reads are not wrapped in a transaction: the caller owns the transaction boundary. + */ + async #readCollectionsFromDb(keys: FactCollectionKey[]): Promise> { + const result = new Map(); + for (const key of keys) { + const collection = await this.#loadCommittedCollection(key); + if (collection) { + result.set(key.toString(), collection); + } + } + return result; + } + + #readCollectionsFromDbByType(typeKey: FactCollectionTypeKeyStr) { + return this.#store.transactionAsync(async () => { + const factReadsByCollection = new Map>>(); + for await (const [collectionKey, factKey] of this.#factsByCollection.entriesAsync({ + start: `${typeKey}:`, + end: `${typeKey};`, + })) { + let reads = factReadsByCollection.get(collectionKey); + if (!reads) { + reads = new Map>(); + factReadsByCollection.set(collectionKey, reads); + } + reads.set(factKey, this.#facts.getAsync(factKey)); + } + + const result = new Map(); + for (const [collectionKey, reads] of factReadsByCollection) { + const collection = await this.#assembleCollection(FactCollectionKey.fromString(collectionKey), reads); + result.set(collectionKey, collection); + } + return result; + }); + } + + async #loadCommittedCollection(collectionKey: FactCollectionKey): Promise { + // Kick off each fact read while iterating the index so a DB request is always pending. Draining the cursor and only + // then reading the facts one `await` at a time would let the IndexedDB transaction auto-commit at the boundary + // (IndexedDB auto-commits once control returns to the event loop with no pending request), throwing mid-read on the + // browser backend. + const factReads = new Map>(); + for await (const factKey of this.#factsByCollection.getValuesAsync(collectionKey.toString())) { + factReads.set(factKey, this.#facts.getAsync(factKey)); + } + if (factReads.size === 0) { + return undefined; + } + + return this.#assembleCollection(collectionKey, factReads); + } + + /** + * Assemble a collection's facts from a set of in-flight DB reads. + */ + async #assembleCollection( + collectionKey: FactCollectionKey, + reads: Map>, + ): Promise { + const factKeys = [...reads.keys()]; + const bufs = await Promise.all(reads.values()); + + // Await-free tail: deserialize. No DB ops from here on. + const facts = new Map(); + for (let i = 0; i < factKeys.length; i++) { + const factKey = factKeys[i]; + const buf = bufs[i]; + if (!buf) { + // Defensive: a #factsByCollection entry must always reference a live #facts entry. A missing one means the + // indexes are corrupt. + throw new Error(`Fact not found for factKey ${factKey}`); + } + const stored = StoredFact.fromBuffer(buf); + if (stored.factCollectionKey.toString() !== collectionKey.toString()) { + // Defensive: every read fact must belong to the collection being assembled. A mismatch means the indexes are + // corrupt. + throw new Error(`Fact ${factKey} does not belong to collection ${collectionKey}`); + } + facts.set(factKey, stored.toFact()); + } + return { key: collectionKey, facts }; + } + + /** + * Assembles the current view of a collection of collections together with their facts, combining the committed with + * staged data. + * + * When `typeKey` is given, staged records of other types are ignored, so the result holds only that type. + */ + #foldStagedOps( + committed: Map, + jobId: string, + typeKey?: FactCollectionTypeKeyStr, + ): Map { + const result = new Map(); + + // Copy to avoid mutating the contents of `committed`. + for (const [collectionKey, { key, facts }] of committed) { + result.set(collectionKey, { key, facts: new Map(facts) }); + } + for (const op of this.#stagedOpsFor(jobId)) { + switch (op.kind) { + case 'recordFact': + this.#foldRecordFact(result, op, typeKey); + break; + case 'deleteFactCollection': + this.#foldDeleteFactCollection(result, op); + break; + default: { + const _exhaustive: never = op; + throw new Error(`Unhandled FactStore staged op kind: ${JSON.stringify(_exhaustive)}`); + } + } + } + return result; + } + + /** + * Folds a staged `recordFact` op into view. + * + * When `typeKey` is given, an op of another type is ignored, keeping the view limited to that type. + */ + #foldRecordFact( + result: Map, + op: Extract, + typeKey?: FactCollectionTypeKeyStr, + ): void { + const key = op.fact.factCollectionKey; + + // Type filter doesn't match, nothing to do with this fact + if (typeKey !== undefined && key.factCollectionTypeKey().toString() !== typeKey) { + return; + } + + const collectionKey = key.toString(); + let collection = result.get(collectionKey); + + // Collection didn't exist before this point, the created fact brings it into existence + if (!collection) { + collection = { key, facts: new Map() }; + result.set(collectionKey, collection); + } + + const fKey = factKeyStrOf(op.fact); + if (!collection.facts.has(fKey)) { + collection.facts.set(fKey, op.fact.toFact()); + } + } + + /** + * Folds a staged `deleteFactCollection` op into the view: the scope-qualified collection is removed outright. + */ + #foldDeleteFactCollection( + result: Map, + op: Extract, + ): void { + result.delete(op.key.toString()); + } + + /** + * Writes a fact to persistent storage. Idempotent: an identical fact (same scope-qualified key) is left untouched. + */ + async #commitFact(fact: StoredFact): Promise { + const factKey = factKeyStrOf(fact); + if (await this.#facts.hasAsync(factKey)) { + this.logger.debug(`Ignoring already recorded fact`, { factKey }); + return; + } + await this.#facts.set(factKey, fact.toBuffer()); + await this.#factsByCollection.set(fact.factCollectionKey.toString(), factKey); + if (fact.originBlock !== undefined) { + await this.#factsByBlock.set(fact.originBlock.blockNumber, factKey); + } + } + + /** + * Deletes every fact under the (scope-qualified) collection key. + * + * Caller must wrap in a transaction. + */ + async #deleteCollection(collectionKey: FactCollectionKeyStr): Promise { + // Snapshot the fact index before mutating so we never delete from the cursor we are iterating, kicking off each + // fact-body read during the scan so a DB request stays pending across the cursor-to-mutation boundary (a drained + // cursor with no read in flight would let the transaction auto-commit before the deletes). + const factReads = new Map>(); + for await (const factKey of this.#factsByCollection.getValuesAsync(collectionKey)) { + factReads.set(factKey, this.#facts.getAsync(factKey)); + } + await Promise.all( + Array.from(factReads, async ([factKey, read]) => { + const buf = await read; + if (!buf) { + // A #factsByCollection entry must always reference a live #facts entry, a missing one means the indexes are + // corrupt. + throw new Error(`Fact not found for factKey ${factKey}`); + } + await this.#deleteFact(factKey, StoredFact.fromBuffer(buf)); + }), + ); + } + + /** + * Deletes a fact from the primary store and all its indexes (`#factsByCollection`, plus `#factsByBlock` if + * retractable). + * + * Caller must wrap in a transaction. + */ + async #deleteFact(factKey: FactKeyStr, fact: StoredFact): Promise { + await this.#facts.delete(factKey); + await this.#factsByCollection.deleteValue(fact.factCollectionKey.toString(), factKey); + if (fact.originBlock !== undefined) { + await this.#factsByBlock.deleteValue(fact.originBlock.blockNumber, factKey); + } + } + + /** + * Returns the job's staged-ops array, creating it on first access. + * */ + #stagedOpsFor(jobId: string): StagedOp[] { + let ops = this.#opsForJob.get(jobId); + if (ops === undefined) { + ops = []; + this.#opsForJob.set(jobId, ops); + } + return ops; + } + + #clearJobData(jobId: string) { + this.#opsForJob.delete(jobId); + this.#jobLocks.delete(jobId); + } + + async #withJobLock(jobId: string, fn: () => Promise): Promise { + let lock = this.#jobLocks.get(jobId); + if (!lock) { + lock = new Semaphore(1); + this.#jobLocks.set(jobId, lock); + } + await lock.acquire(); + try { + return await fn(); + } finally { + lock.release(); + } + } +} diff --git a/yarn-project/pxe/src/storage/fact_store/fact_store_keys.test.ts b/yarn-project/pxe/src/storage/fact_store/fact_store_keys.test.ts new file mode 100644 index 000000000000..2c2e656f3045 --- /dev/null +++ b/yarn-project/pxe/src/storage/fact_store/fact_store_keys.test.ts @@ -0,0 +1,43 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import { FactCollectionKey, FactCollectionTypeKey } from './fact_store_keys.js'; + +describe('fact store keys', () => { + const contract = AztecAddress.fromBigInt(100n); + const scope = AztecAddress.fromBigInt(1n); + const type = new Fr(7n); + const id = new Fr(42n); + + const collectionKey = () => + FactCollectionKey.from({ contractAddress: contract, scope, factCollectionTypeId: type, factCollectionId: id }); + + it('encodes scope between contract and type in the collection key', () => { + expect(collectionKey().toString()).toBe(`${contract}:${scope}:${type}:${id}`); + }); + + it('encodes scope in the type key and carries it through factCollectionTypeKey()', () => { + expect(collectionKey().factCollectionTypeKey().toString()).toBe(`${contract}:${scope}:${type}`); + expect( + FactCollectionTypeKey.from({ contractAddress: contract, scope, factCollectionTypeId: type }).toString(), + ).toBe(`${contract}:${scope}:${type}`); + }); + + it('round-trips a collection key through fromString', () => { + expect(FactCollectionKey.fromString(collectionKey().toString())).toEqual(collectionKey()); + }); + + it('rejects the zero address as scope', () => { + expect(() => + FactCollectionKey.from({ + contractAddress: contract, + scope: AztecAddress.ZERO, + factCollectionTypeId: type, + factCollectionId: id, + }), + ).toThrow('scope must not be the zero address'); + expect(() => + FactCollectionTypeKey.from({ contractAddress: contract, scope: AztecAddress.ZERO, factCollectionTypeId: type }), + ).toThrow('scope must not be the zero address'); + }); +}); diff --git a/yarn-project/pxe/src/storage/fact_store/fact_store_keys.ts b/yarn-project/pxe/src/storage/fact_store/fact_store_keys.ts new file mode 100644 index 000000000000..7126ab43b6b4 --- /dev/null +++ b/yarn-project/pxe/src/storage/fact_store/fact_store_keys.ts @@ -0,0 +1,75 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import type { FieldsOf } from '@aztec/foundation/types'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +/** + * The block a retractable fact originates from. + */ +export type OriginBlock = { blockNumber: number; blockHash: Fr }; + +/** Facts are always tied to a real scope; the zero address is not a valid fact scope. */ +function assertNonZeroScope(scope: AztecAddress): void { + if (scope.equals(AztecAddress.ZERO)) { + throw new Error('scope must not be the zero address'); + } +} + +/** Identifies all fact collections of one type within a contract, for one scope. */ +export class FactCollectionTypeKey { + constructor( + public readonly contractAddress: AztecAddress, + public readonly scope: AztecAddress, + public readonly factCollectionTypeId: Fr, + ) { + assertNonZeroScope(scope); + } + + static from(fields: FieldsOf): FactCollectionTypeKey { + return new FactCollectionTypeKey(fields.contractAddress, fields.scope, fields.factCollectionTypeId); + } + + toString(): string { + return `${this.contractAddress}:${this.scope}:${this.factCollectionTypeId}`; + } +} + +/** Uniquely identifies a single fact collection, isolated by scope; all its facts share this key. */ +export class FactCollectionKey { + constructor( + public readonly contractAddress: AztecAddress, + public readonly scope: AztecAddress, + public readonly factCollectionTypeId: Fr, + public readonly factCollectionId: Fr, + ) { + assertNonZeroScope(scope); + } + + static from(fields: FieldsOf): FactCollectionKey { + return new FactCollectionKey( + fields.contractAddress, + fields.scope, + fields.factCollectionTypeId, + fields.factCollectionId, + ); + } + + /** Inverse of toString */ + static fromString(str: string): FactCollectionKey { + const [contractAddress, scope, factCollectionTypeId, factCollectionId] = str.split(':'); + return new FactCollectionKey( + AztecAddress.fromString(contractAddress), + AztecAddress.fromString(scope), + Fr.fromString(factCollectionTypeId), + Fr.fromString(factCollectionId), + ); + } + + /** The key grouping this collection with the other collections of its type within the same contract and scope. */ + factCollectionTypeKey(): FactCollectionTypeKey { + return new FactCollectionTypeKey(this.contractAddress, this.scope, this.factCollectionTypeId); + } + + toString(): string { + return `${this.contractAddress}:${this.scope}:${this.factCollectionTypeId}:${this.factCollectionId}`; + } +} diff --git a/yarn-project/pxe/src/storage/fact_store/index.ts b/yarn-project/pxe/src/storage/fact_store/index.ts new file mode 100644 index 000000000000..c9ec479aa6c3 --- /dev/null +++ b/yarn-project/pxe/src/storage/fact_store/index.ts @@ -0,0 +1,3 @@ +export { FactStore, type FactCollection } from './fact_store.js'; +export { FactCollectionKey, FactCollectionTypeKey, type OriginBlock } from './fact_store_keys.js'; +export type { Fact } from './stored_fact.js'; diff --git a/yarn-project/pxe/src/storage/fact_store/stored_fact.test.ts b/yarn-project/pxe/src/storage/fact_store/stored_fact.test.ts new file mode 100644 index 000000000000..c514229a6576 --- /dev/null +++ b/yarn-project/pxe/src/storage/fact_store/stored_fact.test.ts @@ -0,0 +1,69 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import { FactCollectionKey } from './fact_store_keys.js'; +import { StoredFact, factKeyStrOf } from './stored_fact.js'; + +describe('StoredFact', () => { + const contract = AztecAddress.fromBigInt(100n); + const scope = AztecAddress.fromBigInt(1n); + const collectionType = new Fr(7n); + const collectionId = new Fr(42n); + const factType = new Fr(3n); + const key = FactCollectionKey.from({ + contractAddress: contract, + scope, + factCollectionTypeId: collectionType, + factCollectionId: collectionId, + }); + + it('round-trips a retractable fact through buffer serialization', () => { + const fact = new StoredFact(key, factType, [new Fr(9n), new Fr(10n)], { + blockNumber: 12, + blockHash: new Fr(0xabcn), + }); + const back = StoredFact.fromBuffer(fact.toBuffer()); + expect(back).toEqual(fact); + expect(back.isRetractable).toBe(true); + }); + + it('round-trips a non-retractable fact (no origin block)', () => { + const fact = new StoredFact(key, factType, [new Fr(9n)], undefined); + const back = StoredFact.fromBuffer(fact.toBuffer()); + expect(back).toEqual(fact); + expect(back.isRetractable).toBe(false); + }); + + it('derives stable composite keys including the origin block', () => { + const nonRetractable = new StoredFact(key, factType, [new Fr(9n)], undefined); + expect(nonRetractable.factCollectionKey.factCollectionTypeKey().toString()).toBe( + `${contract}:${scope}:${collectionType}`, + ); + expect(nonRetractable.factCollectionKey.toString()).toBe(`${contract}:${scope}:${collectionType}:${collectionId}`); + expect(factKeyStrOf(nonRetractable)).toBe( + nonRetractable.factCollectionKey.toString() + `:${factType}:${nonRetractable.payloadHash()}:none`, + ); + + const blockHash = new Fr(0xabcn); + const retractable = new StoredFact(key, factType, [new Fr(9n)], { blockNumber: 5, blockHash }); + expect(factKeyStrOf(retractable)).toBe( + retractable.factCollectionKey.toString() + `:${factType}:${retractable.payloadHash()}:5:${blockHash}`, + ); + }); + + it('keys the same payload at different origin blocks as distinct facts', () => { + const noOrigin = new StoredFact(key, factType, [new Fr(9n)], undefined); + const atBlock5 = new StoredFact(key, factType, [new Fr(9n)], { blockNumber: 5, blockHash: new Fr(1n) }); + const atBlock10 = new StoredFact(key, factType, [new Fr(9n)], { blockNumber: 10, blockHash: new Fr(2n) }); + expect(factKeyStrOf(noOrigin)).not.toBe(factKeyStrOf(atBlock5)); + expect(factKeyStrOf(atBlock5)).not.toBe(factKeyStrOf(atBlock10)); + }); + + it('derives distinct payload hashes for distinct payloads', () => { + const a = new StoredFact(key, factType, [new Fr(1n)], undefined); + const b = new StoredFact(key, factType, [new Fr(2n)], undefined); + const c = new StoredFact(key, factType, [new Fr(1n)], undefined); + expect(a.payloadHash()).not.toEqual(b.payloadHash()); + expect(a.payloadHash()).toEqual(c.payloadHash()); + }); +}); diff --git a/yarn-project/pxe/src/storage/fact_store/stored_fact.ts b/yarn-project/pxe/src/storage/fact_store/stored_fact.ts new file mode 100644 index 000000000000..f3572e8ef3f0 --- /dev/null +++ b/yarn-project/pxe/src/storage/fact_store/stored_fact.ts @@ -0,0 +1,80 @@ +import { sha256ToField } from '@aztec/foundation/crypto/sha256'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import { FactCollectionKey, type OriginBlock } from './fact_store_keys.js'; + +/** A fact as returned by the fact store. */ +export type Fact = { factTypeId: Fr; payload: Fr[]; originBlock: OriginBlock | undefined }; + +/** + * A single immutable fact belonging to a fact collection. + */ +export class StoredFact { + constructor( + public readonly factCollectionKey: FactCollectionKey, + public readonly factTypeId: Fr, + public readonly payload: Fr[], + public readonly originBlock: OriginBlock | undefined, + ) {} + + /** Whether this fact is deleted on block pruning (true) or survives reorgs (false). */ + get isRetractable(): boolean { + return this.originBlock !== undefined; + } + + /** Stable digest of the payload, used for fact idempotency. */ + payloadHash(): Fr { + return sha256ToField([this.payload.length, ...this.payload]); + } + + /** Returns the externally facing view of this fact. */ + toFact(): Fact { + return { factTypeId: this.factTypeId, payload: this.payload, originBlock: this.originBlock }; + } + + toBuffer(): Buffer { + return serializeToBuffer( + this.factCollectionKey.contractAddress, + this.factCollectionKey.scope, + this.factCollectionKey.factCollectionTypeId, + this.factCollectionKey.factCollectionId, + this.factTypeId, + this.payload.length, + ...this.payload, + this.originBlock !== undefined, + this.originBlock ? this.originBlock.blockNumber : 0, + this.originBlock ? this.originBlock.blockHash : Fr.ZERO, + ); + } + + static fromBuffer(buffer: Buffer | BufferReader): StoredFact { + const reader = BufferReader.asReader(buffer); + const contractAddress = reader.readObject(AztecAddress); + const scope = reader.readObject(AztecAddress); + const factCollectionTypeId = reader.readObject(Fr); + const factCollectionId = reader.readObject(Fr); + const factTypeId = reader.readObject(Fr); + const payloadLen = reader.readNumber(); + const payload = reader.readArray(payloadLen, Fr); + const hasOriginBlock = reader.readBoolean(); + const blockNumber = reader.readNumber(); + const blockHash = reader.readObject(Fr); + const originBlock = hasOriginBlock ? { blockNumber, blockHash } : undefined; + return new StoredFact( + new FactCollectionKey(contractAddress, scope, factCollectionTypeId, factCollectionId), + factTypeId, + [...payload], + originBlock, + ); + } +} + +/** + * Builds the serialized key that identifies a fact in the store. + */ +export function factKeyStrOf(fact: StoredFact): string { + const origin = fact.originBlock ? `${fact.originBlock.blockNumber}:${fact.originBlock.blockHash}` : 'none'; + return `${fact.factCollectionKey}:${fact.factTypeId}:${fact.payloadHash()}:${origin}`; +} diff --git a/yarn-project/pxe/src/storage/index.ts b/yarn-project/pxe/src/storage/index.ts index 47c5b840f7ee..1ac82688c9fe 100644 --- a/yarn-project/pxe/src/storage/index.ts +++ b/yarn-project/pxe/src/storage/index.ts @@ -2,6 +2,7 @@ export * from './address_store/index.js'; export * from './anchor_block_store/index.js'; export * from './capsule_store/index.js'; export * from './contract_store/index.js'; +export * from './fact_store/index.js'; export * from './note_store/index.js'; export * from './tagging_store/index.js'; export * from './metadata.js'; diff --git a/yarn-project/pxe/src/storage/metadata.ts b/yarn-project/pxe/src/storage/metadata.ts index e242d8c59327..44387e7a5b2f 100644 --- a/yarn-project/pxe/src/storage/metadata.ts +++ b/yarn-project/pxe/src/storage/metadata.ts @@ -1 +1 @@ -export const PXE_DATA_SCHEMA_VERSION = 8; +export const PXE_DATA_SCHEMA_VERSION = 9; diff --git a/yarn-project/pxe/src/storage/open_pxe_stores.ts b/yarn-project/pxe/src/storage/open_pxe_stores.ts index 573b85a40984..a38aadac6022 100644 --- a/yarn-project/pxe/src/storage/open_pxe_stores.ts +++ b/yarn-project/pxe/src/storage/open_pxe_stores.ts @@ -7,6 +7,7 @@ import { AddressStore } from './address_store/address_store.js'; import { AnchorBlockStore } from './anchor_block_store/anchor_block_store.js'; import { CapsuleStore } from './capsule_store/capsule_store.js'; import { ContractStore } from './contract_store/contract_store.js'; +import { FactStore } from './fact_store/fact_store.js'; import { NoteStore } from './note_store/note_store.js'; import { PrivateEventStore } from './private_event_store/private_event_store.js'; import { RecipientTaggingStore, SenderAddressBookStore, SenderTaggingStore } from './tagging_store/index.js'; @@ -26,6 +27,7 @@ export type PxeStores = { capsuleStore: CapsuleStore; keyStore: KeyStore; l2TipsStore: L2TipsKVStore; + factStore: FactStore; }; /** @@ -45,5 +47,6 @@ export function openPxeStores(store: AztecAsyncKVStore, initialBlockHash: BlockH capsuleStore: new CapsuleStore(store), keyStore: new KeyStore(store), l2TipsStore: new L2TipsKVStore(store, 'pxe', initialBlockHash), + factStore: new FactStore(store), }; }