diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts new file mode 100644 index 000000000000..9ab521186a71 --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts @@ -0,0 +1,365 @@ +import type { Archiver } from '@aztec/archiver'; +import type { AztecNodeService } from '@aztec/aztec-node'; +import { EthAddress } from '@aztec/aztec.js/addresses'; +import { Fr } from '@aztec/aztec.js/fields'; +import type { Logger } from '@aztec/aztec.js/log'; +import { waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils'; +import { asyncMap } from '@aztec/foundation/async-map'; +import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { times } from '@aztec/foundation/collection'; +import { SecretValue } from '@aztec/foundation/config'; +import { retryUntil } from '@aztec/foundation/retry'; +import { bufferToHex } from '@aztec/foundation/string'; +import { timeoutPromise } from '@aztec/foundation/timer'; +import { type L2Block, L2BlockSourceEvents, type L2Tips } from '@aztec/stdlib/block'; +import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; + +import { jest } from '@jest/globals'; +import { privateKeyToAccount } from 'viem/accounts'; + +import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; +import { EpochsTestContext } from './epochs_test.js'; + +jest.setTimeout(1000 * 60 * 15); + +const NODE_COUNT = 4; + +/** + * E2E test for the "missed L1 publish" scenario under proposer pipelining. + * + * Each of 4 nodes holds exactly one validator key. We pick four consecutive slots + * (slotZero, slotOne, slotTwo, slotThree) such that the proposers for slotOne, slotTwo, and + * slotThree are three distinct validators, then warp to one L1 block before slotZero begins. + * The proposer for slotOne is configured to skip its L1 publish. + * + * With pipelining, the proposer for slot N+1 builds and gossips its checkpoint during slot N, + * then publishes that checkpoint to L1 during slot N+1. So gossip-driven `proposed` chain + * advances arrive one slot earlier than the L1-driven `checkpointed` advance. + * + * Expected behavior: + * - During slotZero, the pipelined proposer for slotOne gossips its build → every node's + * `proposed` tip advances to a block at slotOne. + * - During slotOne, the pipelined proposer for slotTwo gossips on top of the slotOne proposal → + * `proposed` advances to a block at slotTwo. Meanwhile the proposer for slotOne attempts L1 + * publish but is configured to skip it, so no checkpoint lands. + * - When slotOne ends with no checkpoint mined, every node's archiver prunes the + * uncheckpointed slotOne and slotTwo blocks; we verify rollback via the prune event. + * We then re-enable publishing on the formerly suppressed node so recovery can proceed. + * - During slotTwo, the pipelined proposer for slotThree builds on top of the (now genesis) + * checkpointed tip → `proposed` advances again. + * - During slotThree, that pipelined work is published → `checkpointed` finally advances. + */ +describe('e2e_epochs/epochs_missed_l1_publish', () => { + let logger: Logger; + let test: EpochsTestContext; + let nodes: AztecNodeService[]; + + afterEach(async () => { + jest.restoreAllMocks(); + await test?.teardown(); + }); + + it('all nodes prune and recover when proposer fails to publish to L1', async () => { + // Build 4 distinct validators (V1..V4). One key per node, no overlap. + const validators = times(NODE_COUNT, i => { + const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); + const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); + return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; + }); + + test = await EpochsTestContext.setup({ + numberOfAccounts: 0, + initialValidators: validators, + enableProposerPipelining: true, + inboxLag: 2, + mockGossipSubNetwork: true, + disableAnvilTestWatcher: true, + startProverNode: false, + aztecEpochDuration: 4, + aztecProofSubmissionEpochs: 1024, + enforceTimeTable: true, + ethereumSlotDuration: 6, + aztecSlotDuration: 36, + blockDurationMs: 8000, + attestationPropagationTime: 0.5, + l1PublishingTime: 2, + aztecTargetCommitteeSize: NODE_COUNT, + skipInitialSequencer: true, + }); + + logger = test.logger; + + // One node per validator. dontStartSequencer until after the warp so timing is deterministic. + nodes = await asyncMap(validators, ({ privateKey }, i) => + test.createValidatorNode([privateKey], { + dontStartSequencer: true, + coinbase: EthAddress.fromNumber(0xa + i), + buildCheckpointIfEmpty: true, + minTxsPerBlock: 0, + }), + ); + + const attesterAddresses = validators.map(v => v.attester); + logger.warn('Validator nodes created', { + validators: attesterAddresses.map((a, i) => ({ idx: i, attester: a.toString() })), + }); + + // Find slotOne (>=4 ahead) such that proposers for slotOne, slotTwo, slotThree are three + // distinct validators. The +4 margin (vs +2 in equivocation) gives the warp+sequencer-start + // path enough headroom to reach the build window for slotZero even if node creation jitters. + const { slot: currentSlot } = test.epochCache.getEpochAndSlotNow(); + const scanStart = currentSlot + 4; + const scanEnd = currentSlot + 60; + let slotOne: SlotNumber | undefined; + let proposerOne: EthAddress | undefined; + let proposerTwo: EthAddress | undefined; + let proposerThree: EthAddress | undefined; + for (let candidate = scanStart; candidate <= scanEnd; candidate++) { + const [p1, p2, p3] = await Promise.all([ + test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate)), + test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate + 1)), + test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate + 2)), + ]); + if (p1 && p2 && p3 && !p1.equals(p2) && !p1.equals(p3) && !p2.equals(p3)) { + slotOne = SlotNumber(candidate); + proposerOne = p1; + proposerTwo = p2; + proposerThree = p3; + break; + } + } + if (slotOne === undefined || !proposerOne || !proposerTwo || !proposerThree) { + throw new Error(`Could not find a slot in [${scanStart}, ${scanEnd}] with three distinct consecutive proposers`); + } + + const slotZero = SlotNumber(slotOne - 1); + const slotTwo = SlotNumber(slotOne + 1); + const slotThree = SlotNumber(slotOne + 2); + + const proposerOneNodeIndex = validators.findIndex(v => v.attester.equals(proposerOne!)); + if (proposerOneNodeIndex < 0) { + throw new Error(`No node holds the key for proposer ${proposerOne}`); + } + + logger.warn(`Selected target slotOne=${slotOne}`, { + slotOne, + slotZero, + slotTwo, + slotThree, + proposerOne: proposerOne.toString(), + proposerOneNodeIndex, + proposerTwo: proposerTwo.toString(), + proposerThree: proposerThree.toString(), + }); + + // Prevent the proposer for slotOne from publishing the checkpoint to L1 (build & gossip still happen). + await nodes[proposerOneNodeIndex].setConfig({ skipPublishingCheckpointsPercent: 100 }); + + // Subscribe to the prune event on every node before sequencers start, so we never miss it. + // We capture the L2 tips synchronously inside the handler — the archiver has already removed + // the pruned blocks at emit time, so this snapshot reflects the rolled-back state before any + // new pipelined block can be applied. + type PruneObservation = { slotNumber: SlotNumber; blocks: L2Block[]; tipsAtPrune: L2Tips }; + const prunePromises: Promise[] = nodes.map( + (node, idx) => + new Promise(resolve => { + const archiver = node.getBlockSource() as Archiver; + // eslint-disable-next-line @typescript-eslint/no-misused-promises + archiver.events.once(L2BlockSourceEvents.L2PruneUncheckpointed, async ev => { + const tipsAtPrune = await node.getL2Tips(); + logger.warn(`Node ${idx} pruned uncheckpointed blocks`, { + slotNumber: ev.slotNumber, + blocks: ev.blocks.map(b => ({ number: b.number, slot: b.header.globalVariables.slotNumber })), + tipsAtPrune, + }); + resolve({ slotNumber: ev.slotNumber, blocks: ev.blocks, tipsAtPrune }); + }); + }), + ); + + // Warp L1 to one L1 block before slotZero begins. Pipelining will then engage during slotZero. + const slotZeroStart = getTimestampForSlot(slotZero, test.constants); + const warpTo = slotZeroStart - BigInt(test.L1_BLOCK_TIME_IN_S); + logger.warn(`Warping L1 to timestamp ${warpTo} (one L1 block before slot ${slotZero})`); + await test.context.cheatCodes.eth.warp(Number(warpTo), { resetBlockInterval: true }); + + // Check that the chain is empty + const node = nodes[0]; + const blockNumber = await node.getBlockNumber(); + expect(blockNumber).toEqual(0); + + // Start all sequencers. + const sequencers = nodes.map(n => n.getSequencer()!); + const { failEvents } = test.watchSequencerEvents(sequencers, i => ({ validator: `V${i + 1}` })); + + // Subscribe to the proposerTwo pipelined-discard event — this is the most direct signal + // that the pipelined slotTwo work was correctly thrown away because parent slotOne did not land. + const proposerTwoNodeIndex = validators.findIndex(v => v.attester.equals(proposerTwo!)); + const pipelinedDiscardEvents: { slot: SlotNumber; checkpointNumber: number; reason: string }[] = []; + sequencers[proposerTwoNodeIndex].getSequencer().on('pipelined-checkpoint-discarded', args => { + pipelinedDiscardEvents.push({ slot: args.slot, checkpointNumber: args.checkpointNumber, reason: args.reason }); + logger.warn(`proposerTwo (node ${proposerTwoNodeIndex}) discarded pipelined work`, args); + }); + + await Promise.all(sequencers.map(s => s.start())); + logger.warn('All sequencers started'); + + const slotAdvanceTimeout = test.L2_SLOT_DURATION_IN_S * 3; + + // (1) During slotZero: the pipelined proposer for slotOne broadcasts. Every node sees a proposed block at slotOne. + logger.warn(`Waiting for proposed chain to reach slot ${slotOne} on all nodes (build during slotZero)`); + await Promise.all( + nodes.map((node, idx) => + retryUntil( + async () => { + const tips = await node.getL2Tips(); + if (tips.proposed.number === 0) { + return false; + } + const block = await node.getBlock(tips.proposed.number); + return !!block && block.header.globalVariables.slotNumber === slotOne; + }, + `node ${idx} proposed advanced to slot ${slotOne}`, + slotAdvanceTimeout, + 0.5, + ), + ), + ); + + // (2) During slotOne: the pipelined proposer for slotTwo broadcasts on top of slotOne → proposed reaches slotTwo. + logger.warn(`Waiting for proposed chain to reach slot ${slotTwo} on all nodes (build during slotOne)`); + await Promise.all( + nodes.map((node, idx) => + retryUntil( + async () => { + const tips = await node.getL2Tips(); + if (tips.proposed.number === 0) { + return false; + } + const block = await node.getBlock(tips.proposed.number); + return !!block && block.header.globalVariables.slotNumber === slotTwo; + }, + `node ${idx} proposed advanced to slot ${slotTwo}`, + slotAdvanceTimeout, + 0.5, + ), + ), + ); + + // (3) Wait until slotOne has fully ended on L1 — the archiver only prunes once slotAtNextL1Block > slotOne. + // The end-of-slotOne timestamp equals the start-of-slotTwo timestamp. + const slotOneEndTimestamp = getTimestampForSlot(slotTwo, test.constants); + logger.warn(`Waiting until L1 timestamp ${slotOneEndTimestamp} (end of slot ${slotOne})`); + await waitUntilL1Timestamp(test.l1Client, slotOneEndTimestamp, undefined, test.L2_SLOT_DURATION_IN_S * 3); + + // (4) After slotOne ends without a checkpoint, all nodes should prune. + // Verify rollback via the prune event itself: the pruned slot must equal slotOne, and the + // pruned blocks must include the broadcast blocks for slotOne (proposerOne) and slotTwo + // (pipelined proposerTwo, whose work is now invalid because parent slotOne did not land). + logger.warn('Waiting for L2PruneUncheckpointed on every node'); + const pruneTimeoutMs = test.L2_SLOT_DURATION_IN_S * 2 * 1000; + const pruneObservations = await Promise.all( + prunePromises.map((p, idx) => + Promise.race([p, timeoutPromise(pruneTimeoutMs, `Node ${idx} did not emit prune event in time`)]), + ), + ); + + logger.warn('Asserting prune event details on every node'); + for (const [idx, obs] of pruneObservations.entries()) { + expect({ idx, slotNumber: obs.slotNumber }).toEqual({ idx, slotNumber: slotOne }); + // proposerOne broadcasts during slotZero, so its block must always be in the pruned set. + // The pipelined slotTwo broadcast may or may not have arrived in time on every node, so + // we don't strictly require it here. + const prunedSlots = obs.blocks.map(b => b.header.globalVariables.slotNumber); + expect(prunedSlots).toContain(slotOne); + } + + // (5) Allow the formerly suppressed node to publish again so the chain can recover. + logger.warn(`Re-enabling checkpoint publishing on node ${proposerOneNodeIndex}`); + await nodes[proposerOneNodeIndex].setConfig({ skipPublishingCheckpointsPercent: 0 }); + + // (6) During slotTwo: the pipelined proposer for slotThree builds and broadcasts → proposed advances again. + // The chain must have rewound past slotOne and slotTwo and now build on whatever was + // checkpointed before slotZero — genesis, in this test, since no checkpoints have landed yet. + const postPruneProposedNumbers = pruneObservations.map(o => o.tipsAtPrune.proposed.number); + expect(postPruneProposedNumbers[0]).toBe(0); + + logger.warn(`Waiting for proposed chain to advance to slot ${slotThree} on all nodes (build during slotTwo)`); + await Promise.all( + nodes.map((node, idx) => + retryUntil( + async () => { + const tips = await node.getL2Tips(); + if (tips.proposed.number === 0) { + return false; + } + const block = await node.getBlock(tips.proposed.number); + return !!block && block.header.globalVariables.slotNumber >= slotThree; + }, + `node ${idx} proposed advanced to slot >= ${slotThree}`, + slotAdvanceTimeout, + 0.5, + ), + ), + ); + + // The first block in the chain after the prune must be the slotThree block — there should be + // nothing between genesis and the new pipelined work, since slotOne and slotTwo were pruned. + for (const node of nodes) { + const blocks = await node.getBlocks(BlockNumber(1), 50); + const firstSlotThreeIdx = blocks.findIndex(b => b.header.globalVariables.slotNumber === slotThree); + expect(firstSlotThreeIdx).toEqual(0); + } + + // (7) During slotThree: proposerThree publishes → checkpointed advances on every node. + logger.warn(`Waiting for checkpointed chain to reach slot >= ${slotThree} on all nodes`); + await Promise.all( + nodes.map((node, idx) => + retryUntil( + async () => { + const tips = await node.getL2Tips(); + if (tips.checkpointed.checkpoint.number === 0) { + return false; + } + const block = await node.getBlock(tips.checkpointed.block.number); + return ( + !!block && block.header.globalVariables.slotNumber >= slotThree && tips.checkpointed.block.number > 0 + ); + }, + `node ${idx} checkpointed advanced to slot >= ${slotThree}`, + slotAdvanceTimeout, + 0.5, + ), + ), + ); + + // Sanity: the only fail events we tolerate are the deliberate skip-publish on the suppressed + // node for slotOne, the pipelined-discard knock-on from proposerTwo (its parent slotOne + // never landed), and proposer-rollup-check noise that any non-proposer emits when the rollup + // contract rejects them. + const unexpectedFailEvents = failEvents.filter(e => { + if ( + e.type === 'checkpoint-publish-failed' && + e.sequencerIndex === proposerOneNodeIndex + 2 && + e.slot === slotOne + ) { + return false; + } + if ( + e.type === 'checkpoint-publish-failed' && + e.sequencerIndex === proposerTwoNodeIndex + 2 && + e.slot === slotTwo + ) { + return false; + } + if (e.type === 'proposer-rollup-check-failed') { + return false; + } + return true; + }); + if (unexpectedFailEvents.length > 0) { + logger.error('Unexpected fail events from sequencers', unexpectedFailEvents); + } + expect(unexpectedFailEvents).toEqual([]); + }); +});