From 03ffbc3b977d95f2250eb61be9ac6f96c56f067a Mon Sep 17 00:00:00 2001 From: aminsammara Date: Wed, 22 Jul 2026 21:53:51 +0000 Subject: [PATCH 1/2] feat(slasher): warn and expose metrics when own validators are slash targets The node now detects when its own validators are targeted by onchain slashing, at two points in the lifecycle: - On VoteCast events, it decodes the vote and emits a WARN (once per round and validator) plus an aztec.slasher.own_validator.targeted_count metric, early enough to react within the execution delay / veto window. - On RoundExecuted, it filters the Slashed events to own validators and emits a WARN plus aztec.slasher.own_validator.slashed_count and slashed_amount metrics. Metrics are labelled per validator address so nodes running multiple validators get per-validator series, seeded to zero at startup. Nodes running no validators skip the VoteCast subscription entirely. Fixes A-1443 --- .../src/contracts/slashing_proposer.ts | 9 +- .../slasher/src/factory/create_facade.ts | 6 +- .../src/factory/create_implementation.ts | 5 + yarn-project/slasher/src/metrics.ts | 41 ++++- .../slasher/src/slasher_client.test.ts | 147 +++++++++++++++++- yarn-project/slasher/src/slasher_client.ts | 63 +++++++- .../slasher/src/slasher_client_facade.ts | 3 + yarn-project/telemetry-client/src/metrics.ts | 17 ++ 8 files changed, 280 insertions(+), 11 deletions(-) diff --git a/yarn-project/ethereum/src/contracts/slashing_proposer.ts b/yarn-project/ethereum/src/contracts/slashing_proposer.ts index c4939573bf1..de44d1df0bd 100644 --- a/yarn-project/ethereum/src/contracts/slashing_proposer.ts +++ b/yarn-project/ethereum/src/contracts/slashing_proposer.ts @@ -234,13 +234,16 @@ export class SlashingProposerContract { }; } - /** Returns the last vote emitted for a given round */ - public async getLastVote(round: bigint) { + /** + * Returns the last vote emitted for a given round + * @param slashingAmounts - The slash amount per vote unit, to avoid re-reading them from the contract + */ + public async getLastVote(round: bigint, slashingAmounts?: [bigint, bigint, bigint]) { const { voteCount } = await this.getRound(round); const validators = (await this.contract.simulate.getSlashTargetCommittees([round])).result.flat(); const vote = await this.contract.read.getVotes([round, voteCount - 1n]); const decoded = decodeSlashConsensusVotes(hexToBuffer(vote)); - const slashAmounts = await this.getSlashingAmounts(); + const slashAmounts = slashingAmounts ?? (await this.getSlashingAmounts()); return decoded .map((units, i) => ({ validator: EthAddress.fromString(validators[i]), diff --git a/yarn-project/slasher/src/factory/create_facade.ts b/yarn-project/slasher/src/factory/create_facade.ts index aa92d29509a..b5a075a749d 100644 --- a/yarn-project/slasher/src/factory/create_facade.ts +++ b/yarn-project/slasher/src/factory/create_facade.ts @@ -25,7 +25,10 @@ export async function createSlasherFacade( watchers: Watcher[], dateProvider: DateProvider, epochCache: EpochCache, - /** List of own validator addresses to add to the slashValidatorNever list unless slashSelfAllowed is true */ + /** + * List of own validator addresses. Added to the slashValidatorNever list unless slashSelfAllowed is true, and used + * (independently of slashSelfAllowed) to warn and record metrics when they are targeted by onchain slashing. + */ validatorAddresses: EthAddress[] = [], logger = createLogger('slasher'), ): Promise { @@ -76,6 +79,7 @@ export async function createSlasherFacade( dateProvider, kvStore, rollupRegisteredAtL2Slot, + validatorAddresses, logger, ); } diff --git a/yarn-project/slasher/src/factory/create_implementation.ts b/yarn-project/slasher/src/factory/create_implementation.ts index f9fe11a6d71..9eb4e9a61f7 100644 --- a/yarn-project/slasher/src/factory/create_implementation.ts +++ b/yarn-project/slasher/src/factory/create_implementation.ts @@ -2,6 +2,7 @@ import { EpochCache } from '@aztec/epoch-cache'; import { RollupContract, SlashingProposerContract } from '@aztec/ethereum/contracts'; import type { ViemClient } from '@aztec/ethereum/types'; import type { SlotNumber } from '@aztec/foundation/branded-types'; +import type { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; import { DateProvider } from '@aztec/foundation/timer'; import { AztecLMDBStoreV2 } from '@aztec/kv-store/lmdb-v2'; @@ -24,6 +25,7 @@ export async function createSlasherImplementation( dateProvider: DateProvider, kvStore: AztecLMDBStoreV2, rollupRegisteredAtL2Slot: SlotNumber, + ownValidators: EthAddress[] = [], logger = createLogger('slasher'), ) { const proposer = await rollup.getSlashingProposer(); @@ -39,6 +41,7 @@ export async function createSlasherImplementation( epochCache, kvStore, rollupRegisteredAtL2Slot, + ownValidators, logger, ); } @@ -53,6 +56,7 @@ async function createSlasher( epochCache: EpochCache, kvStore: AztecLMDBStoreV2, rollupRegisteredAtL2Slot: SlotNumber, + ownValidators: EthAddress[] = [], logger = createLogger('slasher'), ): Promise { const settings = { ...(await getSlasherSettings(rollup, slashingProposer)), rollupRegisteredAtL2Slot }; @@ -73,6 +77,7 @@ async function createSlasher( epochCache, dateProvider, offensesStore, + ownValidators, logger, ); } diff --git a/yarn-project/slasher/src/metrics.ts b/yarn-project/slasher/src/metrics.ts index dfa5994162a..90fd21e800a 100644 --- a/yarn-project/slasher/src/metrics.ts +++ b/yarn-project/slasher/src/metrics.ts @@ -1,19 +1,58 @@ +import type { EthAddress } from '@aztec/foundation/eth-address'; import { + Attributes, Metrics, type TelemetryClient, type UpDownCounter, createUpDownCounterWithDefault, } from '@aztec/telemetry-client'; +import { formatEther } from 'viem/utils'; + export class SlasherMetrics { private readonly roundExecuted: UpDownCounter; + private readonly ownValidatorTargeted: UpDownCounter; + private readonly ownValidatorSlashedCount: UpDownCounter; + private readonly ownValidatorSlashedAmount: UpDownCounter; - constructor(client: TelemetryClient, name = 'Slasher') { + constructor(client: TelemetryClient, ownValidators: EthAddress[] = [], name = 'Slasher') { const meter = client.getMeter(name); this.roundExecuted = createUpDownCounterWithDefault(meter, Metrics.SLASHER_ROUND_EXECUTED_COUNT); + + // Seed a zero-valued series per own validator so dashboards show the series before any slashing event; + // an empty array seeds nothing for nodes that run no validators. + const seedAttributes = + ownValidators.length > 0 ? { [Attributes.ATTESTER_ADDRESS]: ownValidators.map(v => v.toString()) } : []; + this.ownValidatorTargeted = createUpDownCounterWithDefault( + meter, + Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT, + seedAttributes, + ); + this.ownValidatorSlashedCount = createUpDownCounterWithDefault( + meter, + Metrics.SLASHER_OWN_VALIDATOR_SLASHED_COUNT, + seedAttributes, + ); + this.ownValidatorSlashedAmount = createUpDownCounterWithDefault( + meter, + Metrics.SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT, + seedAttributes, + ); } public recordRoundExecuted(): void { this.roundExecuted.add(1); } + + /** Records that an onchain slashing vote named one of the node's own validators as a target. */ + public recordOwnValidatorTargeted(validator: EthAddress): void { + this.ownValidatorTargeted.add(1, { [Attributes.ATTESTER_ADDRESS]: validator.toString() }); + } + + /** Records an executed slash against one of the node's own validators. */ + public recordOwnValidatorSlashed(validator: EthAddress, amount: bigint): void { + const attributes = { [Attributes.ATTESTER_ADDRESS]: validator.toString() }; + this.ownValidatorSlashedCount.add(1, attributes); + this.ownValidatorSlashedAmount.add(parseFloat(formatEther(amount)), attributes); + } } diff --git a/yarn-project/slasher/src/slasher_client.test.ts b/yarn-project/slasher/src/slasher_client.test.ts index c94e12c000b..226c001c368 100644 --- a/yarn-project/slasher/src/slasher_client.test.ts +++ b/yarn-project/slasher/src/slasher_client.test.ts @@ -9,7 +9,7 @@ import { DateProvider } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb'; import type { SlasherConfig } from '@aztec/stdlib/interfaces/server'; import { type Offense, OffenseType, type ProposerSlashAction } from '@aztec/stdlib/slashing'; -import { Metrics } from '@aztec/telemetry-client'; +import { Attributes, Metrics } from '@aztec/telemetry-client'; import { BenchmarkTelemetryClient } from '@aztec/telemetry-client/bench'; import { jest } from '@jest/globals'; @@ -174,7 +174,11 @@ describe('SlasherClient', () => { slashingProposer.listenToRoundExecuted.mockReturnValue(() => {}); // Create consensus slasher client with proper constructor parameters - slasherClient = new TestSlasherClient( + slasherClient = createClient(); + }); + + const createClient = (ownValidators: EthAddress[] = []) => + new TestSlasherClient( config, settings, slashingProposer, @@ -184,10 +188,10 @@ describe('SlasherClient', () => { mockEpochCache, dateProvider, offensesStore, + ownValidators, logger, - new SlasherMetrics(telemetryClient), + new SlasherMetrics(telemetryClient, ownValidators), ); - }); afterEach(async () => { await slasherClient.stop(); @@ -1110,6 +1114,137 @@ describe('SlasherClient', () => { }); }); }); + + describe('self-slashing detection', () => { + let ownValidator: EthAddress; + const round = 5n; + const proposer = EthAddress.fromNumber(200).toString(); + + // Recreates the telemetry client along with the slasher client so metric assertions only see this client's data + const setupClient = (ownValidators: EthAddress[]) => { + telemetryClient = new BenchmarkTelemetryClient(); + slasherClient = createClient(ownValidators); + }; + + const getPoints = (name: string) => + telemetryClient + .getMeters() + .flatMap(meter => meter.metrics) + .find(metric => metric.name === name)?.points ?? []; + + beforeEach(() => { + ownValidator = committee[7]; + setupClient([ownValidator]); + }); + + it('warns and increments the targeted metric when a vote names an own validator', async () => { + slashingProposer.getLastVote.mockResolvedValue([ + { validator: ownValidator, slashAmount: slashingUnit }, + { validator: committee[1], slashAmount: slashingUnit }, + ]); + const warnSpy = jest.spyOn(logger, 'warn'); + + await slasherClient.handleVoteCast(round, proposer); + + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(slashingProposer.getLastVote).toHaveBeenCalledWith(round, settings.slashingAmounts); + const points = getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name); + expect(points.map(point => point.value)).toEqual([0, 1]); + expect(points.at(-1)?.attributes?.[Attributes.ATTESTER_ADDRESS]).toEqual(ownValidator.toString()); + }); + + it('warns only once per round and validator', async () => { + slashingProposer.getLastVote.mockResolvedValue([{ validator: ownValidator, slashAmount: slashingUnit }]); + const warnSpy = jest.spyOn(logger, 'warn'); + + await slasherClient.handleVoteCast(round, proposer); + await slasherClient.handleVoteCast(round, proposer); + + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0, 1]); + }); + + it('warns again when a later round also names the validator', async () => { + slashingProposer.getLastVote.mockResolvedValue([{ validator: ownValidator, slashAmount: slashingUnit }]); + const warnSpy = jest.spyOn(logger, 'warn'); + + await slasherClient.handleVoteCast(round, proposer); + await slasherClient.handleVoteCast(round + 1n, proposer); + + expect(warnSpy).toHaveBeenCalledTimes(2); + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0, 1, 1]); + }); + + it('does not warn when votes only name other validators', async () => { + slashingProposer.getLastVote.mockResolvedValue([{ validator: committee[1], slashAmount: slashingUnit }]); + const warnSpy = jest.spyOn(logger, 'warn'); + + await slasherClient.handleVoteCast(round, proposer); + + expect(warnSpy).not.toHaveBeenCalled(); + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0]); + }); + + it('warns for each own validator named in a vote', async () => { + setupClient([committee[7], committee[8], committee[9]]); + slashingProposer.getLastVote.mockResolvedValue([ + { validator: committee[7], slashAmount: slashingUnit }, + { validator: committee[8], slashAmount: slashingUnit * 2n }, + { validator: committee[1], slashAmount: slashingUnit }, + ]); + const warnSpy = jest.spyOn(logger, 'warn'); + + await slasherClient.handleVoteCast(round, proposer); + + expect(warnSpy).toHaveBeenCalledTimes(2); + const points = getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name); + expect(points.map(point => point.value)).toEqual([0, 0, 0, 1, 1]); + expect( + points.filter(point => point.value === 1).map(point => point.attributes?.[Attributes.ATTESTER_ADDRESS]), + ).toEqual([committee[7].toString(), committee[8].toString()]); + }); + + it('warns and records metrics when an own validator is slashed at round execution', async () => { + rollup.getSlashEvents.mockResolvedValue([ + { attester: ownValidator, amount: slashingUnit * 2n }, + { attester: committee[1], amount: slashingUnit }, + ]); + const warnSpy = jest.spyOn(logger, 'warn'); + + await slasherClient.handleRoundExecuted(7n, 2n, '0x1'); + + expect(warnSpy).toHaveBeenCalledTimes(1); + const countPoints = getPoints(Metrics.SLASHER_OWN_VALIDATOR_SLASHED_COUNT.name); + expect(countPoints.map(point => point.value)).toEqual([0, 1]); + expect(countPoints.at(-1)?.attributes?.[Attributes.ATTESTER_ADDRESS]).toEqual(ownValidator.toString()); + // The 2e18 wei slash is recorded as 2 in eth units + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT.name).map(point => point.value)).toEqual([0, 2]); + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0]); + expect(getPoints(Metrics.SLASHER_ROUND_EXECUTED_COUNT.name).map(point => point.value)).toEqual([0, 1]); + }); + + it('subscribes to vote cast events only when running own validators', async () => { + setupClient([]); + await slasherClient.start(); + expect(slashingProposer.listenToVoteCast).not.toHaveBeenCalled(); + await slasherClient.stop(); + + setupClient([ownValidator]); + await slasherClient.start(); + expect(slashingProposer.listenToVoteCast).toHaveBeenCalledTimes(1); + }); + + it('warns again for a round once its dedupe entry expires', async () => { + slashingProposer.getLastVote.mockResolvedValue([{ validator: ownValidator, slashAmount: slashingUnit }]); + const warnSpy = jest.spyOn(logger, 'warn'); + + await slasherClient.handleVoteCast(round, proposer); + await slasherClient.handleNewRound(round + BigInt(settings.slashingLifetimeInRounds) + 1n); + await slasherClient.handleVoteCast(round, proposer); + + expect(warnSpy).toHaveBeenCalledTimes(2); + }); + }); }); // Test helper class that exposes protected methods for testing @@ -1122,6 +1257,10 @@ class TestSlasherClient extends SlasherClient { return super.handleNewRound(round); } + public override handleVoteCast(round: bigint, proposer: string): Promise { + return super.handleVoteCast(round, proposer); + } + public override getExecuteSlashAction(slotNumber: SlotNumber): Promise { return super.getExecuteSlashAction(slotNumber); } diff --git a/yarn-project/slasher/src/slasher_client.ts b/yarn-project/slasher/src/slasher_client.ts index 441b110e106..8ac447cf06c 100644 --- a/yarn-project/slasher/src/slasher_client.ts +++ b/yarn-project/slasher/src/slasher_client.ts @@ -91,6 +91,8 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient protected unwatchCallbacks: (() => void)[] = []; protected roundMonitor: SlashRoundMonitor; protected offensesCollector: SlashOffensesCollector; + /** Rounds mapped to own validators already warned about, so each (round, validator) pair warns only once. */ + private readonly warnedSelfSlashVotes = new Map>(); constructor( private config: SlasherClientConfig, @@ -102,8 +104,9 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient private epochCache: EpochCache, private dateProvider: DateProvider, private offensesStore: SlasherOffensesStore, + private readonly ownValidators: EthAddress[] = [], private log = createLogger('slasher:consensus'), - private readonly metrics = new SlasherMetrics(getTelemetryClient()), + private readonly metrics = new SlasherMetrics(getTelemetryClient(), ownValidators), ) { this.roundMonitor = new SlashRoundMonitor(settings, dateProvider); this.offensesCollector = new SlashOffensesCollector(config, settings, watchers, offensesStore); @@ -125,6 +128,16 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient ), ); + // Listen for VoteCast events to warn early when a vote names one of our own validators as a slash target + if (this.ownValidators.length > 0) { + this.unwatchCallbacks.push( + this.slashingProposer.listenToVoteCast( + ({ round, proposer }) => + void this.handleVoteCast(round, proposer).catch(err => this.log.error('Error handling vote cast', err)), + ), + ); + } + // Check for round changes this.unwatchCallbacks.push(this.roundMonitor.listenToNewRound(round => this.handleNewRound(round))); @@ -159,14 +172,60 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient /** Triggered on a time basis when we enter a new slashing round. Clears expired offenses. */ protected async handleNewRound(round: bigint) { this.log.info(`Starting new slashing round ${round}`); + this.pruneSelfSlashVoteWarnings(round); await this.offensesCollector.handleNewRound(round); } - /** Called when we see a RoundExecuted event on the SlashingProposer (just for logging). */ + /** Called when we see a RoundExecuted event on the SlashingProposer (just for logging and metrics). */ protected async handleRoundExecuted(round: bigint, slashCount: bigint, l1BlockHash: Hex) { this.metrics.recordRoundExecuted(); const slashes = await this.rollup.getSlashEvents(l1BlockHash); this.log.info(`Slashing round ${round} has been executed with ${slashCount} slashes`, { slashes }); + + for (const { attester, amount } of slashes.filter(slash => this.isOwnValidator(slash.attester))) { + this.log.warn(`Own validator ${attester} was slashed for ${amount}`, { + round, + validator: attester.toString(), + amount, + l1BlockHash, + }); + this.metrics.recordOwnValidatorSlashed(attester, amount); + } + } + + /** Called when we see a VoteCast event. Warns once per round and validator when a vote names one of our own. */ + protected async handleVoteCast(round: bigint, proposer: string) { + const votes = await this.slashingProposer.getLastVote(round, this.settings.slashingAmounts); + for (const { validator, slashAmount } of votes.filter(vote => this.isOwnValidator(vote.validator))) { + const warned = this.warnedSelfSlashVotes.get(round) ?? new Set(); + if (warned.has(validator.toString())) { + continue; + } + warned.add(validator.toString()); + this.warnedSelfSlashVotes.set(round, warned); + + this.log.warn(`Detected onchain slashing vote against own validator ${validator}`, { + round, + validator: validator.toString(), + slashAmount, + proposer, + }); + this.metrics.recordOwnValidatorTargeted(validator); + } + } + + private isOwnValidator(address: EthAddress): boolean { + return this.ownValidators.some(validator => validator.equals(address)); + } + + /** Drops warning dedupe entries for rounds past their slashing lifetime, as they can no longer receive votes. */ + private pruneSelfSlashVoteWarnings(currentRound: bigint) { + const oldestLiveRound = currentRound - BigInt(this.settings.slashingLifetimeInRounds); + for (const round of this.warnedSelfSlashVotes.keys()) { + if (round < oldestLiveRound) { + this.warnedSelfSlashVotes.delete(round); + } + } } /** diff --git a/yarn-project/slasher/src/slasher_client_facade.ts b/yarn-project/slasher/src/slasher_client_facade.ts index f902a48d7a4..499f1da6aa3 100644 --- a/yarn-project/slasher/src/slasher_client_facade.ts +++ b/yarn-project/slasher/src/slasher_client_facade.ts @@ -2,6 +2,7 @@ import { EpochCache } from '@aztec/epoch-cache'; import { RollupContract } from '@aztec/ethereum/contracts'; import type { ViemClient } from '@aztec/ethereum/types'; import type { SlotNumber } from '@aztec/foundation/branded-types'; +import type { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; import { DateProvider } from '@aztec/foundation/timer'; import { AztecLMDBStoreV2 } from '@aztec/kv-store/lmdb-v2'; @@ -31,6 +32,7 @@ export class SlasherClientFacade implements SlasherClientInterface { private dateProvider: DateProvider, private kvStore: AztecLMDBStoreV2, private rollupRegisteredAtL2Slot: SlotNumber, + private ownValidators: EthAddress[] = [], private logger = createLogger('slasher'), ) {} @@ -83,6 +85,7 @@ export class SlasherClientFacade implements SlasherClientInterface { this.dateProvider, this.kvStore, this.rollupRegisteredAtL2Slot, + this.ownValidators, this.logger, ); } diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 7342f8bd466..03f58f5470f 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -548,6 +548,23 @@ export const SLASHER_ROUND_EXECUTED_COUNT: MetricDefinition = { description: 'The number of slashing rounds executed', valueType: ValueType.INT, }; +export const SLASHER_OWN_VALIDATOR_TARGETED_COUNT: MetricDefinition = { + name: 'aztec.slasher.own_validator.targeted_count', + description: + "The number of times one of the node's own validators was named as a slash target by an onchain vote, deduplicated per round and validator", + valueType: ValueType.INT, +}; +export const SLASHER_OWN_VALIDATOR_SLASHED_COUNT: MetricDefinition = { + name: 'aztec.slasher.own_validator.slashed_count', + description: "The number of times one of the node's own validators was slashed, from executed Slashed events", + valueType: ValueType.INT, +}; +export const SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT: MetricDefinition = { + name: 'aztec.slasher.own_validator.slashed_amount', + description: "Cumulative amount slashed from the node's own validators, from executed Slashed events", + unit: 'eth', + valueType: ValueType.DOUBLE, +}; export const SEQUENCER_CHECKPOINT_SUCCESS_COUNT: MetricDefinition = { name: 'aztec.sequencer.checkpoint.success_count', description: 'The number of times checkpoint publishing succeeded', From ea026e9c117da2f3cb82df3d022788922358dad9 Mon Sep 17 00:00:00 2001 From: aminsammara Date: Mon, 27 Jul 2026 18:14:37 +0000 Subject: [PATCH 2/2] fix(slasher): tally own-validator slash votes per committee position Quorum is tallied per committee position, not per validator address, so a validator sitting in several of a round's committees held several independent tallies that were being summed. Also drops the per-attester metric label in favour of node-level series, warns on every vote with the running count against quorum, memoizes the immutable slashing amounts, seeds the round tally at startup, and collapses batched VoteCast deliveries. --- .../src/contracts/slashing_proposer.test.ts | 45 +++- .../src/contracts/slashing_proposer.ts | 111 ++++++++-- yarn-project/slasher/src/metrics.ts | 51 ++--- .../slasher/src/slasher_client.test.ts | 199 ++++++++++++++---- yarn-project/slasher/src/slasher_client.ts | 140 ++++++++---- yarn-project/telemetry-client/src/metrics.ts | 22 +- 6 files changed, 437 insertions(+), 131 deletions(-) diff --git a/yarn-project/ethereum/src/contracts/slashing_proposer.test.ts b/yarn-project/ethereum/src/contracts/slashing_proposer.test.ts index 14135df8a10..f02375da615 100644 --- a/yarn-project/ethereum/src/contracts/slashing_proposer.test.ts +++ b/yarn-project/ethereum/src/contracts/slashing_proposer.test.ts @@ -17,7 +17,7 @@ import { createExtendedL1Client } from '../client.js'; import { DefaultL1ContractsConfig } from '../config.js'; import { type DeployAztecL1ContractsArgs, deployAztecL1Contracts } from '../deploy_aztec_l1_contracts.js'; import type { Anvil } from '../test/start_anvil.js'; -import { RollupContract, decodeSlashConsensusVotes } from './index.js'; +import { RollupContract, collapseVoteCastLogs, decodeSlashConsensusVotes } from './index.js'; import { SlashingProposerContract } from './slashing_proposer.js'; describe('SlashingProposer', () => { @@ -290,4 +290,47 @@ describe('SlashingProposer', () => { expect(votes).toEqual([0, 0, 1, 2, 3, 2, 1, 0]); }); }); + + describe('collapseVoteCastLogs', () => { + const voteCastLog = (round: bigint, slot: bigint, proposer: string) => ({ args: { round, slot, proposer } }); + + it('passes a single log through', () => { + const collapsed = collapseVoteCastLogs([voteCastLog(5n, 100n, '0xaaa')]); + + expect(collapsed).toEqual([{ round: 5n, slot: SlotNumber(100), proposer: '0xaaa' }]); + }); + + it('keeps only the last log of a round', () => { + const collapsed = collapseVoteCastLogs([voteCastLog(5n, 100n, '0xaaa'), voteCastLog(5n, 101n, '0xbbb')]); + + expect(collapsed).toEqual([{ round: 5n, slot: SlotNumber(101), proposer: '0xbbb' }]); + }); + + it('keeps the last log of every round when a batch spans a round boundary', () => { + const collapsed = collapseVoteCastLogs([ + voteCastLog(5n, 100n, '0xaaa'), + voteCastLog(5n, 101n, '0xbbb'), + voteCastLog(6n, 102n, '0xccc'), + ]); + + expect(collapsed).toEqual([ + { round: 5n, slot: SlotNumber(101), proposer: '0xbbb' }, + { round: 6n, slot: SlotNumber(102), proposer: '0xccc' }, + ]); + }); + + it('returns nothing for an empty batch', () => { + expect(collapseVoteCastLogs([])).toEqual([]); + }); + + it('drops logs with undecoded args', () => { + const collapsed = collapseVoteCastLogs([ + { args: { round: undefined, slot: 100n, proposer: '0xaaa' } }, + { args: { round: 5n, slot: undefined, proposer: '0xaaa' } }, + { args: { round: 5n, slot: 100n, proposer: undefined } }, + ]); + + expect(collapsed).toEqual([]); + }); + }); }); diff --git a/yarn-project/ethereum/src/contracts/slashing_proposer.ts b/yarn-project/ethereum/src/contracts/slashing_proposer.ts index de44d1df0bd..138a90d390e 100644 --- a/yarn-project/ethereum/src/contracts/slashing_proposer.ts +++ b/yarn-project/ethereum/src/contracts/slashing_proposer.ts @@ -3,6 +3,8 @@ import type { ViemClient } from '@aztec/ethereum/types'; import { mergeAbis, tryExtractEvent } from '@aztec/ethereum/utils'; import { SlotNumber } from '@aztec/foundation/branded-types'; import { Buffer32 } from '@aztec/foundation/buffer'; +import { chunk, times } from '@aztec/foundation/collection'; +import { memoize } from '@aztec/foundation/decorators'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Signature } from '@aztec/foundation/eth-signature'; import { hexToBuffer } from '@aztec/foundation/string'; @@ -64,6 +66,12 @@ export class SlashingProposerContract { return this.contract.read.EXECUTION_DELAY_IN_ROUNDS(); } + /** + * Returns the slash amounts for the three slash unit levels, which are immutable on the contract. + * Memoized: the first call's promise is cached for the instance's lifetime, including a rejection, so make an + * awaited call during startup before relying on this from event handlers. + */ + @memoize public getSlashingAmounts(): Promise<[bigint, bigint, bigint]> { return Promise.all([ this.contract.read.SLASH_AMOUNT_SMALL(), @@ -234,22 +242,47 @@ export class SlashingProposerContract { }; } + /** Returns the validators eligible to be voted against in a round, in the order votes encode them */ + public async getSlashTargetValidators(round: bigint): Promise { + const { result } = await this.contract.simulate.getSlashTargetCommittees([round]); + return result.flat().map(validator => EthAddress.fromString(validator)); + } + /** - * Returns the last vote emitted for a given round - * @param slashingAmounts - The slash amount per vote unit, to avoid re-reading them from the contract + * Returns the slash amount voted for each target validator by a single vote of a round. + * @param index - Position of the vote within the round, from 0 (inclusive) to the round's vote count (exclusive) */ - public async getLastVote(round: bigint, slashingAmounts?: [bigint, bigint, bigint]) { + public async getVoteAt(round: bigint, index: bigint): Promise { + const [validators, vote, slashAmounts] = await Promise.all([ + this.getSlashTargetValidators(round), + this.contract.read.getVotes([round, index]), + this.getSlashingAmounts(), + ]); + return decodeVote(vote, validators, slashAmounts); + } + + /** + * Returns every vote cast so far in a round, oldest first. The target validators and slash amounts are read once + * for the whole round rather than per vote, so this is much cheaper than reading each vote individually. + */ + public async getVotesForRound(round: bigint): Promise { + const [{ voteCount }, validators, slashAmounts] = await Promise.all([ + this.getRound(round), + this.getSlashTargetValidators(round), + this.getSlashingAmounts(), + ]); + // A round can hold as many votes as it has slots, too many for a single parallel burst of reads + const votes: Hex[] = []; + for (const indices of chunk(times(Number(voteCount), BigInt), VOTE_READ_BATCH_SIZE)) { + votes.push(...(await Promise.all(indices.map(index => this.contract.read.getVotes([round, index]))))); + } + return votes.map(vote => decodeVote(vote, validators, slashAmounts)); + } + + /** Returns the last vote emitted for a given round */ + public async getLastVote(round: bigint) { const { voteCount } = await this.getRound(round); - const validators = (await this.contract.simulate.getSlashTargetCommittees([round])).result.flat(); - const vote = await this.contract.read.getVotes([round, voteCount - 1n]); - const decoded = decodeSlashConsensusVotes(hexToBuffer(vote)); - const slashAmounts = slashingAmounts ?? (await this.getSlashingAmounts()); - return decoded - .map((units, i) => ({ - validator: EthAddress.fromString(validators[i]), - slashAmount: slashAmounts[units - 1] ?? 0n, - })) - .filter(v => v.slashAmount > 0n); + return await this.getVoteAt(round, voteCount - 1n); } /** @@ -257,16 +290,13 @@ export class SlashingProposerContract { * @param callback - Callback function to handle vote cast events * @returns Unwatch function */ - public listenToVoteCast(callback: (args: { round: bigint; proposer: string }) => void): () => void { + public listenToVoteCast(callback: (args: VoteCastEventArgs) => void): () => void { return this.contract.watchEvent.VoteCast( {}, { onLogs: logs => { - for (const log of logs) { - const { round, proposer } = log.args; - if (round !== undefined && proposer) { - callback({ round, proposer }); - } + for (const args of collapseVoteCastLogs(logs)) { + callback(args); } }, }, @@ -297,6 +327,49 @@ export class SlashingProposerContract { } } +/** Maximum number of parallel getVotes reads when fetching a whole round. */ +const VOTE_READ_BATCH_SIZE = 32; + +/** + * The validators a single slashing vote targets, with the amount voted for each. The position is the validator's + * index in the round's flattened slash target committees — the unit the contract tallies quorum by. A validator + * sitting in several of the round's committees holds several positions, each with its own tally. + */ +export type SlashVote = { validator: EthAddress; slashAmount: bigint; position: number }[]; + +function decodeVote(vote: Hex, validators: EthAddress[], slashAmounts: [bigint, bigint, bigint]): SlashVote { + return decodeSlashConsensusVotes(hexToBuffer(vote)) + .map((units, position) => ({ + validator: validators[position], + slashAmount: slashAmounts[units - 1] ?? 0n, + position, + })) + .filter(v => v.slashAmount > 0n); +} + +/** Arguments decoded from a VoteCast event. */ +export type VoteCastEventArgs = { round: bigint; slot: SlotNumber; proposer: string }; + +/** + * Collapses a batch of VoteCast logs down to the latest log of each round. + * + * A vote is resolved by reading the round's most recent entry, so acting on every log in a batch would read that + * same entry once per log: the newest vote gets counted repeatedly while the ones behind it are never read at all. + * Batches only occur when L1 log delivery falls behind, but the resulting miscount lasts for the rest of the round. + */ +export function collapseVoteCastLogs( + logs: { args: { round?: bigint; slot?: bigint; proposer?: string } }[], +): VoteCastEventArgs[] { + const latestPerRound = new Map(); + for (const { args } of logs) { + const { round, slot, proposer } = args; + if (round !== undefined && slot !== undefined && proposer) { + latestPerRound.set(round, { round, slot: SlotNumber.fromBigInt(slot), proposer }); + } + } + return [...latestPerRound.values()]; +} + /** * Decodes a Buffer containing slash votes back into an array of numbers. * Each vote is represented as a 2-bit value (0, 1, 2, or 3) representing slashing units. diff --git a/yarn-project/slasher/src/metrics.ts b/yarn-project/slasher/src/metrics.ts index 90fd21e800a..1f41158a3f7 100644 --- a/yarn-project/slasher/src/metrics.ts +++ b/yarn-project/slasher/src/metrics.ts @@ -1,6 +1,5 @@ -import type { EthAddress } from '@aztec/foundation/eth-address'; import { - Attributes, + type Gauge, Metrics, type TelemetryClient, type UpDownCounter, @@ -14,45 +13,47 @@ export class SlasherMetrics { private readonly ownValidatorTargeted: UpDownCounter; private readonly ownValidatorSlashedCount: UpDownCounter; private readonly ownValidatorSlashedAmount: UpDownCounter; + private readonly ownValidatorCurrentRoundVotesMax: Gauge; + private readonly quorumSize: Gauge; - constructor(client: TelemetryClient, ownValidators: EthAddress[] = [], name = 'Slasher') { + constructor(client: TelemetryClient, name = 'Slasher') { const meter = client.getMeter(name); this.roundExecuted = createUpDownCounterWithDefault(meter, Metrics.SLASHER_ROUND_EXECUTED_COUNT); - - // Seed a zero-valued series per own validator so dashboards show the series before any slashing event; - // an empty array seeds nothing for nodes that run no validators. - const seedAttributes = - ownValidators.length > 0 ? { [Attributes.ATTESTER_ADDRESS]: ownValidators.map(v => v.toString()) } : []; - this.ownValidatorTargeted = createUpDownCounterWithDefault( - meter, - Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT, - seedAttributes, - ); - this.ownValidatorSlashedCount = createUpDownCounterWithDefault( - meter, - Metrics.SLASHER_OWN_VALIDATOR_SLASHED_COUNT, - seedAttributes, - ); + this.ownValidatorTargeted = createUpDownCounterWithDefault(meter, Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT); + this.ownValidatorSlashedCount = createUpDownCounterWithDefault(meter, Metrics.SLASHER_OWN_VALIDATOR_SLASHED_COUNT); this.ownValidatorSlashedAmount = createUpDownCounterWithDefault( meter, Metrics.SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT, - seedAttributes, ); + this.ownValidatorCurrentRoundVotesMax = meter.createGauge(Metrics.SLASHER_OWN_VALIDATOR_CURRENT_ROUND_VOTES_MAX); + this.quorumSize = meter.createGauge(Metrics.SLASHER_QUORUM_SIZE); } public recordRoundExecuted(): void { this.roundExecuted.add(1); } + /** Records the quorum a validator must reach in a round to be slashed, so dashboards can plot the threshold. */ + public recordQuorumSize(quorum: number): void { + this.quorumSize.record(quorum); + } + /** Records that an onchain slashing vote named one of the node's own validators as a target. */ - public recordOwnValidatorTargeted(validator: EthAddress): void { - this.ownValidatorTargeted.add(1, { [Attributes.ATTESTER_ADDRESS]: validator.toString() }); + public recordOwnValidatorTargeted(): void { + this.ownValidatorTargeted.add(1); + } + + /** + * Records how close the most-voted committee position held by the node's own validators is to quorum this round. + * Recorded as an absolute value rather than a delta so a vote seen across a round rollover cannot make it drift. + */ + public recordCurrentRoundVotesMax(votes: number): void { + this.ownValidatorCurrentRoundVotesMax.record(votes); } /** Records an executed slash against one of the node's own validators. */ - public recordOwnValidatorSlashed(validator: EthAddress, amount: bigint): void { - const attributes = { [Attributes.ATTESTER_ADDRESS]: validator.toString() }; - this.ownValidatorSlashedCount.add(1, attributes); - this.ownValidatorSlashedAmount.add(parseFloat(formatEther(amount)), attributes); + public recordOwnValidatorSlashed(amount: bigint): void { + this.ownValidatorSlashedCount.add(1); + this.ownValidatorSlashedAmount.add(parseFloat(formatEther(amount))); } } diff --git a/yarn-project/slasher/src/slasher_client.test.ts b/yarn-project/slasher/src/slasher_client.test.ts index 226c001c368..5a8550dcdfd 100644 --- a/yarn-project/slasher/src/slasher_client.test.ts +++ b/yarn-project/slasher/src/slasher_client.test.ts @@ -9,7 +9,7 @@ import { DateProvider } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb'; import type { SlasherConfig } from '@aztec/stdlib/interfaces/server'; import { type Offense, OffenseType, type ProposerSlashAction } from '@aztec/stdlib/slashing'; -import { Attributes, Metrics } from '@aztec/telemetry-client'; +import { Metrics } from '@aztec/telemetry-client'; import { BenchmarkTelemetryClient } from '@aztec/telemetry-client/bench'; import { jest } from '@jest/globals'; @@ -190,7 +190,7 @@ describe('SlasherClient', () => { offensesStore, ownValidators, logger, - new SlasherMetrics(telemetryClient, ownValidators), + new SlasherMetrics(telemetryClient), ); afterEach(async () => { @@ -1118,6 +1118,7 @@ describe('SlasherClient', () => { describe('self-slashing detection', () => { let ownValidator: EthAddress; const round = 5n; + const slot = SlotNumber(100); const proposer = EthAddress.fromNumber(200).toString(); // Recreates the telemetry client along with the slasher client so metric assertions only see this client's data @@ -1137,71 +1138,151 @@ describe('SlasherClient', () => { setupClient([ownValidator]); }); + const getVotesMax = () => getPoints(Metrics.SLASHER_OWN_VALIDATOR_CURRENT_ROUND_VOTES_MAX.name); + + /** A vote entry naming a validator, defaulting its committee index as the flattened committee position */ + const voteAgainst = (validator: EthAddress, opts: { position?: number; slashAmount?: bigint } = {}) => ({ + validator, + slashAmount: opts.slashAmount ?? slashingUnit, + position: opts.position ?? committee.findIndex(member => member.equals(validator)), + }); + it('warns and increments the targeted metric when a vote names an own validator', async () => { - slashingProposer.getLastVote.mockResolvedValue([ - { validator: ownValidator, slashAmount: slashingUnit }, - { validator: committee[1], slashAmount: slashingUnit }, - ]); + slashingProposer.getLastVote.mockResolvedValue([voteAgainst(ownValidator), voteAgainst(committee[1])]); const warnSpy = jest.spyOn(logger, 'warn'); - await slasherClient.handleVoteCast(round, proposer); + await slasherClient.handleVoteCast(round, slot, proposer); expect(warnSpy).toHaveBeenCalledTimes(1); - expect(slashingProposer.getLastVote).toHaveBeenCalledWith(round, settings.slashingAmounts); - const points = getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name); - expect(points.map(point => point.value)).toEqual([0, 1]); - expect(points.at(-1)?.attributes?.[Attributes.ATTESTER_ADDRESS]).toEqual(ownValidator.toString()); + expect(warnSpy).toHaveBeenCalledWith( + `Own validator ${ownValidator} targeted by slashing vote (1 of ${settings.slashingQuorumSize} votes needed to slash)`, + expect.objectContaining({ + round, + slot, + proposer, + validator: ownValidator.toString(), + votes: 1, + quorum: settings.slashingQuorumSize, + }), + ); + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0, 1]); + expect(getVotesMax().map(point => point.value)).toEqual([0, 1]); }); - it('warns only once per round and validator', async () => { - slashingProposer.getLastVote.mockResolvedValue([{ validator: ownValidator, slashAmount: slashingUnit }]); + it('warns on every vote against a validator, not just the first of a round', async () => { + slashingProposer.getLastVote.mockResolvedValue([voteAgainst(ownValidator)]); const warnSpy = jest.spyOn(logger, 'warn'); - await slasherClient.handleVoteCast(round, proposer); - await slasherClient.handleVoteCast(round, proposer); + await slasherClient.handleVoteCast(round, slot, proposer); + await slasherClient.handleVoteCast(round, slot, proposer); - expect(warnSpy).toHaveBeenCalledTimes(1); - expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0, 1]); + expect(warnSpy).toHaveBeenCalledTimes(2); + expect(warnSpy).toHaveBeenLastCalledWith( + expect.stringContaining(`(2 of ${settings.slashingQuorumSize} votes needed to slash)`), + expect.objectContaining({ votes: 2 }), + ); + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0, 1, 1]); + // The round tally climbs towards the quorum, unlike the cumulative counter + expect(getVotesMax().map(point => point.value)).toEqual([0, 1, 2]); + }); + + it('reports how close the most targeted validator is when several are named', async () => { + setupClient([committee[7], committee[8]]); + slashingProposer.getLastVote + .mockResolvedValueOnce([voteAgainst(committee[7]), voteAgainst(committee[8])]) + .mockResolvedValueOnce([voteAgainst(committee[7])]); + + await slasherClient.handleVoteCast(round, slot, proposer); + await slasherClient.handleVoteCast(round, slot, proposer); + + expect(getVotesMax().at(-1)?.value).toEqual(2); }); - it('warns again when a later round also names the validator', async () => { - slashingProposer.getLastVote.mockResolvedValue([{ validator: ownValidator, slashAmount: slashingUnit }]); + it('resets the round tally when a new round starts', async () => { + slashingProposer.getLastVote.mockResolvedValue([voteAgainst(ownValidator)]); const warnSpy = jest.spyOn(logger, 'warn'); - await slasherClient.handleVoteCast(round, proposer); - await slasherClient.handleVoteCast(round + 1n, proposer); + await slasherClient.handleVoteCast(round, slot, proposer); + await slasherClient.handleVoteCast(round + 1n, slot, proposer); expect(warnSpy).toHaveBeenCalledTimes(2); - expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0, 1, 1]); + expect(getVotesMax().map(point => point.value)).toEqual([0, 1, 0, 1]); + }); + + it('keeps the round tally when the clock announces the round a vote already rolled to', async () => { + slashingProposer.getLastVote.mockResolvedValue([voteAgainst(ownValidator)]); + + await slasherClient.handleVoteCast(round, slot, proposer); + await slasherClient.handleNewRound(round); + await slasherClient.handleVoteCast(round, slot, proposer); + + expect(getVotesMax().at(-1)?.value).toEqual(2); + }); + + it('zeroes the round tally when a round passes without any votes', async () => { + slashingProposer.getLastVote.mockResolvedValue([voteAgainst(ownValidator)]); + + await slasherClient.handleVoteCast(round, slot, proposer); + await slasherClient.handleNewRound(round + 1n); + + expect(getVotesMax().at(-1)?.value).toEqual(0); + }); + + it('ignores votes cast for a round that has already closed', async () => { + slashingProposer.getLastVote.mockResolvedValue([voteAgainst(ownValidator)]); + const warnSpy = jest.spyOn(logger, 'warn'); + + await slasherClient.handleVoteCast(round + 1n, slot, proposer); + await slasherClient.handleVoteCast(round, slot, proposer); + + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(getVotesMax().at(-1)?.value).toEqual(1); }); it('does not warn when votes only name other validators', async () => { - slashingProposer.getLastVote.mockResolvedValue([{ validator: committee[1], slashAmount: slashingUnit }]); + slashingProposer.getLastVote.mockResolvedValue([voteAgainst(committee[1])]); const warnSpy = jest.spyOn(logger, 'warn'); - await slasherClient.handleVoteCast(round, proposer); + await slasherClient.handleVoteCast(round, slot, proposer); expect(warnSpy).not.toHaveBeenCalled(); expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0]); + expect(getVotesMax().map(point => point.value)).toEqual([0, 0]); }); it('warns for each own validator named in a vote', async () => { setupClient([committee[7], committee[8], committee[9]]); slashingProposer.getLastVote.mockResolvedValue([ - { validator: committee[7], slashAmount: slashingUnit }, - { validator: committee[8], slashAmount: slashingUnit * 2n }, - { validator: committee[1], slashAmount: slashingUnit }, + voteAgainst(committee[7]), + voteAgainst(committee[8], { slashAmount: slashingUnit * 2n }), + voteAgainst(committee[1]), ]); const warnSpy = jest.spyOn(logger, 'warn'); - await slasherClient.handleVoteCast(round, proposer); + await slasherClient.handleVoteCast(round, slot, proposer); expect(warnSpy).toHaveBeenCalledTimes(2); - const points = getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name); - expect(points.map(point => point.value)).toEqual([0, 0, 0, 1, 1]); - expect( - points.filter(point => point.value === 1).map(point => point.attributes?.[Attributes.ATTESTER_ADDRESS]), - ).toEqual([committee[7].toString(), committee[8].toString()]); + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0, 1, 1]); + }); + + it('tallies per committee position when a validator sits in several of the round committees', async () => { + // A single vote names the validator once per position it holds, but each position races quorum separately, + // so this is one vote of quorum rather than two + slashingProposer.getLastVote.mockResolvedValue([ + voteAgainst(ownValidator, { position: 7 }), + voteAgainst(ownValidator, { position: 7 + settings.targetCommitteeSize }), + ]); + const warnSpy = jest.spyOn(logger, 'warn'); + + await slasherClient.handleVoteCast(round, slot, proposer); + + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining(`(1 of ${settings.slashingQuorumSize} votes needed to slash)`), + expect.objectContaining({ votes: 1 }), + ); + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0, 1]); + expect(getVotesMax().at(-1)?.value).toEqual(1); }); it('warns and records metrics when an own validator is slashed at round execution', async () => { @@ -1214,10 +1295,8 @@ describe('SlasherClient', () => { await slasherClient.handleRoundExecuted(7n, 2n, '0x1'); expect(warnSpy).toHaveBeenCalledTimes(1); - const countPoints = getPoints(Metrics.SLASHER_OWN_VALIDATOR_SLASHED_COUNT.name); - expect(countPoints.map(point => point.value)).toEqual([0, 1]); - expect(countPoints.at(-1)?.attributes?.[Attributes.ATTESTER_ADDRESS]).toEqual(ownValidator.toString()); - // The 2e18 wei slash is recorded as 2 in eth units + expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_SLASHED_COUNT.name).map(point => point.value)).toEqual([0, 1]); + // The 2e18 base-unit slash is recorded as 2 whole staking-asset tokens expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT.name).map(point => point.value)).toEqual([0, 2]); expect(getPoints(Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT.name).map(point => point.value)).toEqual([0]); expect(getPoints(Metrics.SLASHER_ROUND_EXECUTED_COUNT.name).map(point => point.value)).toEqual([0, 1]); @@ -1234,15 +1313,41 @@ describe('SlasherClient', () => { expect(slashingProposer.listenToVoteCast).toHaveBeenCalledTimes(1); }); - it('warns again for a round once its dedupe entry expires', async () => { - slashingProposer.getLastVote.mockResolvedValue([{ validator: ownValidator, slashAmount: slashingUnit }]); + it('exports the quorum size so the round tally can be read against it', async () => { + await slasherClient.start(); + + expect(getPoints(Metrics.SLASHER_QUORUM_SIZE.name).map(point => point.value)).toEqual([ + settings.slashingQuorumSize, + ]); + }); + + it('seeds the round tally from the votes already cast when starting mid-round', async () => { + slashingProposer.getVotesForRound.mockResolvedValue(times(3, () => [voteAgainst(ownValidator)])); const warnSpy = jest.spyOn(logger, 'warn'); - await slasherClient.handleVoteCast(round, proposer); - await slasherClient.handleNewRound(round + BigInt(settings.slashingLifetimeInRounds) + 1n); - await slasherClient.handleVoteCast(round, proposer); + await slasherClient.start(); - expect(warnSpy).toHaveBeenCalledTimes(2); + expect(getVotesMax().at(-1)?.value).toEqual(3); + // Replayed votes are counted but not warned about, since the operator cannot act on them any sooner + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('counts live votes on top of the seeded tally', async () => { + slashingProposer.getVotesForRound.mockResolvedValue(times(3, () => [voteAgainst(ownValidator)])); + slashingProposer.getLastVote.mockResolvedValue([voteAgainst(ownValidator)]); + + await slasherClient.start(); + await slasherClient.handleVoteCast(slasherClient.getCurrentRound(), slot, proposer); + + expect(getVotesMax().at(-1)?.value).toEqual(4); + }); + + it('starts with an empty tally when the votes already cast cannot be read', async () => { + slashingProposer.getVotesForRound.mockRejectedValue(new Error('L1 unavailable')); + + await slasherClient.start(); + + expect(slashingProposer.listenToVoteCast).toHaveBeenCalledTimes(1); }); }); }); @@ -1257,8 +1362,12 @@ class TestSlasherClient extends SlasherClient { return super.handleNewRound(round); } - public override handleVoteCast(round: bigint, proposer: string): Promise { - return super.handleVoteCast(round, proposer); + public override handleVoteCast(round: bigint, slot: SlotNumber, proposer: string): Promise { + return super.handleVoteCast(round, slot, proposer); + } + + public getCurrentRound(): bigint { + return this.roundMonitor.getCurrentRound().round; } public override getExecuteSlashAction(slotNumber: SlotNumber): Promise { diff --git a/yarn-project/slasher/src/slasher_client.ts b/yarn-project/slasher/src/slasher_client.ts index 8ac447cf06c..b9c8103ff5c 100644 --- a/yarn-project/slasher/src/slasher_client.ts +++ b/yarn-project/slasher/src/slasher_client.ts @@ -1,6 +1,6 @@ import { EthAddress } from '@aztec/aztec.js/addresses'; import type { EpochCache } from '@aztec/epoch-cache'; -import { RollupContract, SlasherContract, SlashingProposerContract } from '@aztec/ethereum/contracts'; +import { RollupContract, type SlashVote, SlasherContract, SlashingProposerContract } from '@aztec/ethereum/contracts'; import { maxBigint } from '@aztec/foundation/bigint'; import { SlotNumber } from '@aztec/foundation/branded-types'; import { compactArray, partition, times } from '@aztec/foundation/collection'; @@ -91,8 +91,16 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient protected unwatchCallbacks: (() => void)[] = []; protected roundMonitor: SlashRoundMonitor; protected offensesCollector: SlashOffensesCollector; - /** Rounds mapped to own validators already warned about, so each (round, validator) pair warns only once. */ - private readonly warnedSelfSlashVotes = new Map>(); + /** + * Slashing votes cast during a single round against committee positions held by the node's own validators, keyed + * by the position's index in the round's flattened slash target committees. The contract tallies quorum per + * position, and a validator sitting in several of the round's committees holds several independent positions. + * A vote is only ever cast for the round current at the time, so only one round is ever tracked. + */ + private ownValidatorVotes: { round: bigint; countByPosition: Map } = { + round: -1n, + countByPosition: new Map(), + }; constructor( private config: SlasherClientConfig, @@ -106,7 +114,7 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient private offensesStore: SlasherOffensesStore, private readonly ownValidators: EthAddress[] = [], private log = createLogger('slasher:consensus'), - private readonly metrics = new SlasherMetrics(getTelemetryClient(), ownValidators), + private readonly metrics = new SlasherMetrics(getTelemetryClient()), ) { this.roundMonitor = new SlashRoundMonitor(settings, dateProvider); this.offensesCollector = new SlashOffensesCollector(config, settings, watchers, offensesStore); @@ -115,9 +123,13 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient public async start() { this.log.debug('Starting slasher client...'); - this.roundMonitor.start(); await this.offensesCollector.start(); + // Check for round changes. Registered before the monitor starts so a round boundary crossed while the seed + // below is awaiting L1 is announced rather than silently swallowed. + this.unwatchCallbacks.push(this.roundMonitor.listenToNewRound(round => this.handleNewRound(round))); + this.roundMonitor.start(); + // Listen for RoundExecuted events this.unwatchCallbacks.push( this.slashingProposer.listenToRoundExecuted( @@ -130,17 +142,20 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient // Listen for VoteCast events to warn early when a vote names one of our own validators as a slash target if (this.ownValidators.length > 0) { + this.metrics.recordQuorumSize(this.settings.slashingQuorumSize); + // Seeded before subscribing so no vote is counted by both the seed and the subscription. Votes landing + // between the seed's reads and the subscription attaching are missed; the tally is best-effort. + await this.seedOwnValidatorVotes(); this.unwatchCallbacks.push( this.slashingProposer.listenToVoteCast( - ({ round, proposer }) => - void this.handleVoteCast(round, proposer).catch(err => this.log.error('Error handling vote cast', err)), + ({ round, slot, proposer }) => + void this.handleVoteCast(round, slot, proposer).catch(err => + this.log.error('Error handling vote cast', err), + ), ), ); } - // Check for round changes - this.unwatchCallbacks.push(this.roundMonitor.listenToNewRound(round => this.handleNewRound(round))); - this.log.info(`Started slasher client`); return Promise.resolve(); } @@ -172,7 +187,11 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient /** Triggered on a time basis when we enter a new slashing round. Clears expired offenses. */ protected async handleNewRound(round: bigint) { this.log.info(`Starting new slashing round ${round}`); - this.pruneSelfSlashVoteWarnings(round); + // Strictly greater so the clock catching up to a round a vote event already rolled to cannot wipe the tally, + // and gated on own validators so nodes running none do not emit a meaningless gauge. + if (this.ownValidators.length > 0 && round > this.ownValidatorVotes.round) { + this.rollOwnValidatorVotesTo(round); + } await this.offensesCollector.handleNewRound(round); } @@ -189,43 +208,88 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient amount, l1BlockHash, }); - this.metrics.recordOwnValidatorSlashed(attester, amount); + this.metrics.recordOwnValidatorSlashed(amount); } } - /** Called when we see a VoteCast event. Warns once per round and validator when a vote names one of our own. */ - protected async handleVoteCast(round: bigint, proposer: string) { - const votes = await this.slashingProposer.getLastVote(round, this.settings.slashingAmounts); - for (const { validator, slashAmount } of votes.filter(vote => this.isOwnValidator(vote.validator))) { - const warned = this.warnedSelfSlashVotes.get(round) ?? new Set(); - if (warned.has(validator.toString())) { - continue; - } - warned.add(validator.toString()); - this.warnedSelfSlashVotes.set(round, warned); + /** Called when we see a VoteCast event. Warns for every vote that names one of our own validators. */ + protected async handleVoteCast(round: bigint, slot: SlotNumber, proposer: string) { + const votes = await this.slashingProposer.getLastVote(round); + this.countVotesAgainstOwnValidators(round, votes, { slot, proposer }); + } - this.log.warn(`Detected onchain slashing vote against own validator ${validator}`, { - round, - validator: validator.toString(), - slashAmount, - proposer, - }); - this.metrics.recordOwnValidatorTargeted(validator); + /** + * Reads the votes already cast in the current round so the tally survives a restart mid-round. Without this a + * node restarted partway through a round would report a tally near zero for the rest of it, and never warn. + */ + private async seedOwnValidatorVotes() { + const { round } = this.roundMonitor.getCurrentRound(); + try { + const votes = await this.slashingProposer.getVotesForRound(round); + // The round monitor may have rolled the tally past this round while the votes were being read + if (round > this.ownValidatorVotes.round) { + this.rollOwnValidatorVotesTo(round); + } + for (const vote of votes) { + this.countVotesAgainstOwnValidators(round, vote); + } + } catch (error) { + // A node starting up with no votes yet in the round, or an L1 read failure, must not block the slasher + this.log.warn(`Could not seed slashing votes for round ${round}`, { round, error }); } } - private isOwnValidator(address: EthAddress): boolean { - return this.ownValidators.some(validator => validator.equals(address)); - } + /** Adds a vote's slash targets to the current round tally, warning for each of the node's own validators named. */ + private countVotesAgainstOwnValidators( + round: bigint, + votes: SlashVote, + logContext?: { slot: SlotNumber; proposer: string }, + ) { + if (round < this.ownValidatorVotes.round) { + return; // A vote from a round that has already closed, which can no longer reach quorum + } + if (round > this.ownValidatorVotes.round) { + this.rollOwnValidatorVotesTo(round); + } - /** Drops warning dedupe entries for rounds past their slashing lifetime, as they can no longer receive votes. */ - private pruneSelfSlashVoteWarnings(currentRound: bigint) { - const oldestLiveRound = currentRound - BigInt(this.settings.slashingLifetimeInRounds); - for (const round of this.warnedSelfSlashVotes.keys()) { - if (round < oldestLiveRound) { - this.warnedSelfSlashVotes.delete(round); + // The contract tallies quorum per committee position, so the tally is kept per position, but the warning and + // the targeted metric are per (vote, validator) — reporting a validator once at its highest position tally — + // since an operator cares about the validator, not which of its committee seats is being voted on. + const { countByPosition } = this.ownValidatorVotes; + const targeted = new Map(); + for (const { validator, slashAmount, position } of votes.filter(vote => this.isOwnValidator(vote.validator))) { + const count = (countByPosition.get(position) ?? 0) + 1; + countByPosition.set(position, count); + const entry = targeted.get(validator.toString()); + if (!entry || count > entry.votes) { + targeted.set(validator.toString(), { validator, slashAmount, votes: count }); } } + + const quorum = this.settings.slashingQuorumSize; + for (const { validator, slashAmount, votes: count } of targeted.values()) { + this.metrics.recordOwnValidatorTargeted(); + + // Seeded votes are replayed from L1 rather than observed live, so they carry no event context to log + if (logContext) { + this.log.warn( + `Own validator ${validator} targeted by slashing vote (${count} of ${quorum} votes needed to slash)`, + { round, validator: validator.toString(), votes: count, quorum, slashAmount, ...logContext }, + ); + } + } + + this.metrics.recordCurrentRoundVotesMax(Math.max(0, ...countByPosition.values())); + } + + /** Starts a fresh tally for a round, zeroing the gauge so a quiet round does not leave the previous one's value. */ + private rollOwnValidatorVotesTo(round: bigint) { + this.ownValidatorVotes = { round, countByPosition: new Map() }; + this.metrics.recordCurrentRoundVotesMax(0); + } + + private isOwnValidator(address: EthAddress): boolean { + return this.ownValidators.some(validator => validator.equals(address)); } /** diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 03f58f5470f..dc12ba8d346 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -551,7 +551,21 @@ export const SLASHER_ROUND_EXECUTED_COUNT: MetricDefinition = { export const SLASHER_OWN_VALIDATOR_TARGETED_COUNT: MetricDefinition = { name: 'aztec.slasher.own_validator.targeted_count', description: - "The number of times one of the node's own validators was named as a slash target by an onchain vote, deduplicated per round and validator", + "The number of times one of the node's own validators was named as a slash target by an onchain vote, counted " + + 'once per vote per validator. Best-effort: delayed L1 log delivery can miss votes or count one twice.', + valueType: ValueType.INT, +}; +export const SLASHER_OWN_VALIDATOR_CURRENT_ROUND_VOTES_MAX: MetricDefinition = { + name: 'aztec.slasher.own_validator.current_round_votes_max', + description: + "Highest number of votes cast against any committee position held by the node's own validators in the current " + + 'slashing round, to compare against aztec.slasher.quorum_size (the contract tallies quorum per position). ' + + 'Resets to zero when a new round starts. Best-effort: delayed L1 log delivery can miss votes or count one twice.', + valueType: ValueType.INT, +}; +export const SLASHER_QUORUM_SIZE: MetricDefinition = { + name: 'aztec.slasher.quorum_size', + description: 'The number of votes a validator must receive in a round to be slashed', valueType: ValueType.INT, }; export const SLASHER_OWN_VALIDATOR_SLASHED_COUNT: MetricDefinition = { @@ -561,8 +575,10 @@ export const SLASHER_OWN_VALIDATOR_SLASHED_COUNT: MetricDefinition = { }; export const SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT: MetricDefinition = { name: 'aztec.slasher.own_validator.slashed_amount', - description: "Cumulative amount slashed from the node's own validators, from executed Slashed events", - unit: 'eth', + description: + "Cumulative amount slashed from the node's own validators in whole staking-asset tokens, from executed " + + 'Slashed events', + unit: 'tokens', valueType: ValueType.DOUBLE, }; export const SEQUENCER_CHECKPOINT_SUCCESS_COUNT: MetricDefinition = {