feat(slasher): warn and expose metrics when own validators are slash targets

yarn-project/ethereum/src/contracts/slashing_proposer.test.ts

@@ -17,7 +17,7 @@ import { createExtendedL1Client } from '../client.js';
 import { DefaultL1ContractsConfig } from '../config.js';
 import { type DeployAztecL1ContractsArgs, deployAztecL1Contracts } from '../deploy_aztec_l1_contracts.js';
 import type { Anvil } from '../test/start_anvil.js';
-import { RollupContract, decodeSlashConsensusVotes } from './index.js';
+import { RollupContract, collapseVoteCastLogs, decodeSlashConsensusVotes } from './index.js';
 import { SlashingProposerContract } from './slashing_proposer.js';
 
 describe('SlashingProposer', () => {
@@ -290,4 +290,47 @@ describe('SlashingProposer', () => {
       expect(votes).toEqual([0, 0, 1, 2, 3, 2, 1, 0]);
     });
   });
+
+  describe('collapseVoteCastLogs', () => {
+    const voteCastLog = (round: bigint, slot: bigint, proposer: string) => ({ args: { round, slot, proposer } });
+
+    it('passes a single log through', () => {
+      const collapsed = collapseVoteCastLogs([voteCastLog(5n, 100n, '0xaaa')]);
+
+      expect(collapsed).toEqual([{ round: 5n, slot: SlotNumber(100), proposer: '0xaaa' }]);
+    });
+
+    it('keeps only the last log of a round', () => {
+      const collapsed = collapseVoteCastLogs([voteCastLog(5n, 100n, '0xaaa'), voteCastLog(5n, 101n, '0xbbb')]);
+
+      expect(collapsed).toEqual([{ round: 5n, slot: SlotNumber(101), proposer: '0xbbb' }]);
+    });
+
+    it('keeps the last log of every round when a batch spans a round boundary', () => {
+      const collapsed = collapseVoteCastLogs([
+        voteCastLog(5n, 100n, '0xaaa'),
+        voteCastLog(5n, 101n, '0xbbb'),
+        voteCastLog(6n, 102n, '0xccc'),
+      ]);
+
+      expect(collapsed).toEqual([
+        { round: 5n, slot: SlotNumber(101), proposer: '0xbbb' },
+        { round: 6n, slot: SlotNumber(102), proposer: '0xccc' },
+      ]);
+    });
+
+    it('returns nothing for an empty batch', () => {
+      expect(collapseVoteCastLogs([])).toEqual([]);
+    });
+
+    it('drops logs with undecoded args', () => {
+      const collapsed = collapseVoteCastLogs([
+        { args: { round: undefined, slot: 100n, proposer: '0xaaa' } },
+        { args: { round: 5n, slot: undefined, proposer: '0xaaa' } },
+        { args: { round: 5n, slot: 100n, proposer: undefined } },
+      ]);
+
+      expect(collapsed).toEqual([]);
+    });
+  });
 });

yarn-project/ethereum/src/contracts/slashing_proposer.ts

@@ -3,6 +3,8 @@ import type { ViemClient } from '@aztec/ethereum/types';
 import { mergeAbis, tryExtractEvent } from '@aztec/ethereum/utils';
 import { SlotNumber } from '@aztec/foundation/branded-types';
 import { Buffer32 } from '@aztec/foundation/buffer';
+import { chunk, times } from '@aztec/foundation/collection';
+import { memoize } from '@aztec/foundation/decorators';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { Signature } from '@aztec/foundation/eth-signature';
 import { hexToBuffer } from '@aztec/foundation/string';
@@ -64,6 +66,12 @@ export class SlashingProposerContract {
     return this.contract.read.EXECUTION_DELAY_IN_ROUNDS();
   }
 
+  /**
+   * Returns the slash amounts for the three slash unit levels, which are immutable on the contract.
+   * Memoized: the first call's promise is cached for the instance's lifetime, including a rejection, so make an
+   * awaited call during startup before relying on this from event handlers.
+   */
+  @memoize
   public getSlashingAmounts(): Promise<[bigint, bigint, bigint]> {
     return Promise.all([
       this.contract.read.SLASH_AMOUNT_SMALL(),
@@ -234,36 +242,61 @@ export class SlashingProposerContract {
     };
   }
 
+  /** Returns the validators eligible to be voted against in a round, in the order votes encode them */
+  public async getSlashTargetValidators(round: bigint): Promise<EthAddress[]> {
+    const { result } = await this.contract.simulate.getSlashTargetCommittees([round]);
+    return result.flat().map(validator => EthAddress.fromString(validator));
+  }
+
+  /**
+   * Returns the slash amount voted for each target validator by a single vote of a round.
+   * @param index - Position of the vote within the round, from 0 (inclusive) to the round's vote count (exclusive)
+   */
+  public async getVoteAt(round: bigint, index: bigint): Promise<SlashVote> {
+    const [validators, vote, slashAmounts] = await Promise.all([
+      this.getSlashTargetValidators(round),
+      this.contract.read.getVotes([round, index]),
+      this.getSlashingAmounts(),
+    ]);
+    return decodeVote(vote, validators, slashAmounts);
+  }
+
+  /**
+   * Returns every vote cast so far in a round, oldest first. The target validators and slash amounts are read once
+   * for the whole round rather than per vote, so this is much cheaper than reading each vote individually.
+   */
+  public async getVotesForRound(round: bigint): Promise<SlashVote[]> {
+    const [{ voteCount }, validators, slashAmounts] = await Promise.all([
+      this.getRound(round),
+      this.getSlashTargetValidators(round),
+      this.getSlashingAmounts(),
+    ]);
+    // A round can hold as many votes as it has slots, too many for a single parallel burst of reads
+    const votes: Hex[] = [];
+    for (const indices of chunk(times(Number(voteCount), BigInt), VOTE_READ_BATCH_SIZE)) {
+      votes.push(...(await Promise.all(indices.map(index => this.contract.read.getVotes([round, index])))));
+    }
+    return votes.map(vote => decodeVote(vote, validators, slashAmounts));
+  }
+
   /** Returns the last vote emitted for a given round  */
   public async getLastVote(round: bigint) {
     const { voteCount } = await this.getRound(round);
-    const validators = (await this.contract.simulate.getSlashTargetCommittees([round])).result.flat();
-    const vote = await this.contract.read.getVotes([round, voteCount - 1n]);
-    const decoded = decodeSlashConsensusVotes(hexToBuffer(vote));
-    const slashAmounts = await this.getSlashingAmounts();
-    return decoded
-      .map((units, i) => ({
-        validator: EthAddress.fromString(validators[i]),
-        slashAmount: slashAmounts[units - 1] ?? 0n,
-      }))
-      .filter(v => v.slashAmount > 0n);
+    return await this.getVoteAt(round, voteCount - 1n);
   }
 
   /**
    * Listen for VoteCast events
    * @param callback - Callback function to handle vote cast events
    * @returns Unwatch function
    */
-  public listenToVoteCast(callback: (args: { round: bigint; proposer: string }) => void): () => void {
+  public listenToVoteCast(callback: (args: VoteCastEventArgs) => void): () => void {
     return this.contract.watchEvent.VoteCast(
       {},
       {
         onLogs: logs => {
-          for (const log of logs) {
-            const { round, proposer } = log.args;
-            if (round !== undefined && proposer) {
-              callback({ round, proposer });
-            }
+          for (const args of collapseVoteCastLogs(logs)) {
+            callback(args);
           }
         },
       },
@@ -294,6 +327,49 @@ export class SlashingProposerContract {
   }
 }
 
+/** Maximum number of parallel getVotes reads when fetching a whole round. */
+const VOTE_READ_BATCH_SIZE = 32;
+
+/**
+ * The validators a single slashing vote targets, with the amount voted for each. The position is the validator's
+ * index in the round's flattened slash target committees — the unit the contract tallies quorum by. A validator
+ * sitting in several of the round's committees holds several positions, each with its own tally.
+ */
+export type SlashVote = { validator: EthAddress; slashAmount: bigint; position: number }[];
+
+function decodeVote(vote: Hex, validators: EthAddress[], slashAmounts: [bigint, bigint, bigint]): SlashVote {
+  return decodeSlashConsensusVotes(hexToBuffer(vote))
+    .map((units, position) => ({
+      validator: validators[position],
+      slashAmount: slashAmounts[units - 1] ?? 0n,
+      position,
+    }))
+    .filter(v => v.slashAmount > 0n);
+}
+
+/** Arguments decoded from a VoteCast event. */
+export type VoteCastEventArgs = { round: bigint; slot: SlotNumber; proposer: string };
+
+/**
+ * Collapses a batch of VoteCast logs down to the latest log of each round.
+ *
+ * A vote is resolved by reading the round's most recent entry, so acting on every log in a batch would read that
+ * same entry once per log: the newest vote gets counted repeatedly while the ones behind it are never read at all.
+ * Batches only occur when L1 log delivery falls behind, but the resulting miscount lasts for the rest of the round.
+ */
+export function collapseVoteCastLogs(
+  logs: { args: { round?: bigint; slot?: bigint; proposer?: string } }[],
+): VoteCastEventArgs[] {
+  const latestPerRound = new Map<bigint, VoteCastEventArgs>();
+  for (const { args } of logs) {
+    const { round, slot, proposer } = args;
+    if (round !== undefined && slot !== undefined && proposer) {
+      latestPerRound.set(round, { round, slot: SlotNumber.fromBigInt(slot), proposer });
+    }
+  }
+  return [...latestPerRound.values()];
+}
+
 /**
  * Decodes a Buffer containing slash votes back into an array of numbers.
  * Each vote is represented as a 2-bit value (0, 1, 2, or 3) representing slashing units.

yarn-project/slasher/src/factory/create_facade.ts

@@ -25,7 +25,10 @@ export async function createSlasherFacade(
   watchers: Watcher[],
   dateProvider: DateProvider,
   epochCache: EpochCache,
-  /** List of own validator addresses to add to the slashValidatorNever list unless slashSelfAllowed is true */
+  /**
+   * List of own validator addresses. Added to the slashValidatorNever list unless slashSelfAllowed is true, and used
+   * (independently of slashSelfAllowed) to warn and record metrics when they are targeted by onchain slashing.
+   */
   validatorAddresses: EthAddress[] = [],
   logger = createLogger('slasher'),
 ): Promise<SlasherClientInterface> {
@@ -76,6 +79,7 @@ export async function createSlasherFacade(
     dateProvider,
     kvStore,
     rollupRegisteredAtL2Slot,
+    validatorAddresses,
     logger,
   );
 }

yarn-project/slasher/src/factory/create_implementation.ts

@@ -2,6 +2,7 @@ import { EpochCache } from '@aztec/epoch-cache';
 import { RollupContract, SlashingProposerContract } from '@aztec/ethereum/contracts';
 import type { ViemClient } from '@aztec/ethereum/types';
 import type { SlotNumber } from '@aztec/foundation/branded-types';
+import type { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
 import { DateProvider } from '@aztec/foundation/timer';
 import { AztecLMDBStoreV2 } from '@aztec/kv-store/lmdb-v2';
@@ -24,6 +25,7 @@ export async function createSlasherImplementation(
   dateProvider: DateProvider,
   kvStore: AztecLMDBStoreV2,
   rollupRegisteredAtL2Slot: SlotNumber,
+  ownValidators: EthAddress[] = [],
   logger = createLogger('slasher'),
 ) {
   const proposer = await rollup.getSlashingProposer();
@@ -39,6 +41,7 @@ export async function createSlasherImplementation(
       epochCache,
       kvStore,
       rollupRegisteredAtL2Slot,
+      ownValidators,
       logger,
     );
   }
@@ -53,6 +56,7 @@ async function createSlasher(
   epochCache: EpochCache,
   kvStore: AztecLMDBStoreV2,
   rollupRegisteredAtL2Slot: SlotNumber,
+  ownValidators: EthAddress[] = [],
   logger = createLogger('slasher'),
 ): Promise<SlasherClient> {
   const settings = { ...(await getSlasherSettings(rollup, slashingProposer)), rollupRegisteredAtL2Slot };
@@ -73,6 +77,7 @@ async function createSlasher(
     epochCache,
     dateProvider,
     offensesStore,
+    ownValidators,
     logger,
   );
 }

yarn-project/slasher/src/metrics.ts

@@ -1,19 +1,59 @@
 import {
+  type Gauge,
   Metrics,
   type TelemetryClient,
   type UpDownCounter,
   createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
+import { formatEther } from 'viem/utils';
+
 export class SlasherMetrics {
   private readonly roundExecuted: UpDownCounter;
+  private readonly ownValidatorTargeted: UpDownCounter;
+  private readonly ownValidatorSlashedCount: UpDownCounter;
+  private readonly ownValidatorSlashedAmount: UpDownCounter;
+  private readonly ownValidatorCurrentRoundVotesMax: Gauge;
+  private readonly quorumSize: Gauge;
 
   constructor(client: TelemetryClient, name = 'Slasher') {
     const meter = client.getMeter(name);
     this.roundExecuted = createUpDownCounterWithDefault(meter, Metrics.SLASHER_ROUND_EXECUTED_COUNT);
+    this.ownValidatorTargeted = createUpDownCounterWithDefault(meter, Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT);
+    this.ownValidatorSlashedCount = createUpDownCounterWithDefault(meter, Metrics.SLASHER_OWN_VALIDATOR_SLASHED_COUNT);
+    this.ownValidatorSlashedAmount = createUpDownCounterWithDefault(
+      meter,
+      Metrics.SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT,
+    );
+    this.ownValidatorCurrentRoundVotesMax = meter.createGauge(Metrics.SLASHER_OWN_VALIDATOR_CURRENT_ROUND_VOTES_MAX);
+    this.quorumSize = meter.createGauge(Metrics.SLASHER_QUORUM_SIZE);
   }
 
   public recordRoundExecuted(): void {
     this.roundExecuted.add(1);
   }
+
+  /** Records the quorum a validator must reach in a round to be slashed, so dashboards can plot the threshold. */
+  public recordQuorumSize(quorum: number): void {
+    this.quorumSize.record(quorum);
+  }
+
+  /** Records that an onchain slashing vote named one of the node's own validators as a target. */
+  public recordOwnValidatorTargeted(): void {
+    this.ownValidatorTargeted.add(1);
+  }
+
+  /**
+   * Records how close the most-voted committee position held by the node's own validators is to quorum this round.
+   * Recorded as an absolute value rather than a delta so a vote seen across a round rollover cannot make it drift.
+   */
+  public recordCurrentRoundVotesMax(votes: number): void {
+    this.ownValidatorCurrentRoundVotesMax.record(votes);
+  }
+
+  /** Records an executed slash against one of the node's own validators. */
+  public recordOwnValidatorSlashed(amount: bigint): void {
+    this.ownValidatorSlashedCount.add(1);
+    this.ownValidatorSlashedAmount.add(parseFloat(formatEther(amount)));
+  }
 }