feat: merge-train/spartan

release-image/Dockerfile.dockerignore

@@ -30,6 +30,7 @@
 !/yarn-project/noir-protocol-circuits-types/artifacts/
 !/yarn-project/protocol-contracts/artifacts/
 !/yarn-project/standard-contracts/artifacts/
+!/yarn-project/standard-contracts/artifacts-historical/
 !/yarn-project/noir-contracts.js/artifacts/
 !/yarn-project/noir-test-contracts.js/artifacts/
 !/yarn-project/simulator/artifacts/

yarn-project/.gitignore

@@ -61,6 +61,7 @@ cli-wallet/test/data
 cli/src/config/generated
 ethereum/src/generated
 slasher/src/generated
+tmp/
 
 .claude/settings.local.json
 .yarn/*

yarn-project/archiver/src/store/contract_instance_store.ts

@@ -1,4 +1,5 @@
 import type { Fr } from '@aztec/foundation/curves/bn254';
+import { first } from '@aztec/foundation/iterable';
 import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
 import { isProtocolContract } from '@aztec/protocol-contracts';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
@@ -134,19 +135,18 @@ export class ContractInstanceStore {
 
   async getCurrentContractInstanceClassId(address: AztecAddress, timestamp: UInt64, originalClassId: Fr): Promise<Fr> {
     // We need to find the last update before the given timestamp
-    const queryResult = await this.#contractInstanceUpdates
-      .valuesAsync({
+    const serializedUpdate = await first(
+      this.#contractInstanceUpdates.valuesAsync({
         reverse: true,
         start: this.getUpdateKey(address, 0n), // Make sure we only look at updates for this contract
         end: this.getUpdateKey(address, timestamp + 1n), // No update can match this key since it doesn't have a log index. We want the highest key <= timestamp
         limit: 1,
-      })
-      .next();
-    if (queryResult.done) {
+      }),
+    );
+    if (serializedUpdate === undefined) {
       return originalClassId;
     }
 
-    const serializedUpdate = queryResult.value;
     const update = SerializableContractInstanceUpdate.fromBuffer(serializedUpdate);
     if (timestamp < update.timestampOfChange) {
       return update.prevContractClassId.isZero() ? originalClassId : update.prevContractClassId;

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -16,6 +16,7 @@ import {
 import { compactArray, pick, unique } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
+import { first } from '@aztec/foundation/iterable';
 import { BadRequestError } from '@aztec/foundation/json-rpc';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { retryUntil } from '@aztec/foundation/retry';
@@ -463,11 +464,8 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb
   }
 
   public async getMaxPriorityFees(): Promise<GasFees> {
-    for await (const tx of this.p2pClient.iteratePendingTxs({ includeProof: false })) {
-      return tx.getGasSettings().maxPriorityFeesPerGas;
-    }
-
-    return GasFees.from({ feePerDaGas: 0n, feePerL2Gas: 0n });
+    const tx = await first(this.p2pClient.iteratePendingTxs({ includeProof: false }));
+    return tx ? tx.getGasSettings().maxPriorityFeesPerGas : GasFees.from({ feePerDaGas: 0n, feePerL2Gas: 0n });
   }
 
   /**

yarn-project/bb-prover/src/verifier/batch_chonk_verifier.ts

@@ -6,6 +6,7 @@ import { Timer } from '@aztec/foundation/timer';
 import { ProtocolCircuitVks } from '@aztec/noir-protocol-circuits-types/server/vks';
 import type { ClientProtocolCircuitVerifier, IVCProofVerificationResult } from '@aztec/stdlib/interfaces/server';
 import type { Tx } from '@aztec/stdlib/tx';
+import { getTelemetryClient } from '@aztec/telemetry-client';
 
 import { Unpackr } from 'msgpackr';
 import { execFile } from 'node:child_process';
@@ -16,6 +17,7 @@ import * as path from 'node:path';
 import { promisify } from 'node:util';
 
 import type { BBConfig } from '../config.js';
+import { IVCVerifierMetrics } from './queued_chonk_verifier.js';
 
 const execFileAsync = promisify(execFile);
 const RESULT_TIMEOUT_MS = 5 * 60 * 1000;
@@ -58,6 +60,7 @@ export class BatchChonkVerifier implements ClientProtocolCircuitVerifier {
   private sendQueue: SerialQueue;
   private fifoReader: FifoFrameReader;
   private logger = createLogger('bb-prover:batch_chonk_verifier');
+  private metrics: IVCVerifierMetrics;
   /** Maps artifact name to VK index in the batch verifier. */
   private vkIndexMap = new Map<string, number>();
   /** Bound cleanup handler for process exit signals. */
@@ -75,6 +78,7 @@ export class BatchChonkVerifier implements ClientProtocolCircuitVerifier {
     this.fifoReader = new FifoFrameReader();
     this.sendQueue = new SerialQueue();
     this.sendQueue.start(1);
+    this.metrics = new IVCVerifierMetrics(getTelemetryClient(), `BatchChonkVerifier-${label}`);
   }
 
   /** Create and start a BatchChonkVerifier using the protocol circuit VKs. */
@@ -311,6 +315,7 @@ export class BatchChonkVerifier implements ClientProtocolCircuitVerifier {
     const totalDurationMs = pending.totalTimer.ms();
 
     const ivcResult: IVCProofVerificationResult = { valid, durationMs, totalDurationMs };
+    this.metrics.recordIVCVerification(ivcResult);
 
     if (!valid) {
       this.logger.warn(`Proof verification failed for request_id=${result.request_id}: ${result.error_message}`);

yarn-project/bb-prover/src/verifier/queued_chonk_verifier.ts

@@ -16,7 +16,8 @@ import {
 
 import { createHistogram } from 'node:perf_hooks';
 
-class IVCVerifierMetrics {
+/** Records verification timing and failure metrics for an IVC (chonk) proof verifier. */
+export class IVCVerifierMetrics {
   private ivcVerificationHistogram: Histogram;
   private ivcTotalVerificationHistogram: Histogram;
   private ivcFailureCount: UpDownCounter;

yarn-project/end-to-end/src/spartan/n_tps.test.ts

@@ -38,6 +38,7 @@ import {
   getRPCEndpoint,
   hasDeployedHelmRelease,
   installChaosMeshChart,
+  logGossipTxValidationMetrics,
   setupEnvironment,
   startPortForwardForPrometeheus,
   uninstallChaosMesh,
@@ -127,6 +128,15 @@ describe('sustained N TPS test', () => {
 
   afterAll(async () => {
     logger.info('Collecting benchmark metrics and cleaning up...');
+
+    // Log the gossip tx validation breakdown (per-stage timings, tx pool queue stats, chonk verifier
+    // timings) so slow gossip validations observed during the run can be attributed to a stage.
+    if (prometheusClient) {
+      await logGossipTxValidationMetrics(prometheusClient, config.NAMESPACE, TEST_DURATION_SECONDS + 60, logger).catch(
+        err => logger.warn(`Failed to scrape gossip validation metrics: ${err}`, { err }),
+      );
+    }
+
     if (process.env.BENCH_OUTPUT) {
       for (const topic of Object.values(TopicType)) {
         try {

yarn-project/end-to-end/src/spartan/n_tps_prove.test.ts

@@ -35,6 +35,7 @@ import {
   type ServiceEndpoint,
   getEthereumEndpoint,
   getExternalIP,
+  logGossipTxValidationMetrics,
   setupEnvironment,
   startPortForwardForPrometeheus,
 } from './utils.js';
@@ -148,6 +149,15 @@ describe(`prove ${TARGET_TPS}TPS test`, () => {
         server: new URL(`http://127.0.0.1:${freshPromForward.port}`),
       });
 
+      // Log the gossip tx validation breakdown (per-stage timings, tx pool queue stats, chonk
+      // verifier timings) so slow gossip validations during the run can be attributed to a stage.
+      await logGossipTxValidationMetrics(
+        prometheusClient,
+        config.NAMESPACE,
+        epochDurationSeconds + SLOTS_BUFFER * slotDurationSeconds,
+        logger,
+      ).catch(err => logger.warn(`Failed to scrape gossip validation metrics: ${err}`, { err }));
+
       const endSnapshot = await captureMetricsSnapshot(prometheusClient, logger);
 
       // Helper to compute delta, clamping negative values to 0 (handles pod restarts)

yarn-project/end-to-end/src/spartan/utils/gossip_metrics.ts

@@ -0,0 +1,108 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import type { PrometheusClient } from '../../quality_of_service/prometheus_client.js';
+
+/** Runs a PromQL query and returns each series as a label-record plus value. Returns [] on error. */
+async function queryVector(
+  prometheus: PrometheusClient,
+  query: string,
+  logger: Logger,
+): Promise<{ labels: Record<string, string>; value: number }[]> {
+  try {
+    const resp = await prometheus.queryRaw(query);
+    if (resp.status !== 'success' || resp.data.resultType !== 'vector') {
+      logger.warn(`Unexpected Prometheus response for query`, { query, resp });
+      return [];
+    }
+    return resp.data.result.map(({ metric, value }) => ({
+      labels: (metric ?? {}) as Record<string, string>,
+      value: parseFloat(value[1]),
+    }));
+  } catch (err) {
+    logger.warn(`Failed to run Prometheus query: ${err}`, { query });
+    return [];
+  }
+}
+
+/** Formats a grouped vector result as a { labelValue: value } record for structured logging. */
+function toRecord(series: { labels: Record<string, string>; value: number }[], label: string): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const { labels, value } of series) {
+    out[labels[label] ?? 'unknown'] = Math.round(value * 100) / 100;
+  }
+  return out;
+}
+
+/**
+ * Scrapes and logs gossip tx validation timing breakdowns from Prometheus: per-stage validation
+ * durations, tx pool serial queue wait/execution times, and chonk (IVC) proof verifier timings.
+ * Used by the spartan TPS benchmarks to attribute slow gossip validations to a specific stage.
+ */
+export async function logGossipTxValidationMetrics(
+  prometheus: PrometheusClient,
+  namespace: string,
+  windowSeconds: number,
+  logger: Logger,
+): Promise<void> {
+  const ns = `k8s_namespace_name="${namespace}"`;
+  const window = `[${Math.max(60, Math.ceil(windowSeconds))}s]`;
+
+  const stageQuantile = (perc: string) =>
+    `histogram_quantile(${perc}, sum(rate(aztec_p2p_gossip_tx_validation_stage_duration_milliseconds_bucket{${ns}}${window})) by (le, aztec_p2p_tx_validation_stage))`;
+  const stageAvg = () =>
+    `sum(rate(aztec_p2p_gossip_tx_validation_stage_duration_milliseconds_sum{${ns}}${window})) by (aztec_p2p_tx_validation_stage) / ` +
+    `sum(rate(aztec_p2p_gossip_tx_validation_stage_duration_milliseconds_count{${ns}}${window})) by (aztec_p2p_tx_validation_stage)`;
+  const validationQuantile = (perc: string) =>
+    `histogram_quantile(${perc}, sum(rate(aztec_p2p_gossip_message_validation_duration_milliseconds_bucket{${ns}}${window})) by (le, aztec_gossip_topic_name))`;
+  const queueQuantile = (metric: string, perc: string) =>
+    `topk(10, histogram_quantile(${perc}, sum(rate(${metric}{${ns}}${window})) by (le, aztec_mempool_operation)))`;
+  const ivcQuantile = (metric: string, perc: string) =>
+    `histogram_quantile(${perc}, sum(rate(${metric}{${ns}}${window})) by (le))`;
+
+  const stageLabel = 'aztec_p2p_tx_validation_stage';
+  const topicLabel = 'aztec_gossip_topic_name';
+  const operationLabel = 'aztec_mempool_operation';
+
+  const [stageP50, stageP95, stageAvgs, validationP95, slowCount] = await Promise.all([
+    queryVector(prometheus, stageQuantile('0.50'), logger),
+    queryVector(prometheus, stageQuantile('0.95'), logger),
+    queryVector(prometheus, stageAvg(), logger),
+    queryVector(prometheus, validationQuantile('0.95'), logger),
+    queryVector(prometheus, `sum(aztec_p2p_gossip_slow_validation_count{${ns}}) by (${topicLabel})`, logger),
+  ]);
+
+  logger.info('Gossip tx validation stage timings (ms)', {
+    stageP50: toRecord(stageP50, stageLabel),
+    stageP95: toRecord(stageP95, stageLabel),
+    stageAvg: toRecord(stageAvgs, stageLabel),
+    validationP95ByTopic: toRecord(validationP95, topicLabel),
+    slowValidationCountByTopic: toRecord(slowCount, topicLabel),
+  });
+
+  const [queueWaitP95, queueExecutionP95, queueLengthMax] = await Promise.all([
+    queryVector(prometheus, queueQuantile('aztec_mempool_tx_pool_v2_queue_wait_milliseconds_bucket', '0.95'), logger),
+    queryVector(
+      prometheus,
+      queueQuantile('aztec_mempool_tx_pool_v2_queue_execution_milliseconds_bucket', '0.95'),
+      logger,
+    ),
+    queryVector(prometheus, `max(max_over_time(aztec_mempool_tx_pool_v2_queue_length{${ns}}${window}))`, logger),
+  ]);
+
+  logger.info('Tx pool serial queue stats (ms)', {
+    queueWaitP95: toRecord(queueWaitP95, operationLabel),
+    queueExecutionP95: toRecord(queueExecutionP95, operationLabel),
+    queueLengthMax: queueLengthMax[0]?.value,
+  });
+
+  const [ivcVerifyP95, ivcTotalP95] = await Promise.all([
+    queryVector(prometheus, ivcQuantile('aztec_ivc_verifier_time_milliseconds_bucket', '0.95'), logger),
+    queryVector(prometheus, ivcQuantile('aztec_ivc_verifier_total_time_milliseconds_bucket', '0.95'), logger),
+  ]);
+
+  // A large gap between total (queue + verify) and verify indicates a pile-up in the verifier queue.
+  logger.info('Chonk (IVC) proof verifier timings (ms)', {
+    verifyP95: ivcVerifyP95[0]?.value,
+    totalP95: ivcTotalP95[0]?.value,
+  });
+}

yarn-project/end-to-end/src/spartan/utils/index.ts

@@ -69,3 +69,6 @@ export { ChainHealth, type ChainHealthSnapshot } from './health.js';
 
 // Pod log extraction
 export { type BlockBuiltLogEntry, fetchBlockBuiltLogs } from './pod_logs.js';
+
+// Gossip validation metrics scraping
+export { logGossipTxValidationMetrics } from './gossip_metrics.js';

yarn-project/ethereum/src/deploy_l1_contract.ts

@@ -26,7 +26,7 @@ import { RegisterNewRollupVersionPayloadArtifact } from './l1_artifacts.js';
 import { type L1TxUtilsConfig, getL1TxUtilsConfigEnvVars } from './l1_tx_utils/config.js';
 import { createL1TxUtils } from './l1_tx_utils/factory.js';
 import type { L1TxUtils } from './l1_tx_utils/l1_tx_utils.js';
-import type { GasPrice, L1TxConfig, L1TxRequest } from './l1_tx_utils/types.js';
+import type { FeesPerGas, L1TxConfig, L1TxRequest } from './l1_tx_utils/types.js';
 import type { ExtendedViemWalletClient } from './types.js';
 import { formatViemError } from './utils.js';
 
@@ -131,11 +131,11 @@ export class L1Deployer {
   sendTransaction(
     tx: L1TxRequest,
     options?: L1TxConfig,
-  ): Promise<{ txHash: Hex; gasLimit: bigint; gasPrice: GasPrice }> {
+  ): Promise<{ txHash: Hex; gasLimit: bigint; feesPerGas: FeesPerGas }> {
     return this.l1TxUtils.sendTransaction(tx, options).then(({ txHash, state }) => ({
       txHash,
       gasLimit: state.gasLimit,
-      gasPrice: state.gasPrice,
+      feesPerGas: state.feesPerGas,
     }));
   }
 }

yarn-project/ethereum/src/l1_tx_utils/README.md

@@ -128,7 +128,7 @@ def send_and_monitor_tx(tx_request):
 
     # Speed up if stalled and have retries left
     if time_since_last_sent > stall_time and attempts < max_attempts:
-      replacement_tx = make_tx(tx_request, nonce, bump_gas_price(state.gasPrice))
+      replacement_tx = make_tx(tx_request, nonce, bump_fees_per_gas(state.feesPerGas))
       state.status = is_cancel_tx ? 'cancelled' : 'speed-up'
       state.tx_hashes.push(send_tx(replacement_tx))
       state.last_sent_at = now
@@ -144,7 +144,7 @@ def attempt_tx_cancellation(state):
     return
 
   # Send noop tx with same nonce but higher gas
-  cancel_tx = make_noop_tx(state.nonce, bump_gas_price(state.gasPrice))
+  cancel_tx = make_noop_tx(state.nonce, bump_fees_per_gas(state.feesPerGas))
   state.cancelTxHashes.push(send_tx(cancel_tx))
   state.status = 'cancelled'
 

yarn-project/ethereum/src/l1_tx_utils/gas_price_ladder.test.ts → yarn-project/ethereum/src/l1_tx_utils/fees_per_gas_ladder.test.ts

@@ -19,7 +19,7 @@ import { type L1TxRequest, L1TxTimeoutError } from './types.js';
 
 const MNEMONIC = 'test test test test test test test test test test test junk';
 const WEI_CONST = 1_000_000_000n;
-const logger = createLogger('ethereum:test:gas-price-ladder');
+const logger = createLogger('ethereum:test:fees-per-gas-ladder');
 
 const REQUEST: L1TxRequest = {
   to: '0x1234567890123456789012345678901234567890',
@@ -49,7 +49,7 @@ async function runUntilTimeout(gasUtils: L1TxUtils, cheatCodes: EthCheatCodes):
   return promise;
 }
 
-describe('L1TxUtils gas-price ladder capture (integration)', () => {
+describe('L1TxUtils fees-per-gas ladder capture (integration)', () => {
   const initialBaseFee = WEI_CONST; // 1 gwei
   let l1Client: ExtendedViemWalletClient;
   let anvil: Anvil;
@@ -96,7 +96,7 @@ describe('L1TxUtils gas-price ladder capture (integration)', () => {
     await anvil.stop().catch(err => createLogger('cleanup').error(err));
   }, 10_000);
 
-  it('captures the escalating gas-price ladder and surfaces it on timeout', async () => {
+  it('captures the escalating fees-per-gas ladder and surfaces it on timeout', async () => {
     const gasUtils = makeGasUtils({ maxSpeedUpAttempts: 3 });
 
     const err = await runUntilTimeout(gasUtils, cheatCodes);
@@ -107,18 +107,18 @@ describe('L1TxUtils gas-price ladder capture (integration)', () => {
     const { txState } = err as L1TxTimeoutError;
 
     // Initial send plus at least one speed-up.
-    expect(txState.gasPriceHistory).toBeDefined();
-    expect(txState.gasPriceHistory!.length).toBeGreaterThanOrEqual(2);
+    expect(txState.feesPerGasHistory).toBeDefined();
+    expect(txState.feesPerGasHistory!.length).toBeGreaterThanOrEqual(2);
 
     // Priority fees strictly escalate across the ladder.
-    const priorityFees = txState.gasPriceHistory!.map(g => g.maxPriorityFeePerGas);
+    const priorityFees = txState.feesPerGasHistory!.map(g => g.maxPriorityFeePerGas);
     for (let i = 1; i < priorityFees.length; i++) {
       expect(priorityFees[i]).toBeGreaterThan(priorityFees[i - 1]);
     }
 
     // attempts == number of sent txs == ladder length; other snapshot fields are populated.
-    expect(txState.attempts).toBe(txState.gasPriceHistory!.length);
-    expect(txState.finalGasPrice.maxPriorityFeePerGas).toBe(priorityFees[priorityFees.length - 1]);
+    expect(txState.attempts).toBe(txState.feesPerGasHistory!.length);
+    expect(txState.finalFeesPerGas.maxPriorityFeePerGas).toBe(priorityFees[priorityFees.length - 1]);
     expect(txState.gasLimit).toBeGreaterThan(0n);
     expect(txState.nonce).toBeGreaterThanOrEqual(0);
 

yarn-project/ethereum/src/l1_tx_utils/l1_tx_timeout_error.test.ts

@@ -4,11 +4,11 @@ import { L1TxTimeoutError, type TimedOutTxState } from './types.js';
 
 describe('L1TxTimeoutError', () => {
   const txState: TimedOutTxState = {
-    gasPriceHistory: [
+    feesPerGasHistory: [
       { maxFeePerGas: 100n, maxPriorityFeePerGas: 1n },
       { maxFeePerGas: 120n, maxPriorityFeePerGas: 2n },
     ],
-    finalGasPrice: { maxFeePerGas: 120n, maxPriorityFeePerGas: 2n },
+    finalFeesPerGas: { maxFeePerGas: 120n, maxPriorityFeePerGas: 2n },
     attempts: 2,
     nonce: 7,
     gasLimit: 21_000n,
@@ -22,7 +22,7 @@ describe('L1TxTimeoutError', () => {
     expect(err instanceof Error).toBe(true);
   });
 
-  it('carries the gas-price ladder snapshot and message', () => {
+  it('carries the fees-per-gas ladder snapshot and message', () => {
     const err = new L1TxTimeoutError('timed out', txState);
     expect(err.message).toBe('timed out');
     expect(err.txState).toEqual(txState);