diff --git a/barretenberg/cpp/cmake/threading.cmake b/barretenberg/cpp/cmake/threading.cmake index 60e758a75817..5092e8064c62 100644 --- a/barretenberg/cpp/cmake/threading.cmake +++ b/barretenberg/cpp/cmake/threading.cmake @@ -8,6 +8,11 @@ if(MULTITHREADING) # Prevent indirect call type mismatch errors in thread_local destructors # (without this the benchmark flow fails at destruction point for WASM) add_compile_options(-fno-c++-static-destructors) + # Enable WASM SIMD128. V8 TurboFan ≥ 9.1 (Chrome/Node 91+), wasmtime ≥ 2.0, + # all modern browsers support v128 — the WASM binary requires a runtime + # with SIMD enabled. Hot loops in barretenberg (e.g. Phase 5a sched→pts + # copy in round_parallel_msm) gate v128.load/store on __wasm_simd128__. + add_compile_options(-msimd128) endif() #add_compile_options(-fsanitize=thread) #add_link_options(-fsanitize=thread) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/pippenger.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/pippenger.bench.cpp index d1040be2da56..db4a5a3059ce 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/pippenger.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/pippenger.bench.cpp @@ -15,6 +15,10 @@ #include "barretenberg/common/google_bb_bench.hpp" +#include +#include +#include + using namespace benchmark; using Curve = bb::curve::BN254; @@ -57,6 +61,46 @@ BENCHMARK_DEFINE_F(PippengerBench, PippengerUnsafe)(benchmark::State& state) } } +// ===== Round-parallel MSM (A) — window-partitioned, independent of pippenger_unsafe ===== + +BENCHMARK_DEFINE_F(PippengerBench, PippengerRoundParallel)(benchmark::State& state) +{ + const size_t num_threads = static_cast(state.range(0)); + const size_t num_points = static_cast(state.range(1)); + std::span points = srs->get_monomial_points().subspan(0, num_points); + std::span span(&scalars[0], num_points); + bb::PolynomialSpan poly_scalars(0, span); + + const size_t original_concurrency = bb::get_num_cpus(); + bb::set_parallel_for_concurrency(num_threads); + + for (auto _ : state) { + GOOGLE_BB_BENCH_REPORTER(state); + bb::scalar_multiplication::pippenger_round_parallel(poly_scalars, points); + } + + bb::set_parallel_for_concurrency(original_concurrency); +} + +BENCHMARK_DEFINE_F(PippengerBench, PippengerUnsafeThreads)(benchmark::State& state) +{ + const size_t num_threads = static_cast(state.range(0)); + const size_t num_points = static_cast(state.range(1)); + std::span points = srs->get_monomial_points().subspan(0, num_points); + std::span span(&scalars[0], num_points); + bb::PolynomialSpan poly_scalars(0, span); + + const size_t original_concurrency = bb::get_num_cpus(); + bb::set_parallel_for_concurrency(num_threads); + + for (auto _ : state) { + GOOGLE_BB_BENCH_REPORTER(state); + bb::scalar_multiplication::pippenger_unsafe(poly_scalars, points); + } + + bb::set_parallel_for_concurrency(original_concurrency); +} + // ===================== Batch MSM ===================== BENCHMARK_DEFINE_F(PippengerBench, BatchMSM)(benchmark::State& state) @@ -65,21 +109,170 @@ BENCHMARK_DEFINE_F(PippengerBench, BatchMSM)(benchmark::State& state) const size_t poly_size = static_cast(state.range(1)); std::vector> all_scalars(num_polys); - std::vector> scalar_spans; - std::vector> point_spans; + std::vector> scalar_spans; + std::span points = srs->get_monomial_points().subspan(0, poly_size); for (size_t i = 0; i < num_polys; ++i) { all_scalars[i].resize(poly_size); for (auto& s : all_scalars[i]) { s = Fr::random_element(&engine); } - scalar_spans.emplace_back(all_scalars[i]); - point_spans.emplace_back(srs->get_monomial_points().subspan(0, poly_size)); + scalar_spans.emplace_back(0, std::span(all_scalars[i])); + } + + for (auto _ : state) { + GOOGLE_BB_BENCH_REPORTER(state); + bb::scalar_multiplication::MSM::batch_multi_scalar_mul(points, scalar_spans, false); + } +} + +// ===================== Batched MSM — Chonk-representative workloads ===================== +// +// Two paths are benchmarked side-by-side for each scenario: +// - "Batched" calls MSM::batch_multi_scalar_mul (new multi-MSM Phases 1-6b pipeline). +// - "PerMsm" calls pippenger_round_parallel(...) once per MSM (the legacy fallback — +// equivalent to what MSM::batch_multi_scalar_mul did before the multi-MSM dispatcher). +// +// The Batched/PerMsm ratio is the metric of interest: it shows the lift from batching +// the round-parallel scaffolding (GLV doubling, Constantine recoding, schedule build) +// once per batch_commit instead of K times. +// +// Scenarios are picked to mirror the workloads observed in chonk profiles: +// - TranslatorWires_2_17 / 2_14: K=4 dense BN254, the regression case from the +// translator-wire batch_commit hot path. +// - MegaOink_K11: simulates a full Mega oink wire commit (K=11, all same SRS prefix, +// all dense) — the headline target for batching. +// - ECCVMSparse_half: K=4 with ~50% zero scalars, the ECCVM-wire pattern that the +// OLD MSM optimised via non-zero work-unit weighting. +// - DatabusSparse_mostly0: K=8 short polys (n=16384) with ~75% zero scalars — +// the databus-inverse pattern. + +namespace { +struct BatchScenario { + const char* name; + size_t k; // number of MSMs + size_t n; // points per MSM (uniform within each scenario) + double zero_density; // probability of a scalar being zero (0.0 = dense) +}; + +std::vector> build_batch_scalars( + size_t k, size_t n, double zero_density, bb::numeric::RNG& engine, std::span scalar_pool) +{ + std::vector> out(k); + for (size_t m = 0; m < k; ++m) { + out[m].resize(n); + for (size_t i = 0; i < n; ++i) { + const bool zero = zero_density > 0.0 && (static_cast(engine.get_random_uint32() & 0xFFFFFU) / + static_cast(0x100000U)) < zero_density; + out[m][i] = zero ? Fr::zero() : scalar_pool[(m * n + i) % scalar_pool.size()]; + } + } + return out; +} +} // namespace + +BENCHMARK_DEFINE_F(PippengerBench, BatchedChonk)(benchmark::State& state) +{ + const size_t scenario_idx = static_cast(state.range(0)); + // Production K + N values, derived from the actual prover call sites that drive + // `commitment_key.batch_commit` -> `MSM::batch_multi_scalar_mul`: + // + // Translator K=10 N=2^17 — execute_wire_and_sorted_constraints_commitments_round: + // 5 ConcatenatedPolynomials + 5 OrderedRangeConstraints, + // all at full circuit size (MINI_CIRCUIT * CONCAT = 2^13 * 16). + // Dense (no duplicates hint passed). + // MegaOink K=17 N=2^17 — OinkProver::commit_to_wires (Mega): + // 3 base wires (w_l/w_r/w_o, duplicates hint=true) + // + 4 ecc_op_wires (sparse, mostly populated only when ecc ops fire) + // + 10 databus polys (5 buses * 2; mostly zero outside the active bus). + // Approximated here as a dense single density; the per-poly + // heterogeneity is left to a follow-up (would need build_batch_scalars + // to accept per-poly densities/hints). + // DatabusOnly K=10 N=2^14 — isolates the databus sub-batch from Mega oink (mostly-zero + // wires at a smaller size — what the datbus-inverse pattern looks like). + // + // K=4 sub-batches that existed previously were not representative of any prover; removed. + static const std::array scenarios{ { + { "Translator_K10_2_17", 10, 1U << 17, 0.0 }, + { "MegaOink_K17_2_17", 17, 1U << 17, 0.0 }, + { "DatabusOnly_K10_2_14_mostly0", 10, 1U << 14, 0.75 }, + // ECCVM 85-wire batch split into its dense and sparse halves. ~60 wires + // (precompute point-table + msm-region + accumulators + shifted entities) + // are dense; ~25 transcript wires are populated only up to op-queue size + // (CONST_OP_QUEUE_LOG_SIZE = 2^12) in a 2^15 dyadic allocation, so ~87.5% + // zero. Sum of the two scenarios approximates the production ECCVM commit + // batch. BN254 used as a proxy for Grumpkin: at N=2^15 both curves sit + // above their native GLV threshold (2^13), so the dispatcher's only + // cross-MSM amortisation (the shared GLV-doubled prefix) is OFF either + // way and the batched/per-MSM ratio transfers. + { "ECCVM_dense_K60_2_15", 60, 1U << 15, 0.0 }, + { "ECCVM_transcript_K25_2_15", 25, 1U << 15, 0.875 }, + } }; + const auto& sc = scenarios[scenario_idx]; + state.SetLabel(sc.name); + + auto all_scalars = build_batch_scalars(sc.k, sc.n, sc.zero_density, engine, scalars); + std::vector> scalar_spans; + std::span points = srs->get_monomial_points().subspan(0, sc.n); + for (size_t m = 0; m < sc.k; ++m) { + scalar_spans.emplace_back(0, std::span(all_scalars[m])); } for (auto _ : state) { GOOGLE_BB_BENCH_REPORTER(state); - bb::scalar_multiplication::MSM::batch_multi_scalar_mul(point_spans, scalar_spans, false); + bb::scalar_multiplication::MSM::batch_multi_scalar_mul(points, scalar_spans, false); + } +} + +BENCHMARK_DEFINE_F(PippengerBench, PerMsmChonk)(benchmark::State& state) +{ + const size_t scenario_idx = static_cast(state.range(0)); + // Production K + N values, derived from the actual prover call sites that drive + // `commitment_key.batch_commit` -> `MSM::batch_multi_scalar_mul`: + // + // Translator K=10 N=2^17 — execute_wire_and_sorted_constraints_commitments_round: + // 5 ConcatenatedPolynomials + 5 OrderedRangeConstraints, + // all at full circuit size (MINI_CIRCUIT * CONCAT = 2^13 * 16). + // Dense (no duplicates hint passed). + // MegaOink K=17 N=2^17 — OinkProver::commit_to_wires (Mega): + // 3 base wires (w_l/w_r/w_o, duplicates hint=true) + // + 4 ecc_op_wires (sparse, mostly populated only when ecc ops fire) + // + 10 databus polys (5 buses * 2; mostly zero outside the active bus). + // Approximated here as a dense single density; the per-poly + // heterogeneity is left to a follow-up (would need build_batch_scalars + // to accept per-poly densities/hints). + // DatabusOnly K=10 N=2^14 — isolates the databus sub-batch from Mega oink (mostly-zero + // wires at a smaller size — what the datbus-inverse pattern looks like). + // + // K=4 sub-batches that existed previously were not representative of any prover; removed. + static const std::array scenarios{ { + { "Translator_K10_2_17", 10, 1U << 17, 0.0 }, + { "MegaOink_K17_2_17", 17, 1U << 17, 0.0 }, + { "DatabusOnly_K10_2_14_mostly0", 10, 1U << 14, 0.75 }, + // ECCVM 85-wire batch split into its dense and sparse halves. ~60 wires + // (precompute point-table + msm-region + accumulators + shifted entities) + // are dense; ~25 transcript wires are populated only up to op-queue size + // (CONST_OP_QUEUE_LOG_SIZE = 2^12) in a 2^15 dyadic allocation, so ~87.5% + // zero. Sum of the two scenarios approximates the production ECCVM commit + // batch. BN254 used as a proxy for Grumpkin: at N=2^15 both curves sit + // above their native GLV threshold (2^13), so the dispatcher's only + // cross-MSM amortisation (the shared GLV-doubled prefix) is OFF either + // way and the batched/per-MSM ratio transfers. + { "ECCVM_dense_K60_2_15", 60, 1U << 15, 0.0 }, + { "ECCVM_transcript_K25_2_15", 25, 1U << 15, 0.875 }, + } }; + const auto& sc = scenarios[scenario_idx]; + state.SetLabel(sc.name); + + auto all_scalars = build_batch_scalars(sc.k, sc.n, sc.zero_density, engine, scalars); + std::span points = srs->get_monomial_points().subspan(0, sc.n); + + for (auto _ : state) { + GOOGLE_BB_BENCH_REPORTER(state); + for (size_t m = 0; m < sc.k; ++m) { + bb::PolynomialSpan sp(0, std::span(all_scalars[m].data(), sc.n)); + (void)bb::scalar_multiplication::pippenger_round_parallel(sp, points); + } } } @@ -103,10 +296,9 @@ BENCHMARK_DEFINE_F(PippengerBench, BatchMSM_1656)(benchmark::State& state) s = Fr::random_element(&engine); } - std::vector> scalar_spans; - std::vector> point_spans; - scalar_spans.emplace_back(msm_scalars); - point_spans.emplace_back(srs->get_monomial_points().subspan(0, msm_size)); + std::vector> scalar_spans; + scalar_spans.emplace_back(0, std::span(msm_scalars)); + std::span points = srs->get_monomial_points().subspan(0, msm_size); // This is thread-local: restore after the benchmark so other cases in this binary are unaffected. const size_t original_concurrency = bb::get_num_cpus(); @@ -114,12 +306,93 @@ BENCHMARK_DEFINE_F(PippengerBench, BatchMSM_1656)(benchmark::State& state) for (auto _ : state) { GOOGLE_BB_BENCH_REPORTER(state); - bb::scalar_multiplication::MSM::batch_multi_scalar_mul(point_spans, scalar_spans, false); + bb::scalar_multiplication::MSM::batch_multi_scalar_mul(points, scalar_spans, false); } bb::set_parallel_for_concurrency(original_concurrency); } +// ===================== Sparsity-profile single MSM ===================== +// +// Single-MSM pippenger_round_parallel across dyadic sizes 2^15..2^19 under two scalar +// distributions, to A/B the thread-pool backend (new generation-counter pool vs the +// merge-train pool) at the workload shapes that stress the round-parallel scaffolding +// and the dedup pre-pass. +// +// Dense80 — 80% uniformly-random nonzero scalars, 20% zero. Exercises the main +// bucket-accumulation pipeline with light sparsity. dedup_hint=false. +// DupHeavy — 50% unique random, 25% all equal to one random scalar A, 5% all equal +// to another random scalar B, 20% zero. Heavy duplication drives the +// Phase A dedup pre-pass (the most thread-intensive stage), so this is +// the case most sensitive to pool dispatch / oversubscription behavior. +// dedup_hint=true. +// +// Scalars are drawn from the fixture's deterministic debug RNG so the A/B runs on the +// two pool backends see identical inputs. +namespace { +enum class SparsityProfile : uint8_t { Dense80 = 0, DupHeavy = 1 }; + +[[nodiscard]] double uniform01(bb::numeric::RNG& engine) noexcept +{ + return static_cast(engine.get_random_uint32()) / static_cast(std::numeric_limits::max()); +} + +std::vector build_sparsity_scalars(SparsityProfile profile, size_t n, bb::numeric::RNG& engine) +{ + std::vector out(n); + if (profile == SparsityProfile::Dense80) { + for (size_t i = 0; i < n; ++i) { + out[i] = (uniform01(engine) < 0.20) ? Fr::zero() : Fr::random_element(&engine); + } + } else { + const Fr dup_a = Fr::random_element(&engine); + const Fr dup_b = Fr::random_element(&engine); + for (size_t i = 0; i < n; ++i) { + const double r = uniform01(engine); + if (r < 0.20) { + out[i] = Fr::zero(); // 20% zero + } else if (r < 0.45) { + out[i] = dup_a; // 25% duplicate of A + } else if (r < 0.50) { + out[i] = dup_b; // 5% duplicate of B + } else { + out[i] = Fr::random_element(&engine); // 50% unique random + } + } + } + return out; +} +} // namespace + +BENCHMARK_DEFINE_F(PippengerBench, PippengerSparsity)(benchmark::State& state) +{ + const auto profile = static_cast(state.range(0)); + const size_t num_points = static_cast(state.range(1)); + const bool dedup_hint = (profile == SparsityProfile::DupHeavy); + state.SetLabel(profile == SparsityProfile::Dense80 ? "Dense80" : "DupHeavy"); + + // Build the scalar set from a fresh RNG re-seeded deterministically per (profile, size) + // rather than from the shared advancing engine. Two reasons: + // 1. Every benchmark repetition (--benchmark_repetitions) reuses the SAME scalars, so the + // measured variance reflects pool/scheduler noise only, not input variation. + // 2. The input is independent of benchmark execution order, so a filtered subset or a + // pool-toggled A/B run sees byte-identical scalars — the comparison is properly paired. + // The scalar build is outside the timed `for (auto _ : state)` loop regardless, so RNG cost + // never enters the measurement. + const std::uint_fast64_t case_seed = + 0xC0FFEEULL + (static_cast(profile) << 32) + static_cast(num_points); + bb::numeric::RNG& case_engine = bb::numeric::get_debug_randomness(/*reset=*/true, /*seed=*/case_seed); + + std::vector msm_scalars = build_sparsity_scalars(profile, num_points, case_engine); + std::span points = srs->get_monomial_points().subspan(0, num_points); + bb::PolynomialSpan poly_scalars(0, std::span(msm_scalars.data(), num_points)); + + for (auto _ : state) { + GOOGLE_BB_BENCH_REPORTER(state); + (void)bb::scalar_multiplication::pippenger_round_parallel(poly_scalars, points, dedup_hint); + } +} + // ===================== Registration ===================== // Single MSM: 2^14 to 2^20 @@ -128,6 +401,11 @@ BENCHMARK_REGISTER_F(PippengerBench, PippengerUnsafe) ->RangeMultiplier(4) ->Range(1 << 14, 1 << 20); +// Sparsity-profile single MSM: {profile (0=Dense80, 1=DupHeavy), size}, sizes 2^15..2^19. +BENCHMARK_REGISTER_F(PippengerBench, PippengerSparsity) + ->Unit(benchmark::kMillisecond) + ->ArgsProduct({ { 0, 1 }, { 1 << 15, 1 << 16, 1 << 17, 1 << 18, 1 << 19 } }); + // Batch MSM: {num_polynomials, polynomial_size} // AVM-like: 32 polys of size 2^21 (one batch from ~2618 wire polys committed in batches of 32) BENCHMARK_REGISTER_F(PippengerBench, BatchMSM) @@ -141,6 +419,51 @@ BENCHMARK_REGISTER_F(PippengerBench, BatchMSM_1656) ->Args({ 256, 1 << 16 }) ->Args({ 256, 1 << 20 }); +// Chonk-representative batched MSM workloads. Scenario index 0..6 indexes into the +// scenario table above. Run "Batched" and "PerMsm" with matching indices and compare. +BENCHMARK_REGISTER_F(PippengerBench, BatchedChonk)->Unit(benchmark::kMillisecond)->DenseRange(0, 4, 1); +BENCHMARK_REGISTER_F(PippengerBench, PerMsmChonk)->Unit(benchmark::kMillisecond)->DenseRange(0, 4, 1); + +// Grid sweep for A vs B: {threads, size}. N covers 2^7..2^21 (with extra in-between +// points around the GLV crossover). +BENCHMARK_REGISTER_F(PippengerBench, PippengerRoundParallel) + ->Unit(benchmark::kMillisecond) + ->ArgsProduct({ { 1, 4, 8, 12, 16, 32, 64, 128 }, + { 1 << 7, + 1 << 8, + 1 << 9, + 1 << 10, + 1 << 11, + 1 << 12, + 1 << 13, + 1 << 14, + 1 << 15, + 3 << 14, + 1 << 16, + 3 << 15, + 1 << 17, + 3 << 16, + 1 << 18, + 1 << 19, + 1 << 20, + 1 << 21 } }); + +BENCHMARK_REGISTER_F(PippengerBench, PippengerUnsafeThreads) + ->Unit(benchmark::kMillisecond) + ->ArgsProduct({ { 1, 4, 8, 12, 16, 32, 64, 128 }, + { 1 << 9, + 1 << 10, + 1 << 11, + 1 << 12, + 1 << 13, + 1 << 14, + 1 << 15, + 1 << 16, + 1 << 17, + 1 << 18, + 1 << 19, + 1 << 20 } }); + } // namespace BENCHMARK_MAIN(); diff --git a/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/small_msm_matrix.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/small_msm_matrix.bench.cpp new file mode 100644 index 000000000000..be6b9a5ab76f --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/small_msm_matrix.bench.cpp @@ -0,0 +1,293 @@ +/** + * @brief Small-MSM crossover matrix benchmark. + * + * Outputs a single matrix where rows = method and columns = N (number of points). + * Methods compared: + * - jac_fast_mt_always : pippenger_round_parallel_jacobian_fast forced + * maximally-multithreaded (min_pts_per_thread_override = 1). + * - jac_fast_st_always : pippenger_round_parallel_jacobian_fast forced + * single-threaded (min_pts_per_thread_override = SIZE_MAX). + * - small_mul_threaded : trivial_msm_threaded (bb::parallel_for split, + * per-worker straus or jac_fast based on slice size). + * - straus_msm : Element::straus_msm direct (single-threaded). + * + * Two outputs are reported per (method, N): + * - Wall-clock median ns per run. + * - The MIN_JACOBIAN_SIZE crossover (single-threaded jac_fast vs straus_msm). + * + * Build & run: + * cd barretenberg/cpp/build && ninja small_msm_matrix_bench + * ./bin/small_msm_matrix_bench + */ +#include "barretenberg/common/log.hpp" +#include "barretenberg/common/thread.hpp" +#include "barretenberg/ecc/curves/bn254/bn254.hpp" +#include "barretenberg/ecc/groups/element.hpp" +#include "barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include "barretenberg/srs/global_crs.hpp" +#include +#include +#include +#include +#include +#include +#include + +using Curve = bb::curve::BN254; +using Fr = Curve::ScalarField; +using G1 = Curve::AffineElement; +using Element = Curve::Element; + +namespace { + +// Median wall-clock ns across `iters` invocations of `run`. +template double median_ns(Run&& run, size_t iters) +{ + std::vector samples(iters); + for (size_t i = 0; i < iters; ++i) { + const auto t0 = std::chrono::steady_clock::now(); + run(); + const auto t1 = std::chrono::steady_clock::now(); + samples[i] = static_cast(std::chrono::duration_cast(t1 - t0).count()); + } + std::sort(samples.begin(), samples.end()); + return samples[samples.size() / 2]; +} + +// WASM-tuned iteration counts. Quadrupled from the previous tuning to damp +// per-cell variance — each cell now budgets ~200 ms–1 s wall time. +size_t pick_iters(size_t n) +{ + if (n <= 4) { + return 800; + } + if (n <= 16) { + return 400; + } + if (n <= 64) { + return 200; + } + if (n <= 256) { + return 100; + } + if (n <= 1024) { + return 32; + } + if (n <= 4096) { + return 16; + } + return 8; +} + +void print_matrix_header(const std::vector& ns) +{ + std::printf("%-24s", "N"); + for (size_t n : ns) { + std::printf(" %12zu", n); + } + std::printf("\n"); +} + +// `mask[i] == false` skips column i (prints "-" instead of a number). +void print_matrix_row(const char* label, const std::vector& ns_per_run, const std::vector& mask) +{ + std::printf("%-24s", label); + for (size_t i = 0; i < ns_per_run.size(); ++i) { + if (mask[i]) { + std::printf(" %12.0f", ns_per_run[i]); + } else { + std::printf(" %12s", "-"); + } + } + std::printf("\n"); +} + +// Phase 1: precise crossover sweep — at every N in {32, 34, ..., 64}, compare +// single-threaded `straus_msm` against single-threaded `jac_fast`. Returns the +// smallest N where jac_fast wins (or 0 if jac never wins in-range). +size_t run_crossover_sweep(std::span all_points, std::span scalars) +{ + std::printf("\n=== MIN_JACOBIAN_SIZE crossover sweep (single-threaded straus_msm vs jac_fast, ns) ===\n\n"); + std::printf("%-8s %12s %12s %10s\n", "N", "straus", "jac_st", "delta_%"); + + size_t crossover = 0; + constexpr size_t REPEATS = 3; + for (size_t n = 32; n <= 64; n += 2) { + std::span points = all_points.subspan(0, n); + std::span scalars_view(scalars.data(), n); + const size_t iters = pick_iters(n); + + std::vector straus_samples(REPEATS); + std::vector jac_samples(REPEATS); + for (size_t r = 0; r < REPEATS; ++r) { + straus_samples[r] = median_ns( + [&] { + volatile auto v = Element::straus_msm(points, scalars_view); + (void)v; + }, + iters); + jac_samples[r] = median_ns( + [&] { + volatile auto v = + bb::scalar_multiplication::round_parallel_detail::pippenger_round_parallel_jacobian_fast( + scalars_view, points, /*min_pts_per_thread_override=*/SIZE_MAX); + (void)v; + }, + iters); + } + std::sort(straus_samples.begin(), straus_samples.end()); + std::sort(jac_samples.begin(), jac_samples.end()); + const double straus = straus_samples[REPEATS / 2]; + const double jac = jac_samples[REPEATS / 2]; + const double delta_pct = 100.0 * (jac - straus) / straus; + std::printf("%-8zu %12.0f %12.0f %+10.2f\n", n, straus, jac, delta_pct); + if (crossover == 0 && jac < straus) { + crossover = n; + } + } + if (crossover != 0) { + std::printf("\nFirst N where jac_fast_st_always beats straus_msm: %zu\n", crossover); + } else { + std::printf("\nstraus_msm wins across the entire 32..64 sweep.\n"); + } + return crossover; +} + +void run_matrix() +{ + constexpr size_t MAX_N = 1U << 14; + + // Initialise SRS once and reuse the same point span across all cells. + bb::srs::init_file_crs_factory(bb::srs::bb_crs_path()); + auto srs = bb::srs::get_crs_factory()->get_crs(MAX_N); + std::span all_points = srs->get_monomial_points().subspan(0, MAX_N); + + bb::numeric::RNG& engine = bb::numeric::get_debug_randomness(); + std::vector scalars(MAX_N); + for (auto& s : scalars) { + s = Fr::random_element(&engine); + } + + // Phase 1: precise crossover sweep — disabled for the N=1..128 sub-range run. + (void)&run_crossover_sweep; + + // Phase 2: full matrix. + // Column set — sweep small-MSM regime where the four methods can disagree. + // Includes powers of 2 plus a few intermediate values around the suspected + // jacobian crossover, extended out to 16384 since small_mul_threaded was + // still beating jac_fast_mt at 8192. + const std::vector ns = { + 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, + 128, 192, 256, 384, 512, 768, 1024, 2048, 4096, 8192, 12288, 16384, + }; + + // Per-column masks. straus_msm is dropped at N >= 256 (its naive double-and-add + // cost dominates the schedule and saturates the iteration budget). The two + // pippenger_round_parallel variants kick in at N >= 64. + std::vector straus_mask(ns.size()); + std::vector internal_mask(ns.size()); + for (size_t i = 0; i < ns.size(); ++i) { + straus_mask[i] = (ns[i] < 256); + internal_mask[i] = (ns[i] >= 64); + } + std::vector all_mask(ns.size(), true); + + std::vector row_jac_mt(ns.size()); + std::vector row_jac_st(ns.size()); + std::vector row_threaded(ns.size()); + std::vector row_straus(ns.size()); + std::vector row_internal(ns.size()); + + for (size_t col = 0; col < ns.size(); ++col) { + const size_t n = ns[col]; + std::span points = all_points.subspan(0, n); + std::span scalars_view(scalars.data(), n); + std::span mut_scalars_view(scalars.data(), n); + bb::PolynomialSpan poly_scalars(0, scalars_view); + bb::PolynomialSpan mut_poly_scalars(0, mut_scalars_view); + const size_t iters = pick_iters(n); + + row_jac_mt[col] = median_ns( + [&] { + volatile auto r = + bb::scalar_multiplication::round_parallel_detail::pippenger_round_parallel_jacobian_fast( + scalars_view, points, /*min_pts_per_thread_override=*/1); + (void)r; + }, + iters); + + row_jac_st[col] = median_ns( + [&] { + volatile auto r = + bb::scalar_multiplication::round_parallel_detail::pippenger_round_parallel_jacobian_fast( + scalars_view, points, /*min_pts_per_thread_override=*/SIZE_MAX); + (void)r; + }, + iters); + + row_threaded[col] = median_ns( + [&] { + volatile auto r = bb::scalar_multiplication::trivial_msm_threaded(poly_scalars, points); + (void)r; + }, + iters); + + if (straus_mask[col]) { + row_straus[col] = median_ns( + [&] { + volatile auto r = Element::straus_msm(points, scalars_view); + (void)r; + }, + iters); + } + + if (internal_mask[col]) { + row_internal[col] = median_ns( + [&] { + volatile auto r = + bb::scalar_multiplication::pippenger_round_parallel(mut_poly_scalars, points); + (void)r; + }, + iters); + } + } + + std::printf("\n=== small-MSM crossover matrix (median wall-clock ns per run, BN254) ===\n\n"); + print_matrix_header(ns); + print_matrix_row("jac_fast_mt_always", row_jac_mt, all_mask); + print_matrix_row("jac_fast_st_always", row_jac_st, all_mask); + print_matrix_row("small_mul_threaded", row_threaded, all_mask); + print_matrix_row("straus_msm", row_straus, straus_mask); + print_matrix_row("pippenger_internal", row_internal, internal_mask); + + // Best method per N — masked candidates are excluded from the comparison. + std::printf("\nBest method per N:\n"); + for (size_t i = 0; i < ns.size(); ++i) { + struct Cand { + const char* name; + double v; + bool active; + }; + std::array c{ { { "jac_mt", row_jac_mt[i], true }, + { "jac_st", row_jac_st[i], true }, + { "threaded", row_threaded[i], true }, + { "straus", row_straus[i], straus_mask[i] }, + { "internal", row_internal[i], internal_mask[i] } } }; + const Cand* best = nullptr; + for (const Cand& cand : c) { + if (cand.active && (best == nullptr || cand.v < best->v)) { + best = &cand; + } + } + std::printf(" N=%-6zu best=%-12s (%.0f ns)\n", ns[i], best->name, best->v); + } +} + +} // namespace + +int main() +{ + run_matrix(); + return 0; +} diff --git a/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/thread_scaling.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/thread_scaling.bench.cpp deleted file mode 100644 index c7e4c15cb324..000000000000 --- a/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/thread_scaling.bench.cpp +++ /dev/null @@ -1,139 +0,0 @@ -/** - * @brief Pippenger thread-scaling benchmark for heterogeneous scalar distributions. - * - * MSM::batch_multi_scalar_mul partitions work across threads by cumulative per-scalar - * weight (see get_work_units in scalar_multiplication.cpp), where each scalar's weight - * is ceil(bit_length / bits_per_slice) -- i.e. the number of nonzero c-bit slices it - * contributes to bucket accumulation. Small scalars weigh less because their high-order - * slices are zero and get filtered by the zero-bucket pre-sort. This benchmark exercises - * pathological and typical bit-size distributions to verify thread scaling stays uniform. - * - * Distributions contrasted here: - * - Clustered: first half small (32-bit), second half full random -- stresses the - * weighted split; count-based partitioning would give half the threads - * ~all of the heavy work. - * - UniformMixed: small/full randomly interleaved -- isolates heterogeneity alone. - * - AllFull: all full random (z_perm-like baseline). - * - * Expected: all three scale comparably under the weighted partition. - */ -#include "barretenberg/common/thread.hpp" -#include "barretenberg/ecc/curves/bn254/bn254.hpp" -#include "barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp" -#include "barretenberg/numeric/random/engine.hpp" -#include "barretenberg/srs/global_crs.hpp" - -#include - -#include "barretenberg/common/google_bb_bench.hpp" - -using namespace benchmark; - -using Curve = bb::curve::BN254; -using Fr = Curve::ScalarField; -using G1 = Curve::AffineElement; - -namespace { - -constexpr size_t MSM_SIZE = 1 << 20; - -enum class Distribution { Clustered, UniformMixed, AllFull }; - -class ThreadScalingBench : public benchmark::Fixture { - public: - std::shared_ptr> srs; - bb::numeric::RNG& engine = bb::numeric::get_debug_randomness(); - - void SetUp([[maybe_unused]] const ::benchmark::State& state) override - { - if (srs) { - return; - } - bb::srs::init_file_crs_factory(bb::srs::bb_crs_path()); - srs = bb::srs::get_crs_factory()->get_crs(MSM_SIZE); - } - - // 32-bit "small" value -- mimics witness indices, booleans, limbs. - // On BN254 (254-bit field) with ~14 bits per Pippenger slice, only the lowest - // ~2-3 rounds produce nonzero slices for these scalars; the rest get filtered. - Fr small_scalar() { return Fr(static_cast(engine.get_random_uint32())); } - Fr full_scalar() { return Fr::random_element(&engine); } - - std::vector build_scalars(Distribution dist) - { - std::vector scalars(MSM_SIZE); - switch (dist) { - case Distribution::Clustered: - for (size_t i = 0; i < MSM_SIZE / 2; ++i) { - scalars[i] = small_scalar(); - } - for (size_t i = MSM_SIZE / 2; i < MSM_SIZE; ++i) { - scalars[i] = full_scalar(); - } - break; - case Distribution::UniformMixed: - for (size_t i = 0; i < MSM_SIZE; ++i) { - scalars[i] = (engine.get_random_uint32() & 1U) ? small_scalar() : full_scalar(); - } - break; - case Distribution::AllFull: - for (size_t i = 0; i < MSM_SIZE; ++i) { - scalars[i] = full_scalar(); - } - break; - } - return scalars; - } -}; - -static void run_msm(ThreadScalingBench& fx, benchmark::State& state, Distribution dist) -{ - const size_t num_threads = static_cast(state.range(0)); - - // Rebuild per-invocation of the bench is fine: scalars get mutated (Montgomery - // round-trip) inside batch_multi_scalar_mul, and we want consistent input across iterations. - std::vector scalars = fx.build_scalars(dist); - - std::vector> scalar_spans; - std::vector> point_spans; - scalar_spans.emplace_back(scalars); - point_spans.emplace_back(fx.srs->get_monomial_points().subspan(0, MSM_SIZE)); - - const size_t original_concurrency = bb::get_num_cpus(); - bb::set_parallel_for_concurrency(num_threads); - - for (auto _ : state) { - GOOGLE_BB_BENCH_REPORTER(state); - bb::scalar_multiplication::MSM::batch_multi_scalar_mul(point_spans, scalar_spans, false); - } - - bb::set_parallel_for_concurrency(original_concurrency); -} - -BENCHMARK_DEFINE_F(ThreadScalingBench, Clustered)(benchmark::State& state) -{ - run_msm(*this, state, Distribution::Clustered); -} -BENCHMARK_DEFINE_F(ThreadScalingBench, UniformMixed)(benchmark::State& state) -{ - run_msm(*this, state, Distribution::UniformMixed); -} -BENCHMARK_DEFINE_F(ThreadScalingBench, AllFull)(benchmark::State& state) -{ - run_msm(*this, state, Distribution::AllFull); -} - -static void ThreadSweep(benchmark::internal::Benchmark* b) -{ - for (int64_t t : { 1, 2, 4, 8 }) { - b->Arg(t); - } -} - -BENCHMARK_REGISTER_F(ThreadScalingBench, Clustered)->Unit(benchmark::kMillisecond)->Apply(ThreadSweep); -BENCHMARK_REGISTER_F(ThreadScalingBench, UniformMixed)->Unit(benchmark::kMillisecond)->Apply(ThreadSweep); -BENCHMARK_REGISTER_F(ThreadScalingBench, AllFull)->Unit(benchmark::kMillisecond)->Apply(ThreadSweep); - -} // namespace - -BENCHMARK_MAIN(); diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index 147dc8bb5532..5183321a60d3 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -11,13 +11,17 @@ */ #include "barretenberg/common/bb_bench.hpp" +#include "barretenberg/common/log.hpp" #include "barretenberg/common/ref_span.hpp" #include "barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp" #include "barretenberg/polynomials/polynomial.hpp" #include "barretenberg/srs/factories/crs_factory.hpp" #include "barretenberg/srs/global_crs.hpp" +#include +#include #include +#include #include #include #include @@ -70,7 +74,7 @@ template class CommitmentKey { * @param polynomial a univariate polynomial p(X) = ∑ᵢ aᵢ⋅Xⁱ * @return Commitment computed as C = [p(x)] = ∑ᵢ aᵢ⋅Gᵢ */ - Commitment commit(PolynomialSpan polynomial) const + Commitment commit(PolynomialSpan polynomial, bool has_duplicates_hint = false) const { BB_BENCH_NAME("CommitmentKey::commit"); std::span point_table = get_monomial_points(); @@ -81,7 +85,7 @@ template class CommitmentKey { " points with an SRS of size ", get_monomial_size())); } - return scalar_multiplication::pippenger_unsafe(polynomial, point_table); + return scalar_multiplication::pippenger_unsafe(polynomial, point_table, has_duplicates_hint); }; /** * @brief Batch commitment to multiple polynomials @@ -89,48 +93,32 @@ template class CommitmentKey { * The input polynomials are not const because batch_mul modifies them and then restores them back. * * @param polynomials vector of polynomial spans to commit to + * @param has_duplicates_hints optional per-polynomial hints (parallel to polynomials): + * a non-zero entry opts that polynomial's MSM into the dedup pre-pass. * @return std::vector vector of commitments, one for each polynomial */ std::vector batch_commit(RefSpan> polynomials, - size_t max_batch_size = std::numeric_limits::max()) const + std::span has_duplicates_hints = {}) const { BB_BENCH_NAME("CommitmentKey::batch_commit"); - // We can only commit max_batch_size at a time - // This is to prevent excessive memory usage in the pippenger algorithm - // First batch, create the commitments vector - std::vector commitments; - - for (size_t i = 0; i < polynomials.size();) { - // Note: have to be careful how we compute this to not overlow e.g. max_batch_size + 1 would - size_t batch_size = std::min(max_batch_size, polynomials.size() - i); - size_t batch_end = i + batch_size; - - // Prepare spans for batch MSM - std::vector> points_spans; - std::vector> scalar_spans; - - for (auto& polynomial : polynomials.subspan(i, batch_end - i)) { - size_t consumed_srs = polynomial.start_index() + polynomial.size(); - if (consumed_srs > get_monomial_size()) { - throw_or_abort(format("Attempting to commit to a polynomial that needs ", - consumed_srs, - " points with an SRS of size ", - get_monomial_size())); - } - std::span point_table = get_monomial_points().subspan(polynomial.start_index()); - scalar_spans.emplace_back(polynomial.coeffs()); - points_spans.emplace_back(point_table); - } + std::vector> scalar_spans; + scalar_spans.reserve(polynomials.size()); - // Perform batch MSM - auto results = scalar_multiplication::MSM::batch_multi_scalar_mul(points_spans, scalar_spans, false); - for (const auto& result : results) { - commitments.emplace_back(result); + for (auto& polynomial : polynomials) { + const size_t consumed_srs = polynomial.start_index() + polynomial.size(); + if (consumed_srs > get_monomial_size()) { + throw_or_abort(format("Attempting to commit to a polynomial that needs ", + consumed_srs, + " points with an SRS of size ", + get_monomial_size())); } - i += batch_size; + scalar_spans.emplace_back(polynomial.start_index(), polynomial.coeffs()); } - return commitments; + + auto results = scalar_multiplication::MSM::batch_multi_scalar_mul( + get_monomial_points(), scalar_spans, /*handle_edge_cases=*/false, has_duplicates_hints); + return std::vector(results.begin(), results.end()); }; // helper builder struct for constructing a batch to commit at once @@ -138,29 +126,22 @@ template class CommitmentKey { CommitmentKey* key; RefVector> wires; std::vector labels; - std::vector*> tail_polys; // optional ZK masking tails (parallel to wires) + std::vector has_duplicates_hints; // per-poly dedup opt-in (parallel to wires) - std::vector commit_and_send_to_verifier(auto transcript, - size_t max_batch_size = std::numeric_limits::max()) + std::vector commit_and_send_to_verifier(auto transcript) { - std::vector commitments = key->batch_commit(wires, max_batch_size); - - // Adjust commitments for wires with masking tails: C' = C_short + commit(tail) + std::vector commitments = key->batch_commit(wires, has_duplicates_hints); for (size_t i = 0; i < commitments.size(); ++i) { - if (i < tail_polys.size() && tail_polys[i] != nullptr && !tail_polys[i]->is_empty()) { - commitments[i] = commitments[i] + key->commit(*tail_polys[i]); - } transcript->send_to_verifier(labels[i], commitments[i]); } - return commitments; } - void add_to_batch(Polynomial& poly, const std::string& label, const Polynomial* tail = nullptr) + void add_to_batch(Polynomial& poly, const std::string& label, bool has_duplicates_hint = false) { wires.push_back(poly); labels.push_back(label); - tail_polys.push_back(tail); + has_duplicates_hints.push_back(has_duplicates_hint ? uint8_t{ 1 } : uint8_t{ 0 }); } }; diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_arena_layout.hpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_arena_layout.hpp new file mode 100644 index 000000000000..22453899e6d4 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_arena_layout.hpp @@ -0,0 +1,336 @@ +// Per-worker arena layout for the round-parallel Pippenger MSM (Zone W slab). +// +// Canonical source of truth for the per-worker byte walk that was previously +// duplicated across `compute_arena_bytes_for_msm`, the live allocator inside +// `pippenger_round_parallel`, and `pippenger_bn254_arena_layout_fits_for_test`. +// The historical arena drift bugs (cluster_offsets miscount, wasm +// aligned_local overflow, NO_GLV abort, t1 abort) all traced to disagreements +// between those copies; this struct removes that class by computing the layout +// once. +// +// The constructor's layout walk mirrors the live allocator's `layout_add` +// sequence exactly, including alignment slop. The sizer's previous +// arithmetic-only formula did not honour per-allocation alignment, so it +// systematically under-counted by a few bytes per slab; the struct fixes that +// by construction. +// +// Phase A and Stage 6 fields overlay the same per-worker bytes because the +// parallel_for invocations are disjoint (Phase A runs on the first window +// batch, Stage 6 runs per batch thereafter, and never on the same worker +// concurrently). `per_worker_union_bytes = max(ts_fixed, pa_layout)`. + +#pragma once + +#include "barretenberg/numeric/bitop/get_msb.hpp" + +#include +#include +#include +#include +#include +#include + +namespace bb::scalar_multiplication::round_parallel_detail { + +// ============================================================================ +// Round-parallel internals exposed to the test suite. +// +// `pippenger_bn254_arena_layout_fits_for_test` is a TU-local helper that walks +// the actual Zone P / Zone W / Zone S allocator for representative inputs and +// asserts the result fits in `compute_arena_bytes_for_msm`'s promise. Its body +// lives in `scalar_multiplication.test.cpp`, which means the helpers it needs +// (`choose_window_bits`, `build_var_window_schedule`, `ChunkOutput`, +// `DEDUP_MAX_*`, `VAR_WINDOW_MAX_WINDOWS`, `compute_arena_bytes_for_msm`) need +// header-visible declarations. +// ============================================================================ + +// Per-window count cap shared by `VariableWindowSchedule` arrays and the live +// allocator's `window_sums_storage` slot. +inline constexpr size_t VAR_WINDOW_MAX_WINDOWS = 128; + +// Dedup pre-pass caps. DEDUP_MAX_CLUSTERS bounds `extra_points` at ≤ 1 MB; +// DEDUP_MAX_MEMBERS bounds the per-worker `cluster_members` slab. +inline constexpr size_t DEDUP_MAX_CLUSTERS = 16384; +inline constexpr size_t DEDUP_MAX_MEMBERS = 32768; + +// Uniform window schedule produced by `build_var_window_schedule`. Holds the +// per-window `c` value and bucket count for downstream sizing/dispatch. +struct VariableWindowSchedule { + size_t num_windows = 0; + std::array window_bits_per_window{}; // window_bits_w for each w + std::array bit_base{}; // B_w = Σ_{k num_buckets{}; // 2^(window_bits_w - 1) + 1 +}; + +// Per-chunk recursive-affine bucket-reduce output (Stage 6b output cell). +template struct ChunkOutput { + typename Curve::Element R{}; + typename Curve::Element L{}; + uint32_t lo = 0; + uint32_t hi = 0; + uint8_t empty = 1; +}; + +// Pick the optimal window size `c`. Native uses a cost model +// `rounds * (n + 15 * buckets)`; WASM uses a closed-form `target_load` formula. +[[nodiscard]] inline uint32_t choose_window_bits(size_t num_points, + size_t num_bits, + size_t n_input, + size_t num_logical_threads) noexcept +{ + constexpr uint32_t MAX_C = 20; + uint32_t best = 2; + +#ifdef __wasm__ + static_cast(num_bits); + const size_t target_load = (n_input > 4096) ? (num_logical_threads * 2 / 3) : (num_logical_threads / 3); + if (target_load == 0 || num_points <= target_load) { + best = 2; + } else { + const size_t ratio = num_points / target_load; + const uint32_t lg = static_cast(numeric::get_msb(ratio)); + best = lg + 1; + if (best < 2) { + best = 2; + } else if (best >= MAX_C) { + best = MAX_C - 1; + } + } +#else + static_cast(n_input); + static_cast(num_logical_threads); + uint64_t best_cost = static_cast(-1); + for (uint32_t window_bits = 2; window_bits < MAX_C; ++window_bits) { + const uint64_t rounds = (num_bits + 2 + window_bits - 1) / window_bits; + const uint64_t buckets = (uint64_t{ 1 } << (window_bits - 1)) + 1; + const uint64_t n = num_points; + constexpr uint64_t BUCKET_ACC_COST = 15; + const uint64_t cost = rounds * (n + (buckets * BUCKET_ACC_COST)); + if (cost < best_cost) { + best_cost = cost; + best = window_bits; + } + } +#endif + + return best; +} + +// Build a uniform window schedule for the given bit budget and chosen `c`. Every window +// is `window_bits` wide except the final one, which takes the remaining bits. The +2 on +// the bit budget accommodates the carry-less top bit of the Constantine recoder. +inline VariableWindowSchedule build_var_window_schedule(size_t num_bits, size_t window_bits) noexcept +{ + VariableWindowSchedule sched{}; + + size_t bits_remaining = num_bits + 2; + size_t bit_offset = 0; + size_t w = 0; + while (bits_remaining > 0 && w < VAR_WINDOW_MAX_WINDOWS) { + const size_t window_bits_w = std::min(window_bits, bits_remaining); + sched.bit_base[w] = static_cast(bit_offset); + sched.window_bits_per_window[w] = static_cast(window_bits_w); + sched.num_buckets[w] = static_cast((size_t{ 1 } << (window_bits_w - 1)) + 1); + bit_offset += window_bits_w; + bits_remaining -= window_bits_w; + ++w; + } + sched.num_windows = w; + return sched; +} + +// Maximum number of independent additions batched per modular inversion in the +// affine-arithmetic group ops (used by Stage 6a/6b). Sizes per-worker +// `points_to_add`, `inversion_scratch`, and `pair_dest` arrays. +inline constexpr size_t BATCH_CAPACITY = 256; + +// Phase A's chunked tree-reduce limit. Capped so the per-worker scratch slab +// (chunk_pts + chunk_ids) stays under ~128 KB. +inline constexpr size_t DEDUP_MAX_CHUNK_MEMBERS = 2048; + +inline constexpr size_t MIN_BATCH_CAPACITY = 32; +inline constexpr size_t MIN_AFFINE_THREAD_RATIO = 2; +inline constexpr size_t SUBCHUNK_ENTRIES_CAP = 2048; +inline constexpr size_t BATCH_MEM_BUDGET = 32ULL * 1024ULL * 1024ULL; + +// Per-bucket-chunk metadata produced by Stage 6a, consumed by Stage 6b's +// cross-thread reduce. +// lo, hi — lowest / highest non-empty digit in the chunk (inclusive) +// buckets_padded — next power of two ≥ (hi - lo + 1) +// empty — 1 iff the chunk had no entries (Stage 6b skips it) +struct AffineBucketChunkInfo { + uint32_t lo = 0; + uint32_t hi = 0; + uint32_t buckets_padded = 0; + uint8_t empty = 1; +}; + +template struct PerWorkerArenaLayout { + using AffineElement = typename Curve::AffineElement; + using BaseField = typename Curve::BaseField; + + // Caps shared between sizer and allocator. Centralised here so the two + // sites can't diverge. + static constexpr size_t PHASE_A_DIRTY_SLOTS_CAP = 4096; // HT_SIZE + static constexpr size_t PHASE_A_BUCKET_REP_CAP = 256; // loose cap + static constexpr size_t PHASE_A_STAGED_CAP = 1024; // loose cap + static constexpr size_t PHASE_A_CHUNK_CAP = DEDUP_MAX_CHUNK_MEMBERS; + static constexpr size_t WORKER_SLAB_ALIGN = alignof(AffineElement); + + // Computed byte sizes (filled by constructor's layout walk). + size_t ts_fixed_layout = 0; // ThreadScratch wpb-independent fields, with align slop + size_t pa_layout = 0; // PhaseAScratch fields, with align slop + size_t per_worker_union_bytes = 0; // = align_up(max(ts_fixed_layout, pa_layout), WORKER_SLAB_ALIGN) + size_t per_worker_per_wpb_layout = 0; // Stage 6 wpb-dependent tail + size_t per_worker_bytes = 0; // = align_up(union + tail, WORKER_SLAB_ALIGN) + + // Constructor performs the canonical layout walk. `windows_per_batch` and + // `dense_stride_est` may be zero — only the wpb-independent parts then + // have meaningful values, useful for the sizer's pre-wpb-solve step. + PerWorkerArenaLayout(size_t chunk_capacity, + size_t global_max_overflow_per_window, + bool dedup_active, + size_t phase_a_cluster_members_cap, + size_t phase_a_cluster_offsets_cap, + size_t windows_per_batch, + size_t dense_stride_est) noexcept + { + auto align_up = [](size_t off, size_t align) -> size_t { return (off + align - 1) & ~(align - 1); }; + auto layout_add = [&](size_t& off, size_t bytes, size_t align) { off = align_up(off, align) + bytes; }; + + // ThreadScratch fixed (curr_pts / curr_buckets / points_to_add / + // inversion_scratch / pair_dest / overflow_slots / overflow_pts). + layout_add(ts_fixed_layout, sizeof(AffineElement) * chunk_capacity, alignof(AffineElement)); + layout_add(ts_fixed_layout, sizeof(uint32_t) * chunk_capacity, alignof(uint32_t)); + layout_add(ts_fixed_layout, sizeof(AffineElement) * 2 * BATCH_CAPACITY, alignof(AffineElement)); + layout_add(ts_fixed_layout, sizeof(BaseField) * BATCH_CAPACITY, alignof(BaseField)); + layout_add(ts_fixed_layout, sizeof(uint32_t) * BATCH_CAPACITY, alignof(uint32_t)); + layout_add(ts_fixed_layout, sizeof(uint32_t) * global_max_overflow_per_window, alignof(uint32_t)); + layout_add(ts_fixed_layout, sizeof(AffineElement) * global_max_overflow_per_window, alignof(AffineElement)); + + // PhaseA (cluster_members / cluster_offsets / dirty_slots / bucket_rep + // / staged / chunk_pts / chunk_ids). Only allocated when dedup_active. + if (dedup_active) { + layout_add(pa_layout, sizeof(uint32_t) * phase_a_cluster_members_cap, alignof(uint32_t)); + layout_add(pa_layout, sizeof(uint32_t) * phase_a_cluster_offsets_cap, alignof(uint32_t)); + layout_add(pa_layout, sizeof(uint16_t) * PHASE_A_DIRTY_SLOTS_CAP, alignof(uint16_t)); + layout_add(pa_layout, sizeof(uint32_t) * PHASE_A_BUCKET_REP_CAP, alignof(uint32_t)); + layout_add(pa_layout, + sizeof(std::pair) * PHASE_A_STAGED_CAP, + alignof(std::pair)); + layout_add(pa_layout, sizeof(AffineElement) * PHASE_A_CHUNK_CAP, alignof(AffineElement)); + layout_add(pa_layout, sizeof(uint32_t) * PHASE_A_CHUNK_CAP, alignof(uint32_t)); + } + + per_worker_union_bytes = align_up(std::max(ts_fixed_layout, pa_layout), WORKER_SLAB_ALIGN); + + // Stage 6 wpb-dependent tail (dense_buckets / is_present / pair + // scratch / chunk_infos). Skipped when windows_per_batch == 0 (sizer's + // pre-wpb-solve call). + if (windows_per_batch != 0) { + const size_t dense_total = windows_per_batch * dense_stride_est; + const size_t dense_pair_max = dense_total / 2; + layout_add(per_worker_per_wpb_layout, sizeof(AffineElement) * dense_total, alignof(AffineElement)); + layout_add(per_worker_per_wpb_layout, sizeof(uint8_t) * dense_total, alignof(uint8_t)); + layout_add(per_worker_per_wpb_layout, + sizeof(std::pair) * dense_pair_max, + alignof(std::pair)); + layout_add(per_worker_per_wpb_layout, sizeof(uint32_t) * dense_pair_max, alignof(uint32_t)); + layout_add(per_worker_per_wpb_layout, sizeof(BaseField) * dense_pair_max, alignof(BaseField)); + layout_add(per_worker_per_wpb_layout, + sizeof(AffineBucketChunkInfo) * windows_per_batch, + alignof(AffineBucketChunkInfo)); + } + + per_worker_bytes = align_up(per_worker_union_bytes + per_worker_per_wpb_layout, WORKER_SLAB_ALIGN); + } +}; + +// Stride upper bound for `s.dense_buckets`: next_pow2(⌈(B-1)/T⌉), with a floor of 2. +[[nodiscard]] inline size_t compute_dense_stride(size_t B_eff, size_t num_threads) noexcept +{ + const size_t per_thread = (B_eff > 1) ? ((B_eff - 1 + num_threads - 1) / num_threads) : size_t{ 1 }; + return std::max(2, std::bit_ceil(per_thread)); +} + +// Upper bound on Σ_t buckets_per_thread[t][w] per window: B + T - 1 (adjacent threads +// may share one boundary bucket). Returns 0 when B_eff == 0. +[[nodiscard]] inline size_t compute_bucket_partials_max(size_t B_eff, size_t num_threads) noexcept +{ + return (B_eff > 0) ? (B_eff - 1 + num_threads - 1) : size_t{ 0 }; +} + +// Per-OS-thread Stage 6a seam overflow capacity (per-window upper bound). +[[nodiscard]] inline size_t compute_global_max_overflow_per_window(size_t n, + size_t num_threads, + size_t subchunk_entries_cap) noexcept +{ + const size_t global_max_chunk_len = (n + num_threads - 1) / num_threads; + return (global_max_chunk_len + subchunk_entries_cap - 1) / subchunk_entries_cap; +} + +// Per-window byte cost for one window in a windows-per-batch slab. Identical formula +// at three sites (sizer outer, sizer per-schedule lambda, live allocator); centralised +// here so they cannot drift. +// +// schedule = 4·n +// HIST slot = max(4·t·B, sizeof(ChunkOutput)·t + 96·t) [H ∪ O overlay] +// DENSE slot = 65 · bucket_partials_max(B, t) [bucket_partials_dense + present] +// bucket_start = 8·(B+1) +// chunk arrays = 8·(t+1) + 8·(t+1) + 8·t + 8·t + 8·t + 16·worker + 8·t +// dense_buckets = 87·worker·stride [s.dense_buckets + aux] +template +[[nodiscard]] inline size_t compute_per_window_bytes( + size_t num_threads, size_t B_eff, size_t n, size_t dense_stride, size_t worker_total) noexcept +{ + const size_t bucket_partials_max = compute_bucket_partials_max(B_eff, num_threads); + const size_t hist_h_bytes_pw = size_t{ 4 } * num_threads * B_eff; + const size_t hist_o_bytes_pw = (sizeof(ChunkOutput) * num_threads) + (size_t{ 96 } * num_threads); + const size_t hist_slot_bytes_pw = std::max(hist_h_bytes_pw, hist_o_bytes_pw); + const size_t dense_slot_bytes_pw = size_t{ 65 } * bucket_partials_max; + return (size_t{ 4 } * n) + hist_slot_bytes_pw + dense_slot_bytes_pw + (size_t{ 8 } * (B_eff + 1)) + + (size_t{ 8 } * (num_threads + 1)) + (size_t{ 8 } * (num_threads + 1)) + (size_t{ 8 } * num_threads) + + (size_t{ 8 } * num_threads) + (size_t{ 8 } * num_threads) + (size_t{ 16 } * worker_total) + + (size_t{ 8 } * num_threads) + (size_t{ 87 } * worker_total * dense_stride); +} + +// Phase-1 prologue bytes living in the per-MSM arena (msb_per_scalar, glv_scalars, +// glv_points, per_thread_msb_hist). Two-copy duplicate eliminated. +[[nodiscard]] inline size_t compute_phase_one_prologue_bytes(size_t n, + bool use_glv, + bool inline_glv_double, + size_t profile_threads) noexcept +{ + return n // msb_per_scalar + + (use_glv ? size_t{ 32 } * n : size_t{ 0 }) // glv_scalars_storage + + (inline_glv_double ? size_t{ 64 } * n : size_t{ 0 }) // glv_points_storage + + (profile_threads * size_t{ 1024 }); // per_thread_msb_hist +} + +struct PhaseACaps { + size_t members_cap; + size_t offsets_cap; +}; + +// Phase A per-worker caps. `members_cap = min(DEDUP_MAX_MEMBERS, n)` is tight (each +// scalar contributes ≤ 1 cluster_member entry). `offsets_cap = cids_per_thread + 2` +// covers the leading-zero sentinel + post-last terminator. +[[nodiscard]] inline PhaseACaps compute_phase_a_caps(size_t n, size_t num_threads) noexcept +{ + return { std::min(DEDUP_MAX_MEMBERS, n), (DEDUP_MAX_CLUSTERS / num_threads) + 2 }; +} + +// Solve `wpb · per_window_bytes ≤ available_budget`, clamped to W_R and ≥ 1. +// Mirrors the three identical wpb-pickers in the sizer and live allocator. +[[nodiscard]] inline size_t solve_wpb(size_t per_window_bytes, size_t available_budget, size_t W_R) noexcept +{ + if (W_R == 0) { + return 1; + } + if (per_window_bytes == 0 || available_budget == 0) { + return std::max(1, W_R); + } + return std::min(std::max(1, available_budget / per_window_bytes), W_R); +} + +} // namespace bb::scalar_multiplication::round_parallel_detail diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_batched.hpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_batched.hpp new file mode 100644 index 000000000000..a2af6967baac --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_batched.hpp @@ -0,0 +1,279 @@ +#pragma once + +// Implementation fragment included from scalar_multiplication_fast.cpp inside +// bb::scalar_multiplication, after pippenger_round_parallel is defined. + +// Multi-MSM_fast driver for `MSM_fast<>::batch_multi_scalar_mul`. The hot path +// (`CommitmentKey::batch_commit` from `commit_to_wires`) batches K MSMs sharing the same +// SRS subspan. We do NOT interleave K MSMs inside a single parallel_for body — that +// K-multiplies the per-thread working set and forces windows_in_batch=1; the single-MSM_fast +// hot path is tuned to fit ~4 MiB in L2 and we want to preserve that. The loop is just +// for m in 0..K: run single-MSM_fast dispatch for MSM_fast m. +// The only cross-MSM_fast amortisation is the GLV-doubled point set: when every member of a +// shared-SRS-prefix group wants GLV, we double the prefix once into a shared buffer and +// each per-MSM_fast call aliases its prefix instead of doubling its own. +namespace round_parallel_detail { + +// One per shared-SRS-prefix group. Membership is keyed on identical +// `point_arrays[m].data()` pointers — that is the actual sharing relation +// `commit_to_wires` exposes. Static-lifetime so the doubled buffer survives +// across calls (typical workloads commit the same SRS prefix repeatedly). +template struct BatchMsmGlvGroup { + const typename Curve::AffineElement* base_ptr = nullptr; // SRS prefix pointer + size_t group_max_n = 0; // max n_input across MSMs in this group + std::span doubled; // length 2 * group_max_n; aliases a prefix of + // the master-group buffer (computed once for + // the largest GLV-using group). Layout + // `[P_0, φP_0, P_1, φP_1, …]` — the first 2*n + // entries are the per-MSM_fast view for n ≤ Nmax. + std::vector member_msms; // indices into `scalar_arrays` of MSMs in this group +}; + +} // namespace round_parallel_detail + +namespace { +// NOLINTNEXTLINE(readability-function-size, readability-function-cognitive-complexity, +// google-readability-function-size) +template +void pippenger_round_parallel_batched(std::span> scalar_arrays, + std::span> point_arrays, + std::vector& out_results, + std::span dedup_hints = {}) noexcept +{ + using AffineElement = typename Curve::AffineElement; + using ScalarField = typename Curve::ScalarField; + using BaseField = typename Curve::BaseField; + + BB_BENCH_NAME("MSM_fast::pippenger_round_parallel_batched"); + + const size_t K = scalar_arrays.size(); + BB_ASSERT_EQ(point_arrays.size(), K); + out_results.assign(K, Curve::Group::point_at_infinity); + + auto hint_for = [&](size_t m) noexcept -> bool { return m < dedup_hints.size() && dedup_hints[m] != 0; }; + + if (K == 0) { + return; + } + if (K == 1) { + const size_t n = std::min(scalar_arrays[0].size(), point_arrays[0].size()); + if (n == 0) { + return; + } + PolynomialSpan sp(0, std::span(scalar_arrays[0].data(), n)); + out_results[0] = pippenger_round_parallel(sp, point_arrays[0], hint_for(0)); + return; + } + + std::vector n_input(K); + for (size_t m = 0; m < K; ++m) { + n_input[m] = std::min(scalar_arrays[m].size(), point_arrays[m].size()); + } + + // Group MSMs by shared SRS pointer; one shared GLV-doubled buffer per group, sized to + // group_max_n. group_uses_glv is a per-group bool but the per-MSM_fast internal dispatch keeps + // each MSM_fast's own GLV decision in case shared doubling is skipped. + using GlvGroup = round_parallel_detail::BatchMsmGlvGroup; + std::vector glv_groups; + + auto find_or_create_group = [&](const AffineElement* base_ptr, size_t n) -> size_t { + for (size_t g = 0; g < glv_groups.size(); ++g) { + if (glv_groups[g].base_ptr == base_ptr) { + glv_groups[g].group_max_n = std::max(glv_groups[g].group_max_n, n); + return g; + } + } + GlvGroup g{}; + g.base_ptr = base_ptr; + g.group_max_n = n; + glv_groups.push_back(std::move(g)); + return glv_groups.size() - 1; + }; + + std::vector msm_to_group(K, std::numeric_limits::max()); + for (size_t m = 0; m < K; ++m) { + if (n_input[m] == 0) { + continue; + } + const size_t g = find_or_create_group(point_arrays[m].data(), n_input[m]); + glv_groups[g].member_msms.push_back(m); + msm_to_group[m] = g; + } + + std::vector group_uses_glv(glv_groups.size(), false); + for (size_t g = 0; g < glv_groups.size(); ++g) { + // GLV decision is per-group on group_max_n. Within a group, every MSM_fast has + // n[m] <= group_max_n; if group_max_n is in the small-N regime, every MSM_fast + // is too, so they all want GLV. If group_max_n is in the large-N regime, + // no MSM_fast in the group wants GLV (they'd be slower with it). + group_uses_glv[g] = glv_groups[g].group_max_n <= round_parallel_detail::GLV_SMALL_N_THRESHOLD; + } + + // Build ONE shared GLV-doubled buffer covering the union of every GLV-using group's + // SRS range, then alias each group's `doubled` into a slice of that buffer. + // + // Every production / test caller of batch_multi_scalar_mul is `commitment_key.batch_commit`, + // which constructs each MSM_fast's point span as `get_monomial_points().subspan(start_index)` + // — sub-spans of a single contiguous `std::vector` SRS. So in every + // batch every group's `base_ptr` lives in the same allocation and offsets are + // necessarily integer multiples of `sizeof(AffineElement)`. The asserts below + // catch a future caller that violates that contract. + std::unique_ptr master_doubled_owner; // NOLINT(cppcoreguidelines-avoid-c-arrays) + { + BB_BENCH_NAME("MSM_fast::pippenger_round_parallel_batched/glv_double_points"); + + const AffineElement* min_base = nullptr; + for (size_t g = 0; g < glv_groups.size(); ++g) { + glv_groups[g].doubled = {}; + if (!group_uses_glv[g]) { + continue; + } + if (min_base == nullptr || std::less{}(glv_groups[g].base_ptr, min_base)) { + min_base = glv_groups[g].base_ptr; + } + } + + if (min_base != nullptr) { + const auto min_addr = reinterpret_cast(min_base); + size_t max_extent_units = 0; + for (size_t g = 0; g < glv_groups.size(); ++g) { + if (!group_uses_glv[g]) { + continue; + } + const auto base_addr = reinterpret_cast(glv_groups[g].base_ptr); + const uintptr_t offset_bytes = base_addr - min_addr; + BB_ASSERT_EQ(offset_bytes % sizeof(AffineElement), + size_t{ 0 }, + "GLV group base_ptr not aligned to AffineElement boundary " + "(point spans must be subranges of a contiguous AffineElement array)"); + const size_t offset_units = offset_bytes / sizeof(AffineElement); + const size_t end_units = offset_units + glv_groups[g].group_max_n; + max_extent_units = std::max(max_extent_units, end_units); + } + + master_doubled_owner = std::make_unique_for_overwrite( + 2 * max_extent_units); // NOLINT(cppcoreguidelines-avoid-c-arrays) + AffineElement* const master_buf = master_doubled_owner.get(); + const BaseField beta = BaseField::cube_root_of_unity(); + bb::parallel_for(bb::get_num_cpus(), [&](const ThreadChunk& chunk) { + BB_BENCH_NAME("MSM_fast::batch_glv_double/worker"); + for (size_t i : chunk.range(max_extent_units)) { + master_buf[2 * i] = min_base[i]; + master_buf[(2 * i) + 1].x = min_base[i].x * beta; + master_buf[(2 * i) + 1].y = -min_base[i].y; + } + }); + + for (size_t g = 0; g < glv_groups.size(); ++g) { + if (!group_uses_glv[g]) { + continue; + } + const auto base_addr = reinterpret_cast(glv_groups[g].base_ptr); + const size_t offset_units = (base_addr - min_addr) / sizeof(AffineElement); + glv_groups[g].doubled = + std::span(master_buf + (2 * offset_units), 2 * glv_groups[g].group_max_n); + } + } + } + + // Shared dynamically-sized arena for all per-MSM_fast internal calls. Sized to the max + // requirement across the batch so each MSM_fast finds enough space. Single allocation + // across the batch (vs one per MSM_fast if we passed {} down). Freed at return. + // dedup_active varies per MSM_fast (gated by per-MSM_fast hint), so the budget query must + // mirror the predicate used inside pippenger_round_parallel. + size_t shared_arena_bytes = 0; + for (size_t m = 0; m < K; ++m) { + if (n_input[m] == 0) { + continue; + } + const size_t g = msm_to_group[m]; + const bool ext_glv = + g != std::numeric_limits::max() && group_uses_glv[g] && !glv_groups[g].doubled.empty(); + // The internal short-circuits to trivial_msm_threaded for tiny MSMs, so the hint + // alone is the right arena-sizing predicate (over-sizing for a path that bails + // is harmless — under-sizing would crash). + const bool dedup_active_m = hint_for(m); + const size_t bytes = compute_arena_bytes_for_msm(n_input[m], ext_glv, dedup_active_m); + shared_arena_bytes = std::max(shared_arena_bytes, bytes); + } + std::unique_ptr shared_arena_owner; // NOLINT(cppcoreguidelines-avoid-c-arrays) + std::span shared_arena; + if (shared_arena_bytes > 0) { + shared_arena_owner = + std::make_unique_for_overwrite(shared_arena_bytes); // NOLINT(cppcoreguidelines-avoid-c-arrays) + shared_arena = std::span(shared_arena_owner.get(), shared_arena_bytes); + } + + // Per-MSM_fast dispatch. Each call runs the full single-MSM_fast pipeline (its own from-Mont and + // to-Mont, schedule, Stage 1-6b). The only batched amortisation we share is the doubled + // SRS prefix above; the rest of the hot path runs at single-MSM_fast cost. + for (size_t m = 0; m < K; ++m) { + const size_t n = n_input[m]; + if (n == 0) { + continue; + } + PolynomialSpan sp(0, std::span(scalar_arrays[m].data(), n)); + + const size_t g = msm_to_group[m]; + std::span external_glv; + if (g != std::numeric_limits::max() && group_uses_glv[g]) { + // `group.doubled` is interleaved `[P_0, φP_0, …]` of length 2*Nmax. The + // first 2*n entries are exactly the per-MSM_fast `[P_0, φP_0, …, P_{n-1}, φP_{n-1}]` + // view, regardless of whether n == Nmax (uniform batch) or n < Nmax (ragged). + external_glv = std::span(glv_groups[g].doubled.data(), 2 * n); + } + + out_results[m] = pippenger_round_parallel(sp, point_arrays[m], hint_for(m), external_glv, shared_arena); + } +} +} // namespace + +template +std::vector MSM_fast::batch_multi_scalar_mul( + std::span points, + std::span> scalars, + bool handle_edge_cases, + std::span dedup_hints) noexcept +{ + BB_BENCH_NAME("MSM_fast::batch_multi_scalar_mul"); + const size_t k = scalars.size(); + + // Adapt the new (single shared points span + per-MSM_fast PolynomialSpan scalars) API to + // the internal dispatcher, which still takes one point sub-span per MSM_fast. Each MSM_fast's + // sub-span is `points[start_index .. start_index + size)`; the dispatcher's existing + // GLV-doubled-buffer grouping then deduplicates across MSMs that fall in the same + // underlying allocation. + std::vector> point_subspans; + std::vector> scalar_subspans; + point_subspans.reserve(k); + scalar_subspans.reserve(k); + for (size_t i = 0; i < k; ++i) { + const size_t start_i = scalars[i].start_index; + BB_ASSERT_LTE(start_i, points.size(), "scalars[m].start_index exceeds shared points span"); + point_subspans.push_back(points.subspan(start_i, points.size() - start_i)); + scalar_subspans.push_back(scalars[i].span); + } + + auto hint_for = [&](size_t m) noexcept -> bool { return m < dedup_hints.size() && dedup_hints[m] != 0; }; + + if (handle_edge_cases) { + std::vector results(k); + for (size_t i = 0; i < k; ++i) { + const size_t n = std::min(point_subspans[i].size(), scalar_subspans[i].size()); + PolynomialSpan scalar_span(0, + std::span(scalar_subspans[i].data(), n)); + results[i] = + AffineElement(pippenger_fast(scalar_span, point_subspans[i], handle_edge_cases, hint_for(i))); + } + return results; + } + + std::vector per_msm_jac; + pippenger_round_parallel_batched(scalar_subspans, point_subspans, per_msm_jac, dedup_hints); + + std::vector results(k); + for (size_t i = 0; i < k; ++i) { + results[i] = AffineElement(per_msm_jac[i]); + } + return results; +} diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_dedup.hpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_dedup.hpp new file mode 100644 index 000000000000..2c6fe586df0f --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_dedup.hpp @@ -0,0 +1,586 @@ +// Input-scalar dedup pre-pass for the round-parallel Pippenger MSM (Phase A). +// +// Detects clusters of input scalars whose canonical (non-Montgomery) value is identical +// and spans more than one signed-Booth window, then combines each cluster's base points +// into a single (rep, combined_point) pair via a chunked batched-affine tree-reduce. +// Stage 4 then sees a redirect_lookup that rewrites the cluster's schedule entries: +// the rep gets DEDUP_REDIRECT_BIT|extra_idx (fetched from extra_points[]), the rest get +// DEDUP_SKIP_BIT and contribute nothing. This carves out two bits of the 32-bit schedule +// encoding (bit 30 = redirect, bit 29 = skip), which is why this header also owns the +// full schedule-bit encoding constants (the sign bit, the dedup bits, and the index +// mask are all co-defined). +// +// The encoding constants and the `dedup_*` workers are pulled into one file so: +// * scalar_multiplication.cpp's Stage 4 / Stage 6a schedule readers see the bit +// constants via this header; +// * the dedup machinery (Phase A workers, hash table, cluster tree-reduce, redirect +// finalize) lives as a self-contained module rather than being scattered through +// the MSM driver. Pure code motion — every function is inline / templated, so the +// compiler sees identical code at identical call sites and codegen is unchanged. + +#pragma once + +#include "./pippenger_arena_layout.hpp" + +#include "barretenberg/common/assert.hpp" +#include "barretenberg/common/bb_bench.hpp" +#include "barretenberg/common/compiler_hints.hpp" +#include "barretenberg/common/thread.hpp" + +#include +#include +#include +#include + +namespace bb::scalar_multiplication::round_parallel_detail { + +// 32-bit schedule-entry encoding. Stage 4 stores only the point sign and scalar index; +// bucket magnitude is recovered from Stage 3's bucket_start ranges in Stage 5/6 because +// the schedule is bucket-contiguous. +// bit 31: sign bit from the packed signed digit +// bit 30: dedup redirect — fetch from extra_points[payload] +// bit 29: dedup skip — non-rep duplicate, carries no contribution +// bits 0..28: scalar_idx, or extra_points index when redirect is set +inline constexpr uint32_t SCHEDULE_SIGN_BIT = uint32_t{ 1 } << 31; +inline constexpr uint32_t DEDUP_REDIRECT_BIT = uint32_t{ 1 } << 30; +inline constexpr uint32_t DEDUP_SKIP_BIT = uint32_t{ 1 } << 29; +inline constexpr uint32_t SCHEDULE_INDEX_MASK = DEDUP_SKIP_BIT - 1; +static_assert((SCHEDULE_SIGN_BIT & DEDUP_REDIRECT_BIT) == 0); +static_assert((SCHEDULE_SIGN_BIT & DEDUP_SKIP_BIT) == 0); +static_assert((DEDUP_REDIRECT_BIT & DEDUP_SKIP_BIT) == 0); +static_assert((SCHEDULE_INDEX_MASK & (SCHEDULE_SIGN_BIT | DEDUP_REDIRECT_BIT | DEDUP_SKIP_BIT)) == 0); +static_assert((SCHEDULE_INDEX_MASK | DEDUP_REDIRECT_BIT | DEDUP_SKIP_BIT) == ~SCHEDULE_SIGN_BIT); +inline constexpr uint32_t DEDUP_INVALID_EXTRA = ~uint32_t{ 0 }; + +[[nodiscard]] inline uint64_t dedup_scalar_fingerprint(const uint64_t* scalar_data) noexcept +{ + return scalar_data[0]; +} + +[[nodiscard]] inline size_t dedup_fingerprint_slot(uint64_t fingerprint, size_t mask) noexcept +{ + uint64_t h = fingerprint * 0x9E3779B97F4A7C15ULL; + h ^= h >> 32; + return static_cast(h) & mask; +} + +// =================================================================================== +// Input-scalar dedup pre-pass. +// =================================================================================== +// +// For each cluster of input scalars whose canonical value is identical and spans more +// than one bucket window of width c (msb >= c), combine the cluster's base points into +// a single (rep, combined_point) pair so Pippenger only iterates the cluster once +// instead of `cluster_size` times. +// +// Detection: sort an index permutation by `scalars[i].data[0]` (a one-limb predicate; +// equal-value scalars are guaranteed to collide on data[0] so they cluster contiguously +// in the sorted output, with at most a few false-collision PAIRS expected per MSM at +// chonk's scale). Walk runs of equal data[0]; verify each pair with a full memcmp. +// +// Combine: build a flat (cluster_pts, cluster_ids) array with same-cluster entries +// contiguous, then run an in-place tree-reduce that pairs adjacent same-cluster-id +// entries via `batch_affine_add_interleaved` (one inversion per BATCH_CAPACITY pairs) +// until each cluster has a single surviving entry. Avoids the per-cluster Element += / +// AffineElement(cast) round-trip that does one inversion per cluster. +// +// Output: a redirect_lookup[n] mapping scalar_idx → final dedup schedule payload +// (DEDUP_REDIRECT_BIT | extra_idx, DEDUP_SKIP_BIT | scalar_idx, or INVALID = no patch). +// Stage 4b ORs that payload with the preserved sign bit. The underlying canonical scalar +// value is left untouched (`scalars` aliases the caller's polynomial and is restored to +// Mont form on exit; mutating it would corrupt downstream consumers). + +template struct DedupResult { + std::span redirect_lookup; // size n; INVALID or encoded dedup payload. + // Allocated from the pippenger arena + // (no zero-init); filled with INVALID + // by a parallel_for before Phase A. + std::span extra_points; // size DEDUP_MAX_CLUSTERS; arena-allocated. + // Phase A writes per-cluster aggregates + // into thread-disjoint cid ranges. + size_t n_dedup_extras = 0; // # extra_points populated by Phase A +}; + +// In-place batched-affine tree-reduce over (pts[0..len), cluster_ids[0..len)) with +// same-cluster entries contiguous. After return, pts[0..result_len) holds one combined +// point per cluster (paired in cluster-id order); ids[0..result_len) tracks the +// surviving cluster_id at each slot. Caller-provided scratch (`scratch_pts`, +// `pair_dest`, `inversion_scratch`) sized to BATCH_CAPACITY pairs. +template +inline size_t dedup_tree_reduce_in_place(typename Curve::AffineElement* pts, + uint32_t* ids, + size_t initial_len, + typename Curve::AffineElement* scratch_pts, + uint32_t* pair_dest, + typename Curve::BaseField* inversion_scratch) noexcept +{ + using AffineElement = typename Curve::AffineElement; + using BaseField = typename Curve::BaseField; + + const auto drain = [&](size_t pair_count) noexcept { + if (pair_count == 0) { + return; + } + bb::group_elements::batch_affine_add_interleaved( + scratch_pts, 2 * pair_count, inversion_scratch); + for (size_t k = 0; k < pair_count; ++k) { + pts[pair_dest[k]] = scratch_pts[pair_count + k]; + } + }; + + size_t curr_len = initial_len; + while (true) { + size_t i = 0; + size_t next_len = 0; + size_t pair_count = 0; + bool made_pair = false; + + while (i < curr_len) { + if (i + 1 < curr_len && ids[i] == ids[i + 1]) { + scratch_pts[2 * pair_count] = pts[i]; + scratch_pts[(2 * pair_count) + 1] = pts[i + 1]; + ids[next_len] = ids[i]; + pair_dest[pair_count] = static_cast(next_len); + ++next_len; + ++pair_count; + i += 2; + made_pair = true; + if (pair_count >= BATCH_CAPACITY) { + drain(pair_count); + pair_count = 0; + } + } else { + pts[next_len] = pts[i]; + ids[next_len] = ids[i]; + ++next_len; + ++i; + } + } + drain(pair_count); + + if (!made_pair) { + break; + } + curr_len = next_len; + } + return curr_len; +} + +// Hard caps that bound the worst-case dedup-on memory at ≤ 4 MB above dedup-off, for +// all possible inputs. +// +// redirect_lookup[n] = 4 n bytes (≤ 2 MB at n = 2^19) +// extra_points[MAX_CLUSTERS] = 64 × MAX_CLUSTERS bytes +// per-thread Phase A scratch: bounded by per-thread chunk size + chunk_pts buffer +// +// All phases ≤ 4 MB regardless of input shape. The caps degrade gracefully: when hit +// we leave un-deduped scalars on the standard pippenger path (still correct, just +// less savings). +// `DEDUP_MAX_CLUSTERS`, `DEDUP_MAX_MEMBERS`, and `DEDUP_MAX_CHUNK_MEMBERS` are defined +// in `pippenger_arena_layout.hpp` so the test harness can size the matching slabs. +static_assert(DEDUP_MAX_CLUSTERS <= size_t{ SCHEDULE_INDEX_MASK } + 1, + "dedup extra-point ids must fit in the schedule payload"); + +// Per-worker Phase A scratch backed by the pippenger arena. Replaces the prior +// `thread_local std::vector<...>` slabs so process-resident memory after the MSM +// drops back to zero and the per-worker working set is deterministic. +// +// All caps below are *loose upper bounds* — when a runtime population exceeds them, +// the cluster-scan / tree-reduce inner loops already fall through to "leave un-deduped" +// behaviour via `clusters_opened >= cid_max - cid_lo` and the `room` calculation in the +// tree-reduce chunk-fill loop. +template struct PhaseAScratch { + // Worst case = every cluster on this worker has exactly one member. Per-worker + // cluster budget is `DEDUP_MAX_CLUSTERS / num_threads`; `DEDUP_MAX_MEMBERS / num_threads` + // is a looser, structurally simpler bound. +1 covers the final partition rounding slop. + std::span cluster_members; + // One entry per opened cluster + the initial 0 sentinel pushed at function entry. + // Cap = (DEDUP_MAX_CLUSTERS / num_threads) + 2 (covers the +1 sentinel and rounding). + std::span cluster_offsets; + // One uint16_t per hash-table slot dirtied since the last bucket; HT_SIZE = 4096 is + // the structural cap — every slot can at most be dirtied once per bucket. + std::span dirty_slots; + // Per-bucket cluster representative scalar_idx. Current code reserves 32; widening + // to 256 covers chonk-wire worst cases (mega-buckets) without resizing. + std::span bucket_rep; + // Per-bucket staged (bucket_cid, idx) pairs awaiting cluster emission. Current code + // reserves 64; widening to 1024 covers the chonk-wire mega-bucket worst case. + std::span> staged; + // Tree-reduce per-iteration working sets. Both capped at DEDUP_MAX_CHUNK_MEMBERS=2048; + // see the constant's definition above. + std::span chunk_pts; + std::span chunk_ids; +}; + +// Per-bucket hash-based dedup. Each thread owns a contiguous range of buckets in +// window 0's schedule. For each bucket, we build a tiny open-addressing hash +// table over the long-scalar entries (msb >= c_threshold) — short entries are +// skipped because their dedup savings (W_nz ≈ 1) are zero. Slot selection uses +// a cheap one-limb fingerprint; full 4-limb memcmp still gates every match. +// Hash collisions resolve via linear probing; same-value collisions become cluster matches. +// Replaces the old "std::sort each bucket then run consecutive-pair walk" +// approach: hash is O(K) per bucket vs O(K log K), avoids the 32-byte memcmp +// comparator entirely (one-limb hash on insert, full compare only on fingerprint +// hits), and keeps thread balance uniform because skipping shorts removes the +// mega-bucket bottleneck. +// +// Output: per-thread cluster_members + cluster_offsets feeding a chunked +// batched-affine tree-reduce, plus encoded redirect_lookup writes +// (rep -> DEDUP_REDIRECT_BIT | cid, non_rep -> DEDUP_SKIP_BIT | idx). +// The thread's cid space is the disjoint per-thread sub-range [cid_lo, cid_max). +template +size_t dedup_phase_a_worker_hash(const uint32_t* schedule_w0, + const size_t* w0_bucket_start, + size_t b_lo, + size_t b_hi, + std::span scalars, + std::span points, + std::span extra_points, + std::span redirect_lookup, + const uint8_t* msb_per_scalar, + size_t c_threshold, + uint32_t cid_lo, + uint32_t cid_max, + PhaseAScratch& scratch) noexcept +{ + using AffineElement = typename Curve::AffineElement; + using BaseField = typename Curve::BaseField; + constexpr uint32_t HT_EMPTY = ~uint32_t{ 0 }; + + // Per-thread hash table — sized for the largest expected bucket. Long- + // scalar density per bucket is highly NON-uniform on chonk wires: the few + // buckets corresponding to digit_0 ∈ {1,2,3,…} hold 700+ long entries with + // 500+ distinct values. A 256-slot table fills up and the open-addressing + // probe goes infinite. A 4096-slot table keeps load <25% even on the worst + // bucket. 4096 × 4 = 16 KB per thread. + // + // We use LAZY CLEARING via a dirty-slot list rather than std::fill_n per + // bucket: a 16 KB fill × ~2 K buckets × 8 threads = 256 MB of write traffic + // per Phase A, which dominates the cluster-scan wall (≈ 280 ms / 450 ms on + // the WASM trace). With lazy clear the per-bucket reset cost scales with + // the number of slots ACTUALLY written (typically 25-700), not 4096. + constexpr size_t HT_SIZE = 4096; + constexpr size_t HT_MASK = HT_SIZE - 1; + static_assert((HT_SIZE & (HT_SIZE - 1)) == 0, "HT_SIZE must be a power of 2"); + std::array ht; + + // The hash table maps scalar_value → either (a) the singleton scalar_idx + // observed first, or (b) a sentinel pointing into cluster_members for an + // already-opened cluster. We disambiguate via a separate parallel slot + // status array (bit-set if slot holds a cluster pointer). To keep the data + // structure simple, we instead use TWO sentinel bits in the high end of + // the uint32_t scalar_idx: + // high bit clear → slot holds a singleton scalar_idx (just one observation) + // high bit set → slot holds (cluster_id | HT_CLUSTER_BIT) + // scalar_idx values fit the schedule payload (29 bits), so the top 3 bits are free. + constexpr uint32_t HT_CLUSTER_BIT = uint32_t{ 1 } << 31; + + // Per-worker arena-backed scratch spans. Caller allocates `scratch` once at the start + // of the MSM (see `pippenger_round_parallel_internal`); we treat them as bounded + // capacity buffers with a logical-size cursor. No allocator churn, no thread_local + // process state to clean up after the MSM returns. + uint32_t* const cluster_members_data = scratch.cluster_members.data(); + const size_t cluster_members_cap = scratch.cluster_members.size(); + size_t cluster_members_size = 0; + uint32_t* const cluster_offsets_data = scratch.cluster_offsets.data(); + const size_t cluster_offsets_cap = scratch.cluster_offsets.size(); + size_t cluster_offsets_size = 0; + uint16_t* const dirty_slots_data = scratch.dirty_slots.data(); + const size_t dirty_slots_cap = scratch.dirty_slots.size(); + size_t dirty_slots_size = 0; + { + BB_BENCH_NAME("MSM::PhaseA/alloc_buffers"); + // Cluster offsets always pushes a 0 sentinel first. + BB_ASSERT_GTE(cluster_offsets_cap, size_t{ 1 }); + cluster_offsets_data[cluster_offsets_size++] = 0; + } + + // Initial fill — uninitialised stack memory could match HT_EMPTY values + // by coincidence. After this, clearing is incremental via dirty_slots. + std::fill_n(ht.data(), HT_SIZE, HT_EMPTY); + // Slot-local one-limb fingerprints for occupied hash-table entries. They are + // valid iff `ht[slot] != HT_EMPTY`; lazy clearing only needs to reset `ht`. + std::array ht_fingerprint; + + uint32_t clusters_opened = 0; + { + BB_BENCH_NAME("MSM::PhaseA/cluster_scan"); + // Per-bucket scratch — both backed by arena spans (caller-allocated). + // - `bucket_rep[bucket_cid]` = scalar_idx of the rep for that in-bucket cluster. + // - `staged[..]` = (bucket_cid, idx) pairs awaiting cluster emission. + uint32_t* const bucket_rep_data = scratch.bucket_rep.data(); + const size_t bucket_rep_cap = scratch.bucket_rep.size(); + size_t bucket_rep_size = 0; + std::pair* const staged_data = scratch.staged.data(); + const size_t staged_cap = scratch.staged.size(); + size_t staged_size = 0; + + for (size_t b = b_lo; b < b_hi; ++b) { + const size_t lo = w0_bucket_start[b]; + const size_t hi = w0_bucket_start[b + 1]; + if (hi - lo < 2) { + continue; + } + + // Lazy clear: reset only slots dirtied by the previous bucket. + for (size_t k = 0; k < dirty_slots_size; ++k) { + ht[dirty_slots_data[k]] = HT_EMPTY; + } + dirty_slots_size = 0; + bucket_rep_size = 0; + staged_size = 0; + + for (size_t i = lo; i < hi; ++i) { + const uint32_t idx = schedule_w0[i] & SCHEDULE_INDEX_MASK; + if (static_cast(msb_per_scalar[idx]) < c_threshold) { + continue; + } + const uint64_t* d = scalars[idx].data; + const uint64_t fingerprint = dedup_scalar_fingerprint(d); + size_t slot = dedup_fingerprint_slot(fingerprint, HT_MASK); + + // Probe-count safety net. With HT_SIZE = 4096 and per-bucket distinct- + // long-value counts up to ~700 on chonk wires, table load is ≤ 17 % + // and the average probe length is ≈ 1.1 — but if any future workload + // produces a bucket dense enough to fill the table, fall back to + // "treat as singleton, don't dedup" rather than infinite-loop. + size_t probe_count = 0; + while (true) { + if (++probe_count > HT_SIZE) { + break; + } + const uint32_t entry = ht[slot]; + if (entry == HT_EMPTY) { + ht[slot] = idx; + ht_fingerprint[slot] = fingerprint; + // If the dirty-slot list overflows its cap we must NOT skip the + // record — every subsequent bucket would then leak slots forward. + // Cap is HT_SIZE so this is structurally unreachable. + if (BB_LIKELY(dirty_slots_size < dirty_slots_cap)) { + dirty_slots_data[dirty_slots_size++] = static_cast(slot); + } + break; + } + if ((entry & HT_CLUSTER_BIT) != 0) { + const uint32_t bucket_cid = entry & ~HT_CLUSTER_BIT; + const uint32_t rep = bucket_rep_data[bucket_cid]; + if (ht_fingerprint[slot] == fingerprint && + std::memcmp(d, scalars[rep].data, sizeof(scalars[rep].data)) == 0) { + // Out of staged-pair capacity: leave this duplicate un-deduped + // (it will go through the standard pippenger path). + if (BB_UNLIKELY(staged_size >= staged_cap)) { + break; + } + staged_data[staged_size++] = { bucket_cid, idx }; + break; + } + slot = (slot + 1) & HT_MASK; + continue; + } + // Singleton at slot: compare values. + if (ht_fingerprint[slot] == fingerprint && + std::memcmp(d, scalars[entry].data, sizeof(scalars[entry].data)) == 0) { + if (clusters_opened >= (cid_max - cid_lo)) { + break; // cap reached, leave un-deduped + } + // Out of bucket_rep / staged capacity: leave un-deduped. + if (BB_UNLIKELY(bucket_rep_size >= bucket_rep_cap || staged_size >= staged_cap)) { + break; + } + const uint32_t bucket_cid = static_cast(bucket_rep_size); + bucket_rep_data[bucket_rep_size++] = entry; + staged_data[staged_size++] = { bucket_cid, idx }; + ht[slot] = HT_CLUSTER_BIT | bucket_cid; + ++clusters_opened; + break; + } + slot = (slot + 1) & HT_MASK; + } + } + + if (bucket_rep_size == 0) { + continue; + } + + // Sort staged non-reps by bucket_cid so each cluster's members are + // contiguous; then emit (rep, non-reps...) per cluster. + std::stable_sort(staged_data, + staged_data + staged_size, + [](const std::pair& a, + const std::pair& b) noexcept { return a.first < b.first; }); + size_t staged_cursor = 0; + for (size_t bc = 0; bc < bucket_rep_size; ++bc) { + // Compute this cluster's member count up front (rep + staged non-reps with + // matching bucket_cid) so we never split a cluster across the slab cap. + // When the next cluster would overflow cluster_members_cap, break cleanly: + // un-flattened cluster reps/members never get a redirect_lookup entry, so + // Stage 4/6a process them as normal scalars with their original signed + // digits. The MSM sum is unchanged; we just deliver less dedup work. + size_t this_cluster_members = 1; // rep + for (size_t sc = staged_cursor; sc < staged_size && staged_data[sc].first == bc; ++sc) { + ++this_cluster_members; + } + if (cluster_members_size + this_cluster_members > cluster_members_cap) { + break; + } + cluster_members_data[cluster_members_size++] = bucket_rep_data[bc]; + while (staged_cursor < staged_size && staged_data[staged_cursor].first == bc) { + cluster_members_data[cluster_members_size++] = staged_data[staged_cursor].second; + ++staged_cursor; + } + // cluster_offsets cap is provably non-overflow given clusters_opened ≤ + // cids_per_thread and cluster_offsets_cap = cids_per_thread + 2; the + // initial 0 sentinel plus at most cids_per_thread end-offsets fits. + cluster_offsets_data[cluster_offsets_size++] = static_cast(cluster_members_size); + } + } + } // MSM::PhaseA/cluster_scan + + // Only flattened clusters are published. `clusters_opened` counts every promoted + // hash-table singleton, including clusters later skipped because cluster_members_cap + // would be exceeded. Skipped clusters intentionally fall through the normal Pippenger + // path because they never get redirect_lookup entries. + const size_t num_clusters = cluster_offsets_size - 1; + if (num_clusters == 0) { + return 0; + } + + // For tree_reduce we need a single contiguous member list; cluster_members_data is + // already such a list, with [cluster_offsets[k], cluster_offsets[k+1]) per cluster. + // cluster_offsets_size = num_clusters + 1 (initial 0 sentinel + one push per cluster). + BB_ASSERT_EQ(cluster_offsets_size, num_clusters + 1, "cluster_offsets layout mismatch"); + + { + BB_BENCH_NAME("MSM::PhaseA/tree_reduce"); + typename Curve::AffineElement* const chunk_pts_data = scratch.chunk_pts.data(); + uint32_t* const chunk_ids_data = scratch.chunk_ids.data(); + const size_t chunk_cap = scratch.chunk_pts.size(); + BB_ASSERT_GTE(chunk_cap, DEDUP_MAX_CHUNK_MEMBERS); + BB_ASSERT_GTE(scratch.chunk_ids.size(), DEDUP_MAX_CHUNK_MEMBERS); + size_t chunk_size = 0; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) + std::array scratch_pts; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) + std::array pair_dest; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) + std::array inversion_scratch; + + size_t cid_cursor = 0; + size_t member_offset_in_cluster = 0; + AffineElement carry{}; + bool has_carry = false; + + while (cid_cursor < num_clusters || has_carry) { + chunk_size = 0; + if (has_carry) { + chunk_pts_data[chunk_size] = carry; + chunk_ids_data[chunk_size] = static_cast(cid_cursor); + ++chunk_size; + has_carry = false; + } + while (cid_cursor < num_clusters && chunk_size < DEDUP_MAX_CHUNK_MEMBERS) { + const size_t cluster_lo = cluster_offsets_data[cid_cursor] + member_offset_in_cluster; + const size_t cluster_hi = cluster_offsets_data[cid_cursor + 1]; + const size_t available = cluster_hi - cluster_lo; + const size_t room = DEDUP_MAX_CHUNK_MEMBERS - chunk_size; + if (available <= room) { + for (size_t k = 0; k < available; ++k) { + chunk_pts_data[chunk_size] = points[cluster_members_data[cluster_lo + k]]; + chunk_ids_data[chunk_size] = static_cast(cid_cursor); + ++chunk_size; + } + ++cid_cursor; + member_offset_in_cluster = 0; + } else { + for (size_t k = 0; k < room; ++k) { + chunk_pts_data[chunk_size] = points[cluster_members_data[cluster_lo + k]]; + chunk_ids_data[chunk_size] = static_cast(cid_cursor); + ++chunk_size; + } + member_offset_in_cluster += room; + break; + } + } + const size_t result_len = dedup_tree_reduce_in_place(chunk_pts_data, + chunk_ids_data, + chunk_size, + scratch_pts.data(), + pair_dest.data(), + inversion_scratch.data()); + const bool last_is_partial = (cid_cursor < num_clusters) && (member_offset_in_cluster > 0); + const size_t whole_count = last_is_partial ? result_len - 1 : result_len; + for (size_t k = 0; k < whole_count; ++k) { + const uint32_t local_cid = chunk_ids_data[k]; + extra_points[cid_lo + local_cid] = chunk_pts_data[k]; + } + if (last_is_partial) { + carry = chunk_pts_data[result_len - 1]; + has_carry = true; + } + } + } // MSM::PhaseA/tree_reduce + + { + BB_BENCH_NAME("MSM::PhaseA/publish_redirects"); + for (size_t k = 0; k < num_clusters; ++k) { + const size_t mlo = cluster_offsets_data[k]; + const size_t mhi = cluster_offsets_data[k + 1]; + const uint32_t rep_idx = cluster_members_data[mlo]; + const uint32_t global_cid = cid_lo + static_cast(k); + redirect_lookup[rep_idx] = DEDUP_REDIRECT_BIT | global_cid; + for (size_t m = mlo + 1; m < mhi; ++m) { + const uint32_t non_rep_idx = cluster_members_data[m]; + redirect_lookup[non_rep_idx] = DEDUP_SKIP_BIT | non_rep_idx; + } + } + } + + return num_clusters; +} + +// Post-Phase-A schedule patcher. Walks a window's already-emitted bucket runs, +// rewrites entries whose scalar_idx has an encoded dedup payload, and compacts +// non-rep DEDUP_SKIP entries out of the schedule. +// The hot Stage 4 emit loop is now dedup-unaware (plain `sched_w[idx] = sign | scalar_idx`); +// all dedup tagging happens here. +// +// This is a free function — NOT a lambda capturing dedup_state by reference — so +// `redirect_lookup` is passed as a raw pointer argument and the inner loop has no +// closure-indirection chain. The only random load per iter is the single +// `redirect_lookup[scalar_idx]` lookup, which lands in L2 for typical MSM sizes. +// `bucket_start` is rewritten in place, so each old bucket end is saved before +// its prefix slot is overwritten with the compacted end. +template +[[gnu::flatten]] inline void dedup_patch_schedule_window(uint32_t* __restrict sched_w, + size_t* __restrict bucket_start, + size_t num_buckets, + const uint32_t* __restrict redirect_lookup) noexcept +{ + static_cast(static_cast(nullptr)); // template tag for symbol disambiguation + size_t write = 0; + size_t old_bucket_start = bucket_start[0]; + bucket_start[0] = 0; + for (size_t bucket = 0; bucket < num_buckets; ++bucket) { + const size_t old_bucket_end = bucket_start[bucket + 1]; + for (size_t read = old_bucket_start; read < old_bucket_end; ++read) { + const uint32_t e = sched_w[read]; + const uint32_t idx = e & SCHEDULE_INDEX_MASK; + const uint32_t patch = redirect_lookup[idx]; + uint32_t out = e; + if (BB_UNLIKELY(patch != DEDUP_INVALID_EXTRA)) { + if ((patch & DEDUP_SKIP_BIT) != 0) { + continue; + } + out = (e & SCHEDULE_SIGN_BIT) | patch; + } + if (write != read || out != e) { + sched_w[write] = out; + } + ++write; + } + old_bucket_start = old_bucket_end; + bucket_start[bucket + 1] = write; + } +} + +} // namespace bb::scalar_multiplication::round_parallel_detail diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_fallbacks.hpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_fallbacks.hpp new file mode 100644 index 000000000000..4d8d8e23c2ca --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_fallbacks.hpp @@ -0,0 +1,112 @@ +#pragma once + +// Implementation fragment included from scalar_multiplication_fast.cpp inside +// bb::scalar_multiplication. + +// Trivial-N fallback. For small n the Pippenger scaffolding (digit extraction, bucket +// scratch allocation, parallel_for dispatch, GLV split, etc.) costs many times more +// than running a Straus-style simultaneous double-and-add in Jacobian. Delegates to +// `Element::straus_msm`, which on endomorphism curves builds a per-point WNAF lookup +// table and amortises ~128 doublings across all N inputs (vs N×128 for naive +// per-point operator*). Robust to all edge cases (zero scalars, points at infinity) +// so this also covers `handle_edge_cases=true` for trivially small N. The single +// Jacobian→affine inversion at the caller boundary (when `MSM_fast<>::msm` constructs an +// `AffineElement` from the returned `Element`) is the only inversion paid. +template +typename Curve::Element trivial_msm(PolynomialSpan scalars_span, + std::span all_points) noexcept +{ + using Element = typename Curve::Element; + using AffineElement = typename Curve::AffineElement; + using ScalarField = typename Curve::ScalarField; + + const size_t n = scalars_span.size(); + if (n == 0) { + return Curve::Group::point_at_infinity; + } + BB_ASSERT_GTE(all_points.size(), scalars_span.start_index + n); + std::span points_view(&all_points[scalars_span.start_index], n); + std::span scalars_view(scalars_span.span.data(), n); + return Element::straus_msm(points_view, scalars_view); +} + +/** + * @brief Multi-threaded straus_msm driver for very-small MSMs. + * + * Splits the input across `bb::parallel_for` workers and runs `Element::straus_msm` on + * each slice. Zero-scalar entries are compacted out before dispatch (callers reach this + * function precisely when n_active << n, so straus_msm shouldn't burn time on dead pairs). + * Sharing the rpmsm pool with the main pippenger_fast keeps per-call dispatch cheap. + */ +template +typename Curve::Element trivial_msm_threaded(PolynomialSpan scalars_span, + std::span all_points) noexcept +{ + using Element = typename Curve::Element; + using AffineElement = typename Curve::AffineElement; + using ScalarField = typename Curve::ScalarField; + const size_t n = scalars_span.size(); + if (n == 0) { + return Curve::Group::point_at_infinity; + } + BB_ASSERT_GTE(all_points.size(), scalars_span.start_index + n); + + // Strip zero-scalar entries before dispatching to straus_msm. straus_msm has + // non-trivial per-scalar fixed cost (per-window bias decode + bucket scatter), and + // when this function fires from the n_active-based fallback in + // pippenger_round_parallel the input span often contains many zeros (the + // dispatch fired precisely because n_active << n). Compacting once up front saves + // straus_msm one pass over the dead entries on every worker slice. + std::vector compact_scalars; + std::vector compact_points; + compact_scalars.reserve(n); + compact_points.reserve(n); + const ScalarField* src_scalars = scalars_span.span.data(); + const AffineElement* src_points = all_points.data() + scalars_span.start_index; + for (size_t i = 0; i < n; ++i) { + if (!src_scalars[i].is_zero()) { + compact_scalars.push_back(src_scalars[i]); + compact_points.push_back(src_points[i]); + } + } + const size_t n_active = compact_scalars.size(); + if (n_active == 0) { + return Curve::Group::point_at_infinity; + } + + // Cap at `bb::get_num_cpus()` rather than `bb::get_num_cpus()`: + // 1. Want one task per OS worker, not lmul-oversubscribed — straus_msm slices + // have non-trivial fixed cost so dynamic-claim averaging isn't worth the + // extra dispatch tax at the trivial-MSM_fast sizes this function handles. + // 2. `bb::get_num_cpus() <= 1` is the chonk-batch-verifier serial gate; the + // `<= 1` early-return below preserves that contract regardless of pool. + const size_t max_threads = bb::get_num_cpus(); + const size_t num_threads = std::min(n_active, max_threads); + if (num_threads <= 1) { + std::span pts(compact_points.data(), n_active); + std::span scs(compact_scalars.data(), n_active); + return Element::straus_msm(pts, scs); + } + + // Each worker runs `Element::straus_msm` over its slice. Note that straus_msm + // accepts Montgomery-form scalars (it converts internally), so callers must pass + // Montgomery-form scalars on entry to this function. + std::vector partials(num_threads, Curve::Group::point_at_infinity); + bb::parallel_for(num_threads, [&](size_t tid) { + BB_BENCH_NAME("MSM_fast::trivial_msm_threaded/worker"); + const size_t lo = (tid * n_active) / num_threads; + const size_t hi = ((tid + 1) * n_active) / num_threads; + const size_t slice_n = hi - lo; + if (slice_n == 0) { + return; + } + std::span pts(compact_points.data() + lo, slice_n); + std::span scs(compact_scalars.data() + lo, slice_n); + partials[tid] = Element::straus_msm(pts, scs); + }); + Element total_result = partials[0]; + for (size_t t = 1; t < num_threads; ++t) { + total_result += partials[t]; + } + return total_result; +} diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_rewrite_review_map.md b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_rewrite_review_map.md new file mode 100644 index 000000000000..076bc6525ef9 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/pippenger_rewrite_review_map.md @@ -0,0 +1,519 @@ +# Pippenger Rewrite Review Map + +This is a reviewer-oriented map of the current Pippenger rewrite stack. It groups the +optimizations by the inefficiency they are trying to exploit, the heuristic or predicate +that activates them, and the specific risks worth reviewing before treating the rewrite as +production-ready. + +## Current Status + +The stack has been rebased after Bernstein-Yang inversion landed separately in +`merge-train/barretenberg` as PR #23426. Treat Bernstein-Yang as a baseline dependency for +this review, not as part of the remaining Pippenger PR diff. When older measurements below +attribute some speedup to "Bernstein-Yang + staged Pippenger", read that as evidence that +the no-dedup path is fast; the currently reviewable Pippenger delta is the staged MSM, +recoding, batching, GLV/dedup plumbing, arena, and thread-pool changes. + +Current branch status: + +- Variable-window split is removed from the production path. +- The dedup cluster-publication bug that broke `ChonkTests.TestCircuitSizes` is fixed by + publishing only flattened clusters. +- The original Chonk/wasm/no-GLV arena-overflow reproductions have been rerun successfully + on the current branch: transfer_1 native, transfer_0 wasm, transfer_0 native with + `BB_MSM_NO_GLV=1`, and the dedup cap fallback assertion. +- New small and large arena regressions exposed a separate sizing drift: the pre-Phase-1 + arena sizer used the full bit budget (`254` or GLV `128`), while the live pipeline shrinks + to `effective_num_bits` before choosing `window_bits` and `windows_per_batch`. The current + fix sizes GLV MSMs and large non-GLV MSMs against the maximum reachable effective-bit + layout. +- `ecc_tests` builds after the rebase; remaining fixture-size test fallout has been local to + scalar-multiplication tests whose inputs exceeded the reduced shared fixture. +- The all-flow native/wasm matrix below is the current "do not regress" target. + +Remaining high-value review items: + +1. Keep the now-removed variable-window split out unless a new benchmark suite proves a + retuned model wins. +2. Decide whether the broad `parallel_for` rewrite belongs in this PR or should be split. +3. Remove or split unrelated build/debug/benchmark clutter before final review. +4. Review dedup as a targeted Chonk optimization, especially cap fallback tests and hint + discipline, but it is no longer the active `TestCircuitSizes` blocker. +5. Keep arena sizing under targeted regression tests for both ends of the workload spectrum: + large recursion-VK MSMs and small GLV Honk commitments. + +## Fixed Correctness Issue: Dedup Cluster Publication + +Earlier branch state failed `ChonkTests.TestCircuitSizes` with: + +```text +Assertion failed: (cluster_offsets_size == num_clus +Expected: 8193 +``` + +This pointed at the dedup Phase A bookkeeping, not at Chonk itself. + +In `dedup_phase_a_worker_hash`, `clusters_opened` is incremented when a singleton is promoted +inside the hash table, before the cluster is flattened into `cluster_members` and +`cluster_offsets`: + +- promotion: `clusters_opened++` +- flattening may stop early when `cluster_members_size + this_cluster_members > cluster_members_cap` +- the old invariant assumed every opened cluster was flattened: + `cluster_offsets_size == num_clusters + 1` + +So when the member cap was hit, `clusters_opened` could count clusters that were deliberately +left unflattened. The fix is to publish `num_clusters = cluster_offsets_size - 1`, i.e. the +number of flattened clusters that actually have `cluster_offsets` entries. Promoted but +unflattened entries then have no redirect and fall through to normal Pippenger as intended. + +## Optimization Inventory + +| Area | Inefficiency targeted | Activation / heuristic | Main code | Review risks | +| --- | --- | --- | --- | --- | +| Constantine signed-window recoding | Carry propagation and branchy per-window scalar decoding | Always used in round-parallel path; precomputes per-window slice params and selects bottom/localized/boundary paths | `compute_constantine_slice_params*`, `get_constantine_packed_digit`, SIMD x4 helpers | Boundary-bit correctness, top-window masking, endian/aliasing assumptions for `uint32_t` scalar view | +| Window-size selection | Bad `c` gives too many rounds or too many buckets | Native cost model `rounds * (n + 15 * buckets)`; WASM closed form using `target_load` from logical thread count | `choose_window_bits`, `window_bits_tuning_oversub_factor` | Platform calibration, small/large crossover, whether `n` should be post-GLV working scalars or original points | +| GLV split | Halve scalar bit length at cost of doubling point count | `n_input <= 2^13` native, `n_input <= 2^16` WASM, or caller supplies external GLV table | `GLV_SMALL_N_THRESHOLD`, `glv_threshold`, GLV split/double path | Sign convention for phi point, input scalar mutation/restoration asymmetry, memory pressure at crossover | +| Effective bit budget | Avoid windows above the actual largest scalar MSB | After Phase 1, `effective_num_bits` is highest non-empty `msb_hist` bin | Phase 1 `msb_hist` and `effective_num_bits` | Off-by-one in histogram bins; interaction with GLV halves and zero sentinel | +| Trivial MSM fallback | Pippenger scaffolding dominates very sparse or tiny active sets | `pts_per_thread < MIN_PTS_PER_THREAD_FOR_PIPPENGER` (`24`) after zero counting | `trivial_msm_threaded`; constant in header | Correct Montgomery lifecycle before `trivial_msm_threaded`; preserving `PolynomialSpan::start_index` semantics | +| Variable-window split | Mixed scalar sizes waste high-bit windows on small scalars | Removed after traced Chonk runs showed a net regression | deleted `choose_var_window_split` cost model and upper-region dispatch | Keep deleted unless a new benchmark suite proves a retuned split model wins | +| Round-parallel pipeline | Legacy per-thread work balance and repeated bucket reductions | Main path after dispatch: stages 1-7 over window batches sized by arena budget | staged pipeline in `pippenger_round_parallel` | Race-free cursor reuse, per-window capacity, Stage 1 and Stage 4 decode equivalence | +| SIMD digit extraction | Scalar decoding is compute-heavy and non-vectorized | `SIMD_BATCH = 64`; 4-wide `uint32_t` vector helpers selected by per-window path | x4 Constantine digit helpers and Stage 1/4 decode loops | Strict aliasing/layout assumptions, tail handling, all-included mask path | +| In-place histogram/prefix reuse | Avoid separate bucket-total and cursor buffers | `digit_cursors` is counts in Stage 1, per-thread offsets in Stage 2, scatter cursors in Stage 4 | Stage 1-4 `digit_cursors` reuse | Stage ordering, no read-after-overwrite mistakes, capacity and bucket 0 handling | +| Dedup pre-pass | Duplicate scalar values in witness/permutation polynomials cause repeated base-point additions | Explicit `dedup_hint`; long scalars only (`msb >= c_threshold`); caps: 16,384 clusters and 32,768 members | `dedup_phase_a_worker_hash`; hints wired through `CommitmentKey` | Fixed cap-publication bug; still review cap fallback tests, duplicate detection by one-limb fingerprint plus memcmp, and GLV interaction | +| Dedup patching | Keep hot Stage 4 loop dedup-free after first batch | First batch emits ordinary schedule, Phase A populates redirects, `dedup_patch_schedule_window` compacts skips; later batches omit skips up front | `dedup_patch_schedule_window`; Stage 1/4 dedup-known paths | First-batch vs later-batch equivalence, sign preservation on redirects, no stale redirects for capped-out clusters | +| Arena zoning | Reduce allocator churn and WASM fragmentation; bound resident scratch | `compute_arena_bytes_for_msm`, `BATCH_MEM_BUDGET = 32 MiB`, Zone P/W/S layout | arena sizer and Zone P/W/S layout in `pippenger_round_parallel` | Sizer and allocator formulas must stay exactly mirrored; must dominate runtime `effective_num_bits` layouts for GLV and non-GLV; absolute alignment; zero-initialization assumptions | +| Per-worker scratch overlay | Avoid summing all scratch lifetimes into memory budget | Phase A and Stage 6 scratch share Zone W union because they run in separate parallel phases | Phase A and Stage 6 Zone W scratch allocation | No overlapping lifetimes; worker id equals task id assumption; later refactors can violate this silently | +| Recursive affine bucket reduction | Replace projective bucket suffix sums with batched affine additions/doublings | Stage 6b always rebalances bucket ranges; stride is power-of-two; trivial stride <= 2 fallback | `recursive_affine_bucket_reduce_strided`; Stage 6b | Algebraic equivalence of `R`/`L`; batch-affine breakeven fallback; handling sparse windows and empty chunks | +| Dense bucket partials | Avoid sorted scans during cross-thread merge | Stage 6a writes dense per-thread bucket rows; Stage 6b looks up overlapping digit ranges directly | Stage 6a dense partials; Stage 6b merge | Boundary buckets shared by original chunks, overflow buffer sizing, present bitmap reset coverage | +| Batched MSM sharing | Chonk commits many MSMs over the same SRS prefix | Batch driver runs one MSM at a time but shares GLV-doubled SRS buffer and one max-sized arena | `pippenger_round_parallel_batched` | Pointer-range grouping assumes shared contiguous SRS allocation; no cross-MSM scalar scheduling is actually batched | + +## Dedup-Specific Review Checklist + +Dedup is now a targeted secondary optimization rather than the active Chonk blocker. It is +enabled only through hints, and public-transfer traces show the hints are concentrated on +duplicate-heavy Honk wires, `Z_PERM`, and small ECCVM polynomials. Review it as a separate +feature before judging the whole rewrite. + +1. Confirm the hinted call sites are the intended duplicate-heavy polynomials, not blanket + activation. Hints enter via `CommitmentKey::commit`, `batch_commit`, and `BatchBuilder`. +2. Keep cap fallback mechanically correct: flattened cluster count, `cluster_offsets_size`, + published redirects, and `extra_points` must describe the same set of clusters. + `clusters_opened` is diagnostic only and may include clusters that intentionally fall + through to normal Pippenger. +3. Add or strengthen tests where the cap is hit by many small clusters, not only one giant + cluster. The existing cap/carry test describes a mega-cluster shape, which would not catch + opened-but-unflattened many-cluster drift. +4. Check first-batch versus later-batch equivalence: Phase A is based on the first emitted + schedule, and redirects are reused for later windows after schedule patching. +5. Check GLV interaction: after GLV, duplicate scalar halves may not correspond to duplicate + original scalars, and points are `[P, phi(P)]`. Dedup is still algebraically valid if it + aggregates points attached to equal working scalar values, but tests should cover it. + +## Suggested Review Order + +1. Keep correctness green on the current branch, especially Chonk flow tests, wasm prove, + `BB_MSM_NO_GLV=1`, UltraHonk small-range tests, recursion-VK tests, and dedup + cap/fallback tests. +2. Lock down algebraic equivalence tests for the staged pipeline using random scalars, + sparse scalars, duplicate-heavy scalars, and GLV threshold boundaries. +3. Review memory safety after correctness: arena sizing mirrors, effective-bit schedule + sizing, worker scratch lifetimes, overflow bounds, and capacity assumptions. +4. Audit PR scope: split or remove benchmark/debug/build clutter and decide whether the global + thread-pool rewrite belongs with Pippenger. +5. Treat benchmark numbers as meaningful only after the scope and correctness questions above + are settled. Remaining calibrated constants include `GLV_SMALL_N_THRESHOLD`, + `BATCH_CAPACITY`, and the 32 MiB arena budget. + +## Independent Clutter / Split-Out Candidates + +Some changes in the branch are not intrinsically part of the Pippenger arithmetic rewrite. +They either change unrelated runtime behavior or add development scaffolding that makes the +review harder. Treat these as candidates for removal or separate PRs unless a bench proves +they are required for the headline result. + +| File / area | Change | Why it is clutter or too broad | Suggested disposition | +| --- | --- | --- | --- | +| `barretenberg/cpp/CMakePresets.json` | Removes the `WASI_SDK_PREFIX=/opt/wasi-sdk` default from the `wasm-threads` preset | Build-system regression; no MSM performance value | Revert in this PR | +| `barretenberg/cpp/src/barretenberg/bbapi/bbapi_chonk.cpp` | Adds `BB_SKIP_SANITY_VERIFY` | Benchmark/debug convenience that weakens the default prove path's self-check | Remove or keep only in a benchmark harness | +| `barretenberg/cpp/src/barretenberg/sumcheck/sumcheck_round.hpp` | Adds one `BB_BENCH_NAME` inside sumcheck | Profiling annotation outside MSM/commitment code | Move to profiling-only cleanup if desired | +| `barretenberg/cpp/src/barretenberg/vm2/constraining/prover.cpp` | Removes `AVM_MAX_MSM_BATCH_SIZE` batching control | Changes AVM prover behavior as a side effect of commitment batching | Revert unless the new commitment API requires it and AVM is measured | +| `barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/*` | Deletes `thread_scaling`, adds `small_msm_matrix`, rewrites `pippenger.bench` | Useful development tooling, but it expands review surface | Split into benchmark/support PR or keep only minimal reproducible benches | + +The global `parallel_for` rewrite in `barretenberg/cpp/src/barretenberg/common/thread.cpp` is +not simple clutter, but it is too broad for a Pippenger PR unless it is necessary for the +measured win. It changes scheduling for every `parallel_for` caller in barretenberg: sumcheck, +translator, VM2, ECCVM, and non-MSM prover code can all regress independently. Test this by +reverting/isolating the thread-pool rewrite and rerunning the native public-transfer bench. If +the MSM rewrite keeps most of the win, split the thread-pool change out. + +Similarly, `barretenberg/cpp/cmake/threading.cmake` adding `-msimd128` may support the wasm +SIMD copy path, but it changes wasm runtime requirements. Keep it only with a separate wasm +compatibility justification and benchmarks; otherwise remove it from the native-focused +Pippenger rewrite. + +Dedup hint plumbing in Oink, ECCVM, and Translator is not independent clutter, but it is +speculative. Keep only hints whose labels show meaningful `duplicate_excess / size` under +`BB_COMMITMENT_DEDUP_TRACE=1`; remove blanket hints that do not pay. + +## Instrumentation + +The branch has local MSM tracing and ablation switches in `scalar_multiplication.cpp`: + +- `BB_MSM_TRACE=1` emits one `BB_MSM_TRACE {...}` line per MSM. +- `BB_COMMITMENT_DEDUP_TRACE=1` emits one `BB_COMMITMENT_DEDUP_TRACE {...}` line per + commitment candidate, including Chonk polynomial labels when the commitment goes through a + batch. +- `BB_IPA_TRACE=1` emits the IPA opening size ladder: one start line and one line per IPA + reduction round. +- `BB_MSM_NO_GLV=1` disables inline and shared batched GLV. +- `BB_MSM_NO_DEDUP=1` ignores dedup hints and sizes the arena accordingly. + +Useful trace fields: + +- `n_input`, `n_working`, `n_active` +- `use_glv`, `external_glv` +- `dedup_hint`, `dedup_active`, `dedup_clusters`, `dedup_ms` +- `effective_num_bits`, `window_bits`, `windows_per_batch` +- `phase1_ms`, `pipeline_ms`, `total_ms` + +For the `ecdsar1+transfer_0_recursions+sponsored_fpc` flow, compare the full branch against: + +```bash +BB_MSM_TRACE=1 +BB_MSM_TRACE=1 BB_MSM_NO_GLV=1 +BB_MSM_TRACE=1 BB_MSM_NO_DEDUP=1 +BB_MSM_TRACE=1 BB_MSM_NO_GLV=1 BB_MSM_NO_DEDUP=1 +``` + +The fastest way to answer the current attribution question is to group trace lines by +`curve`, `n_input`, `use_glv`, and `dedup_clusters`. If the large `2^19` BN254 MSMs +still improve with `use_glv=false` and `dedup_clusters=0`, the staged Pippenger path is +likely a real contributor. If the wins concentrate in `n_input <= 8192` or duplicate-heavy +calls, the headline should be narrowed to GLV, fallback, and dedup-heavy workloads. + +For dedup attribution by Chonk polynomial, run the same flow with: + +```bash +BB_MSM_TRACE=1 BB_COMMITMENT_DEDUP_TRACE=1 BB_IPA_TRACE=1 +``` + +`BB_COMMITMENT_DEDUP_TRACE` reports exact duplicate density only for dedup-hinted +polynomials, so it should stay cheap enough to use on full Chonk flows while answering which +labels are actually responsible for the dedup win. Group by `label`, `size`, and +`duplicate_excess`; the labels with the largest `duplicate_excess / size` should line up with +the MSM trace lines that have large `dedup_clusters`. + +`BB_IPA_TRACE` has no dedup stats because IPA scalars are challenge-derived and call +`pippenger_unsafe` without a duplicate hint. Its purpose is to correlate the Grumpkin IPA +round ladder with `BB_MSM_TRACE` and `batch_mul_with_endomorphism` timings, especially the +`2^15 -> ... -> 1` sequence in ECCVM IPA. + +## Empirical Results + +### `ecdsar1+transfer_0_recursions+sponsored_fpc`, native (clang20-no-avm, 16 threads) + +Historical measurement on branch `lde/zacs-pippenger` before the Bernstein-Yang rebase, +compared with baseline `merge-train/barretenberg` (`4da6ab07f2c`), EC2 single run. The flow +matrix below includes later reruns after instrumentation, variable-split removal, and the +dedup cap publication fix. Because Bernstein-Yang has since landed separately, use these +numbers for workload attribution, not as a clean PR-vs-current-base diff. + +Native Chonk flow matrix: + +| Flow | Circuits | Baseline `ChonkAPI::prove` | Branch `ChonkAPI::prove` | Status | +| --- | --- | --- | --- | --- | +| `ecdsar1+transfer_0_recursions+sponsored_fpc` | 9 | 4.48 s | 3.43 s median | -23.4% | +| `ecdsar1+transfer_1_recursions+private_fpc` | 17 | 7.75 s | 6.10 s | -21.3% | + +| Stage | Baseline | Branch | Delta | +| --- | --- | --- | --- | +| `ChonkAPI::prove` (total) | 4.48 s | 3.46 s | -22.8% | +| `OinkProver::prove` (8 calls, avg/iter) | 891.5 ms (111.4 ms) | 568.6 ms (71.1 ms) | -36.2% | +| `Goblin::prove_eccvm` | 829.5 ms | 574.2 ms | -30.8% | +| `IPA::compute_opening_proof` | 292.1 ms | 170.0 ms | -41.8% | +| `MSM::batch_multi_scalar_mul` (oink, 38 calls) | 1.06 s (27.9 ms) | 659 ms (17.3 ms) | -37.8% | +| `CommitmentKey::commit` (oink wires, 53 calls) | 263.4 ms (4.97 ms) | 151.3 ms (2.85 ms) | -42.6% | +| `CommitmentKey::commit` (z_perm, 5 calls) | 189.2 ms (37.8 ms) | 133.7 ms (26.7 ms) | -29.4% | +| `batch_mul_with_endomorphism` (IPA, 15 calls) | 180.7 ms (12.05 ms) | 108.9 ms (7.26 ms) | -39.7% | +| `ChonkLoad` (msgpack decode, no MSM) | 100.1 ms | 106.8 ms | +6.7% (noise) | + +`IPA::compute_opening_proof` runs on random IPA challenge scalars with no `dedup_hint`, +so its -42% historical delta is attributable to the no-dedup path: round-parallel pipeline, +Bernstein-Yang inversion, and batch-affine bucket accumulation. Since Bernstein-Yang is now +in the base branch, current review should focus on the remaining Pippenger-side pieces of +that no-dedup path. The per-call oink-commit delta (-43%) is roughly the same magnitude, +implying dedup adds at most a few percent over the no-dedup baseline on this workload, not +the 20-30% earlier guess. + +### Native ablations, same flow + +All runs are single-run EC2 native (`clang20-no-avm`, 16 threads), comparing against the +uninstrumented branch wallclock of 3.46 s. The first ablation set was collected before the +dedup publication fix; the `BB_MSM_NO_GLV=1` abort is historical and has since been rerun +successfully. + +| Run | `ChonkAPI::prove` | Delta vs branch | Implication | +| --- | --- | --- | --- | +| Branch, uninstrumented | 3.46 s | baseline | Full rewrite result | +| `BB_MSM_NO_DEDUP=1` | 3.57 s | +0.11 s (+3.2%) | Dedup saves about 110 ms | +| `BB_MSM_NO_GLV=1 BB_MSM_NO_DEDUP=1` | 3.61 s | +0.15 s (+4.3%) | GLV adds about 40 ms on top of dedup | +| `BB_MSM_NO_GLV=1` | historical abort | - | Historical arena/cap symptom; current branch proves this path | + +Attribution against the full baseline-to-branch delta (`4.48 s -> 3.46 s`, 1.02 s saved): + +| Source | Approx saved | Share of baseline wallclock | Share of branch win | +| --- | --- | --- | --- | +| Dedup | 110 ms | ~2.5% | ~12% | +| GLV | 40 ms | ~1% | ~3% | +| Non-dedup, non-GLV rewrite | 870 ms | ~19.5% | ~85% | + +This materially changes the review posture: the rewrite's native win on this flow does not +stand or fall on dedup or GLV. The actual headline is the no-dedup, non-GLV path: staged +affine bucket reduction, batch-affine arithmetic, round-parallel scaffolding, Constantine +recoding, plus Bernstein-Yang in the historical baseline comparison. Since Bernstein-Yang is +now in merge-train, the remaining review should focus on the staged Pippenger machinery. The +no-dedup IPA evidence above is still useful: IPA drops 122 ms historically without duplicate +stripping. + +The old `BB_MSM_NO_GLV=1` abort hit the same `aligned_local + bytes <= bound_bytes` arena +assertion class as the wasm crash, but it no longer reproduces on the current branch. Treat +it as evidence for the fixed dedup cap / removed split-path sizing work, not as an open +arena blocker. + +### Triple-traced public-transfer ablation + +Same `ecdsar1+transfer_0_recursions+sponsored_fpc` native flow with +`BB_MSM_TRACE=1 BB_COMMITMENT_DEDUP_TRACE=1 BB_IPA_TRACE=1`. The extra per-coefficient +duplicate sort raises logging overhead to about 5%, so these deltas are relative to the +traced branch baseline of 3.66 s, not the uninstrumented 3.46 s. + +| Run | `ChonkAPI::prove` | Delta vs traced branch | Implication | +| --- | --- | --- | --- | +| Traced branch | 3.66 s | baseline | Full branch with tracing | +| `BB_MSM_NO_VAR_SPLIT=1` | 3.64 s | -20 ms | Variable split was a small wallclock regression before removal | +| `BB_MSM_NO_DEDUP=1` | 3.75 s | +90 ms | Dedup saves about 90 ms under tracing | + +Dedup payload by hinted label, sorted by `zero_count + duplicate_excess` ("bucket adds +avoided"): + +| Label | Calls | Total n | Zeros | Real dup excess | Avoided | Avoided / n | +| --- | --- | --- | --- | --- | --- | --- | +| `W_4` | 9 | 444,229 | 188,073 | 87,968 | 276,041 | 62.1% | +| `W_O` | 9 | 444,229 | 196,970 | 75,721 | 272,691 | 61.4% | +| `W_R` | 9 | 444,229 | 141,131 | 131,493 | 272,624 | 61.4% | +| `W_L` | 9 | 444,229 | 111,274 | 159,766 | 271,040 | 61.0% | +| `` commit path | 2 | 163,838 | 1 | 87,969 | 87,970 | 53.7% | +| `Z_PERM` | 9 | 444,229 | 1 | 69,576 | 69,577 | 15.7% | +| ECCVM `MSM_X*` / `MSM_Y*` | 1 each | 4,953 each | ~1,100 | ~3,000 | ~4,000 | 67-84% | +| ECCVM `PRECOMPUTE_DX/DY` | 1 each | 4,952 each | 1,085 | 3,494 | 4,579 | 92% | +| ECCVM `TRANSCRIPT_*` accumulators | 1 each | 4,952 each | 4,147-4,478 | 142-763 | 4,610-4,910 | 93-99% | + +The wires are the dominant target: `W_L/R/O/4` account for about 1.09M of 1.31M avoided +bucket additions across the prove, roughly 83% of the dedup payload. `Z_PERM` is the smallest +hinted Honk polynomial by density, but it has essentially no zeros; its 15.7% comes from real +constant-product stretches, not padding. The ECCVM hints are tiny in aggregate but high +density; transcript accumulator hints are mostly a single large zero cluster, so a simpler +zero-strip path may be cheaper there than the full dedup state machine. + +Structural zeros versus real repeats in the main Honk polynomials: + +| Label | Zero share | Real-dup share | +| --- | --- | --- | +| `W_L` | 25% | 36% | +| `W_R` | 32% | 30% | +| `W_O` | 44% | 17% | +| `W_4` | 42% | 20% | +| `Z_PERM` | 0% | 16% | + +This means dedup is not just an expensive zero-stripper. Wires are a mix of sparse padding and +genuine value reuse; `W_L` and `W_R` have more real duplicates than zeros, and `Z_PERM` is +purely real repeats. + +Order-joined MSM timing reproduces the dedup wallclock delta at the MSM level: + +| `n_input` bucket | Calls | Dedup-active calls | `NO_DEDUP - baseline` total_ms | Avg `dedup_clusters` | +| --- | --- | --- | --- | --- | +| 256-1k | 14 | 0 | -1 ms | - | +| 1k-4k | 27 | 0 | -7 ms | - | +| 4k-16k | 85 | 21 | +19 ms | 984 | +| 16k-64k | 37 | 21 | +29 ms | 1,931 | +| 64k-128k | 35 | 21 | +55 ms | 5,111 | +| 128k+ | 3 | 0 | -8 ms | - | +| Total heavy MSMs | 201 | 63 | +87 ms | - | + +About 63% of the dedup gain is in the 64k-128k bucket, exactly the Honk wire/z_perm commits. +The 4k-16k bucket contributes a smaller but real payoff from the ECCVM polynomials. + +Variable-window split looks like an anti-optimization on this Chonk flow: + +| Bucket | Calls | `split=true` in baseline | `NO_VAR_SPLIT - baseline` total_ms | +| --- | --- | --- | --- | +| 16k-64k | 37 | 14 | -17 ms | +| 64k-128k | 35 | 16 | -16 ms | +| Others | 129 | 1 | -11 ms | +| Total heavy MSMs | 201 | 31 | -44 ms | + +The predictor fires 31 times and loses about 1.4 ms per split decision. The current rule +accepts a split when predicted cost is at most 85% of unsplit; on this workload the predictor +is either overestimating split savings or the unsplit path has become fast enough that this +margin was too generous. The variable split path has since been removed from the branch. + +IPA structure from the same trace: one Grumpkin IPA opening uses `poly_length=32768`, 15 +rounds, 30 Pippenger calls, and 15 `batch_mul_with_endomorphism` calls. The round ladder is +`16384 -> ... -> 1`. None of these calls has a dedup hint, so the IPA part of the +historical speedup is entirely non-dedup: Bernstein-Yang inversion plus staged affine bucket +reduction, round-parallel pipeline, and batch-affine arithmetic. After the BY rebase, only +the staged Pippenger pieces remain part of this PR's diff. + +Updated attribution for this flow: + +| Component | Approx effect | Review implication | +| --- | --- | --- | +| Non-dedup, non-GLV, non-var-split Pippenger path | ~960 ms historical saved including BY | Main headline; BY is now baseline, so focus review on remaining staged MSM machinery | +| Dedup | ~90 ms saved | Real and well targeted; mostly Honk wires | +| GLV | ~40 ms saved | Small contributor from prior ablation | +| Variable-window split | ~44 ms regression | Removed; keep it out unless a new benchmark proves otherwise | + +Concrete actions from this trace: + +1. Keep `choose_var_window_split` removed unless a new benchmark suite justifies rebuilding it. +2. Keep dedup as a targeted Chonk optimization; the cap-publication bug is fixed, but tests + should still cover cap fallback shapes. +3. Consider replacing the ECCVM transcript accumulator dedup case with a cheaper zero-heavy + path if it remains measurable after the correctness work. + +### `ecdsar1+transfer_1_recursions+private_fpc`, native + +Baseline `merge-train/barretenberg` (`4da6ab07f2c`) proves this flow in 7.75 s. The current +branch, after variable-split removal and the dedup cap publication fix, proves it in 6.10 s +single-run: a 1.65 s / 21.3% speedup. + +An earlier branch state aborted before timing could be collected: + +```text +aligned_local + bytes <= bound_bytes +1.70 MB needed vs 1.21 MB cap +``` + +This flow is roughly "more of the same" compared with transfer_0: 17 circuits vs 9 circuits, +and baseline wallclock scales from 4.48 s to 7.75 s. Per-circuit baseline time is slightly +lower on transfer_1 (456 ms vs 498 ms), so the private-recursive flow is not a qualitatively +different workload. The current branch now proves this larger real Chonk workload, so the +historical native speedup signal holds beyond the shorter public-transfer flow. + +Baseline slices: + +| Stage | Baseline time | Calls x avg | +| --- | --- | --- | +| `Chonk::accumulate_and_fold` | 4.12 s | 16 x 257.7 ms | +| Dominant Mega `OinkProver::prove` | 2.14 s | 16 x 133.5 ms | +| `commit_to_wires` | 855.8 ms | 17 x 50.3 ms | +| `commit_to_z_perm` | 782.4 ms | 17 x 46.0 ms | +| `commit_to_lookup_counts_and_w4` | 387.5 ms | 17 x 22.8 ms | +| `commit_to_logderiv_inverses` | 225.2 ms | 17 x 13.2 ms | +| `HypernovaFoldingProver::sumcheck` | 894.3 ms | 16 x 55.9 ms | +| `Goblin::prove_eccvm` | 995.0 ms | - | +| `IPA::compute_opening_proof` | 276.3 ms | - | +| `BatchedHonkTranslatorProver::prove` | 944.5 ms | - | +| `MSM::batch_multi_scalar_mul` (top context) | 2.25 s | 70 x 32.1 ms | + +The prior abort is now best treated as a removed-path/cap-publication correctness symptom, +not proof that the whole unsplit arena model is broken. Variable-split removal deleted the +split-specific sizing branch, and the dedup cap fix prevents promoted-but-unflattened +clusters from being published. + +### `BB_MSM_TRACE=1` aggregates, same flow + +525 MSM calls captured. Logging overhead 3.46 -> 3.52 s (~2%). + +| Path | Calls | Total | Avg | +| --- | --- | --- | --- | +| `pippenger_round_parallel` (heavy) | 201 | 1186 ms | 5.90 ms | +| `trivial_pre` / `trivial_post_profile` | 312 | ~0 ms | 0 | +| `empty` | 12 | 0 ms | 0 | + +Heavy-path breakdown by `n_input`: + +| `n_input` | Calls | Total | Avg | Dedup-active calls | Avg `dedup_clusters` | +| --- | --- | --- | --- | --- | --- | +| 256-1k | 14 | 9 ms | 0.64 ms | 0 | - | +| 1k-4k | 27 | 29 ms | 1.07 ms | 0 | - | +| 4k-16k | 85 | 90 ms | 1.06 ms | 21 | 985 | +| 16k-64k | 37 | 336 ms | 9.08 ms | 21 | 1930 | +| **64k-128k** | **35** | **543 ms** | **15.51 ms** | **21** | **5111** | +| 128k+ | 3 | 179 ms | 59.67 ms | 0 | - | + +Observations: + +- The 64k-128k bucket dominates wallclock (543 ms = 15% of total prove). 5111 clusters on + 88-128k inputs corresponds to ~5-7% cluster density - matches the "few huge clusters" + shape from structural-padding zeros and constant z_perm regions. +- Dedup fires on 63 of 201 heavy calls, distributed as exactly 21 in each of the 4k-16k, + 16k-64k, 64k-128k buckets. That is 7 dup-hinted commits per prover stage x 3 prover + stages, i.e. wires + z_perm getting consistent dedup activation. No + `dedup_hint=true,dedup_active=false` cases were observed on this flow. +- 128k+ MSMs (ECCVM/IPA SRS commits) correctly run without dedup; their scalars are + challenges and zero-padding does not appear. +- Trace currently reports `dedup_clusters` but not `dedup_members_flattened` / + `dedup_members_dropped`. Adding those would make cap-fallback behavior directly observable + rather than relying only on code reading and targeted tests. + +### Arena-overflow reproductions and current diagnosis + +Earlier branch states had several `aligned_local + bytes <= bound_bytes` or dedup-layout +assertions. The first group is closed, but later CI found a second arena-sizing bug that is +independent of variable split and dedup publication. + +| Reproduction | Symptom | Current branch outcome | +| --- | --- | --- | +| transfer_0 native + `BB_MSM_NO_GLV=1` | Arena assertion during ablation | Proves in 3.47 s | +| transfer_0 wasm | ~8% arena overflow, 674 KB needed vs 624 KB cap | Proves in 8.71 s | +| transfer_1 native, no flags | ~40% arena overflow, 1.70 MB needed vs 1.21 MB cap | Proves in 6.16 s / 6.10 s single-runs | +| dedup cap fallback | `cluster_offsets_size == num_clusters + 1` drift | Fixed by publishing only flattened clusters | +| `HonkRecursionConstraintTestWithoutPredicate/2.GenerateVKFromConstraints` | large BN254 non-GLV arena assertion, schedule allocation `26,454,272` bytes vs `25,505,329` Zone S cap | Fixed by sizing large non-GLV MSMs against max reachable `effective_num_bits` layout | +| `RangeTests/0.LimbedRangeConstraint133Bits` | small BN254 GLV arena assertion, `507,712` bytes vs `488,933` cap | Fixed by applying the same effective-bit layout sizing to GLV MSMs | + +Current diagnosis: there are at least three distinct fixed correctness issues in the arena / +dedup area, not one generic failure mode. Variable-split removal closed the old split-path +sizing branch, the dedup publication fix closed promoted-but-unflattened clusters, and the +latest arena fix makes the pre-Phase-1 sizer dominate the runtime `effective_num_bits` +schedule choice. Arena zoning remains a top review area because every future Zone P/W/S +allocation change must update both the sizer and the typed allocator layout. + +### Two preset/cmake regressions noted while reproducing + +Outside MSM code itself, the branch silently changed wasm/cmake behavior: + +- `CMakePresets.json` removed the `WASI_SDK_PREFIX=/opt/wasi-sdk` default from the + `wasm-threads` preset environment block. Builds now fail with + `#include ` not found unless `WASI_SDK_PREFIX` is exported externally. +- `cmake/threading.cmake` added `-msimd128` for WASM multithreaded builds. Hot loops + (Phase 5a sched -> pts copy) depend on `v128.load/store` at runtime, so any older + V8/wasmtime would now fail differently. The bench machine runs wasmtime 43, which is + fine; production wasm consumers should be checked. + +### Full bench matrix: all 11 IVC flows x {native, wasm} x {baseline, branch} + +Single-run, EC2 16 threads. Native: `clang20-no-avm`. WASM: `wasm-threads` + wasmtime 43 +with `-W threads=y -W shared-memory=y -S threads=y`. Branch state for these numbers has +variable-split removed and the dedup cap publication fix. Baseline is historical +`merge-train/barretenberg` (`4da6ab07f2c`), so after the Bernstein-Yang rebase the matrix is +best used as the workload coverage and "do not regress" target rather than a clean diff +against today's merge-train. All numbers are `ChonkAPI::prove` wallclock in seconds. + +| Flow | Base nat | Branch nat | Native delta | Base wasm | Branch wasm | WASM delta | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `deploy_ecdsar1+sponsored_fpc` | 5.47 | 4.27 | -21.9% | 14.83 | 10.88 | -26.6% | +| `deploy_schnorr+sponsored_fpc` | 5.19 | 3.99 | -23.1% | 14.04 | 10.15 | -27.7% | +| `ecdsar1+amm_add_liquidity_1_recursions+sponsored_fpc` | 8.69 | 6.97 | -19.8% | 23.64 | 18.11 | -23.4% | +| `ecdsar1+deploy_tokenContract_with_registration+sponsored_fpc` | 5.82 | 4.58 | -21.3% | 15.66 | 11.74 | -25.0% | +| **`ecdsar1+storage_proof_7_layers+sponsored_fpc`** | **13.60** | **11.96** | **-12.1%** | **43.28** | **37.11** | **-14.3%** | +| `ecdsar1+token_bridge_claim_private+sponsored_fpc` | 5.19 | 4.07 | -21.6% | 14.00 | 10.41 | -25.6% | +| `ecdsar1+transfer_0_recursions+private_fpc` | 6.98 | 5.54 | -20.6% | 19.02 | 14.26 | -25.0% | +| `ecdsar1+transfer_0_recursions+sponsored_fpc` | 4.48 | 3.46 | -22.8% | 11.92 | 8.71 | -26.9% | +| `ecdsar1+transfer_1_recursions+private_fpc` | 7.74 | 6.16 | -20.4% | 20.99 | 15.84 | -24.5% | +| `ecdsar1+transfer_1_recursions+sponsored_fpc` | 5.10 | 3.96 | -22.4% | 13.67 | 10.09 | -26.2% | +| `schnorr+deploy_tokenContract_with_registration+sponsored_fpc` | 5.55 | 4.32 | -22.2% | 14.99 | 11.08 | -26.1% | +| **Sum** | **73.81** | **59.28** | **-19.7%** | **206.04** | **158.38** | **-23.1%** | + diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.cpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.cpp index c5f50fdff5b2..792b7e6dc90c 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.cpp +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.cpp @@ -20,7 +20,7 @@ #include "barretenberg/common/mem.hpp" #include "barretenberg/numeric/bitop/get_msb.hpp" -namespace bb::scalar_multiplication { +namespace bb::scalar_multiplication::legacy { // Naive double-and-add fallback for small inputs (< PIPPENGER_THRESHOLD points). template typename Curve::Element small_mul(const typename MSM::MSMData& msm_data) noexcept @@ -626,7 +626,97 @@ template curve::BN254::Element pippenger(PolynomialSpan(PolynomialSpan scalars, std::span points); -} // namespace bb::scalar_multiplication +} // namespace bb::scalar_multiplication::legacy + +template class bb::scalar_multiplication::legacy::MSM; +template class bb::scalar_multiplication::legacy::MSM; + +// =================================================================================== +// Public MSM facade implementation (see scalar_multiplication.hpp). Routes to the +// `_fast` rewrite by default, or `legacy::` when BB_MSM_LEGACY is set. +// =================================================================================== +namespace bb::scalar_multiplication { + +bool use_legacy_msm() noexcept +{ + static const bool legacy_selected = std::getenv("BB_MSM_LEGACY") != nullptr; + return legacy_selected; +} + +template +typename Curve::Element pippenger(PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept +{ + if (use_legacy_msm()) { + return legacy::pippenger(scalars, points, handle_edge_cases); + } + return pippenger_fast(scalars, points, handle_edge_cases, dedup_hint); +} + +template +typename Curve::Element pippenger_unsafe(PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept +{ + if (use_legacy_msm()) { + return legacy::pippenger_unsafe(scalars, points); + } + return pippenger_unsafe_fast(scalars, points, dedup_hint); +} -template class bb::scalar_multiplication::MSM; -template class bb::scalar_multiplication::MSM; +template +typename Curve::AffineElement MSM::msm(std::span points, + PolynomialSpan scalars, + bool handle_edge_cases, + bool dedup_hint) noexcept +{ + return AffineElement(pippenger(scalars, points, handle_edge_cases, dedup_hint)); +} + +template +std::vector MSM::batch_multi_scalar_mul( + std::span points, + std::span> scalars, + bool handle_edge_cases, + std::span dedup_hints) noexcept +{ + if (use_legacy_msm()) { + // Adapt the rewrite's (single shared points + per-MSM PolynomialSpan) shape to the + // legacy per-MSM (points span, scalar span) shape. dedup_hints are dropped. + const size_t k = scalars.size(); + std::vector> legacy_points; + std::vector> legacy_scalars; + legacy_points.reserve(k); + legacy_scalars.reserve(k); + for (size_t i = 0; i < k; ++i) { + const size_t start_i = std::min(scalars[i].start_index, points.size()); + const size_t n = std::min(scalars[i].span.size(), points.size() - start_i); + legacy_points.push_back(points.subspan(start_i, n)); + legacy_scalars.push_back(scalars[i].span); + } + return legacy::MSM::batch_multi_scalar_mul(legacy_points, legacy_scalars, handle_edge_cases); + } + return MSM_fast::batch_multi_scalar_mul(points, scalars, handle_edge_cases, dedup_hints); +} + +template curve::BN254::Element pippenger(PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept; +template curve::Grumpkin::Element pippenger(PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept; +template curve::BN254::Element pippenger_unsafe(PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept; +template curve::Grumpkin::Element pippenger_unsafe( + PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept; +template class MSM; +template class MSM; + +} // namespace bb::scalar_multiplication diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp index c03f679b61bc..04a6fa7b2142 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp @@ -5,6 +5,14 @@ // ===================== #pragma once +// This header hosts TWO implementations behind one facade: +// * `bb::scalar_multiplication::legacy::*` — the pre-rewrite Pippenger MSM, bodies +// byte-identical to merge-train (only wrapped in the `legacy` sub-namespace). +// * the round-parallel rewrite in scalar_multiplication_fast.hpp (`*_fast`, `MSM_fast`). +// The public facade (`pippenger`, `pippenger_unsafe`, `MSM`) at the bottom dispatches to +// the rewrite by default, or to `legacy::` when `use_legacy_msm()` (env BB_MSM_LEGACY). +// Remove the legacy half + the facade dispatch once the rewrite has soaked. +#include "./scalar_multiplication_fast.hpp" #include "barretenberg/ecc/groups/precomputed_generators_bn254_impl.hpp" #include "barretenberg/ecc/groups/precomputed_generators_grumpkin_impl.hpp" @@ -14,7 +22,7 @@ #include "./bitvector.hpp" #include "./process_buckets.hpp" -namespace bb::scalar_multiplication { +namespace bb::scalar_multiplication::legacy { template class MSM { public: @@ -394,4 +402,65 @@ typename Curve::Element pippenger_unsafe(PolynomialSpan; extern template class MSM; +} // namespace bb::scalar_multiplication::legacy + +// =================================================================================== +// Public MSM facade — the surface every caller uses. Dispatches to the `_fast` rewrite +// by default, or `legacy::` when use_legacy_msm() (env BB_MSM_LEGACY, read once). +// Signatures match the rewrite; the legacy branch adapts (legacy has no dedup pre-pass, +// and its batch entry takes per-MSM point spans). +// =================================================================================== +namespace bb::scalar_multiplication { + +[[nodiscard]] bool use_legacy_msm() noexcept; + +template +typename Curve::Element pippenger(PolynomialSpan scalars, + std::span points, + bool handle_edge_cases = true, + bool dedup_hint = false) noexcept; + +template +typename Curve::Element pippenger_unsafe(PolynomialSpan scalars, + std::span points, + bool dedup_hint = false) noexcept; + +extern template curve::BN254::Element pippenger(PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept; +extern template curve::Grumpkin::Element pippenger( + PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept; +extern template curve::BN254::Element pippenger_unsafe( + PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept; +extern template curve::Grumpkin::Element pippenger_unsafe( + PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept; + +template class MSM { + public: + using Element = typename Curve::Element; + using ScalarField = typename Curve::ScalarField; + using AffineElement = typename Curve::AffineElement; + + static AffineElement msm(std::span points, + PolynomialSpan scalars, + bool handle_edge_cases = false, + bool dedup_hint = false) noexcept; + + static std::vector batch_multi_scalar_mul(std::span points, + std::span> scalars, + bool handle_edge_cases = true, + std::span dedup_hints = {}) noexcept; +}; + +extern template class MSM; +extern template class MSM; + } // namespace bb::scalar_multiplication diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.test.cpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.test.cpp index e8e8aaabf393..709aae88eefc 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.test.cpp @@ -4,9 +4,12 @@ #include "barretenberg/ecc/curves/bn254/bn254.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "barretenberg/ecc/curves/types.hpp" +#include "barretenberg/ecc/scalar_multiplication/pippenger_arena_layout.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "barretenberg/polynomials/polynomial.hpp" #include "barretenberg/srs/factories/mem_bn254_crs_factory.hpp" +#include +#include #include #include @@ -14,6 +17,278 @@ using namespace bb; namespace { auto& engine = numeric::get_randomness(); + +// Walks the actual Zone P / Zone W / Zone S allocator for a representative BN254 +// MSM shape and asserts the result fits in `compute_arena_bytes_for_msm`'s promise. +// Mirrors the live allocator inside `pippenger_round_parallel` exactly; the only +// historical drift bugs (cluster_offsets miscount, wasm aligned_local overflow, +// NO_GLV abort, t1 abort) all came from this walk falling out of sync. +bool pippenger_bn254_arena_layout_fits_for_test(size_t n_input, + bool external_glv_provided = false, + bool dedup_active = false, + size_t effective_num_bits_for_test = 0) noexcept +{ + using Curve = curve::BN254; + using ScalarField = typename Curve::ScalarField; + using Element = typename Curve::Element; + using AffineElement = typename Curve::AffineElement; + namespace rpd = scalar_multiplication::round_parallel_detail; + + constexpr size_t FULL_NUM_BITS = ScalarField::modulus.get_msb() + 1; + if (n_input < 4) { + return true; + } + + const bool use_glv = external_glv_provided || (n_input <= rpd::GLV_SMALL_N_THRESHOLD); + const bool inline_glv_double = use_glv && !external_glv_provided; + const size_t n = use_glv ? 2 * n_input : n_input; + const size_t NUM_BITS = use_glv ? size_t{ 128 } : FULL_NUM_BITS; + const size_t arena_capacity = + scalar_multiplication::compute_arena_bytes_for_msm(n_input, external_glv_provided, dedup_active); + if (arena_capacity == 0) { + return true; + } + + const size_t actual_num_bits = (effective_num_bits_for_test == 0 || effective_num_bits_for_test > NUM_BITS) + ? NUM_BITS + : effective_num_bits_for_test; + const size_t num_logical_threads_for_c = + bb::get_num_cpus() * scalar_multiplication::window_bits_tuning_oversub_factor(n_input); + const size_t window_bits = rpd::choose_window_bits(n, actual_num_bits, n_input, num_logical_threads_for_c); + const auto sched = rpd::build_var_window_schedule(actual_num_bits, window_bits); + const size_t num_buckets = (size_t{ 1 } << (window_bits - 1)) + 1; + + using rpd::BATCH_CAPACITY; + constexpr size_t MIN_BATCH_CAPACITY = 32; + constexpr size_t BATCH_MEM_BUDGET = 32ULL * 1024ULL * 1024ULL; + constexpr size_t SUBCHUNK_ENTRIES_CAP = 2048; + + const size_t desired_threads = std::max(1, bb::get_num_cpus()); + const size_t max_threads_for_min_batch = std::max(1, n / MIN_BATCH_CAPACITY); + const size_t num_threads = std::min(desired_threads, max_threads_for_min_batch); + const size_t profile_threads = std::max(1, bb::get_num_cpus()); + const size_t worker_total = num_threads; + + size_t B_eff = num_buckets; + for (size_t w = 0; w < sched.num_windows; ++w) { + B_eff = std::max(B_eff, static_cast(sched.num_buckets[w])); + } + const size_t dense_stride_est = + std::max(2, std::bit_ceil((B_eff > 1) ? ((B_eff - 1 + num_threads - 1) / num_threads) : size_t{ 1 })); + const size_t bucket_partials_per_window_max = (B_eff > 0) ? (B_eff - 1 + num_threads - 1) : 0; + const size_t hist_h_bytes_pw_shared = (size_t{ 4 } * num_threads * B_eff); + const size_t hist_o_bytes_pw_shared = + (sizeof(rpd::ChunkOutput) * num_threads) + (size_t{ 96 } * num_threads); + const size_t hist_slot_bytes_pw_shared = std::max(hist_h_bytes_pw_shared, hist_o_bytes_pw_shared); + const size_t dense_slot_bytes_pw_shared = (size_t{ 65 } * bucket_partials_per_window_max); + const size_t per_window_bytes_shared = + hist_slot_bytes_pw_shared + dense_slot_bytes_pw_shared + (size_t{ 8 } * (B_eff + 1)) + + (size_t{ 8 } * (num_threads + 1)) + (size_t{ 8 } * (num_threads + 1)) + (size_t{ 8 } * num_threads) + + (size_t{ 8 } * num_threads) + (size_t{ 8 } * num_threads) + (size_t{ 16 } * worker_total) + + (size_t{ 8 } * num_threads) + (size_t{ 87 } * worker_total * dense_stride_est); + const size_t capacity_lo = n; + const size_t per_window_bytes_lo = (size_t{ 4 } * capacity_lo) + per_window_bytes_shared; + + const size_t global_max_chunk_len = (n + num_threads - 1) / num_threads; + const size_t global_max_overflow_per_window = + (global_max_chunk_len + SUBCHUNK_ENTRIES_CAP - 1) / SUBCHUNK_ENTRIES_CAP; + const size_t chunk_capacity = std::max(SUBCHUNK_ENTRIES_CAP, 2 * global_max_overflow_per_window); + + const size_t phase_a_cluster_members_cap = std::min(rpd::DEDUP_MAX_MEMBERS, n); + const size_t phase_a_cluster_offsets_cap = (rpd::DEDUP_MAX_CLUSTERS / num_threads) + 2; + + const size_t phase_one_prologue_bytes = n + (use_glv ? size_t{ 32 } * n : size_t{ 0 }) + + (inline_glv_double ? size_t{ 64 } * n : size_t{ 0 }) + + (profile_threads * size_t{ 1024 }); + + const rpd::PerWorkerArenaLayout budget_layout( + /*chunk_capacity=*/SUBCHUNK_ENTRIES_CAP, + global_max_overflow_per_window, + dedup_active, + phase_a_cluster_members_cap, + phase_a_cluster_offsets_cap, + /*windows_per_batch=*/0, + /*dense_stride_est=*/0); + const size_t worker_union_bytes_for_budget = budget_layout.per_worker_union_bytes; + const size_t fixed_overhead = (worker_union_bytes_for_budget * worker_total) + + (size_t{ 96 } * rpd::VAR_WINDOW_MAX_WINDOWS) + (size_t{ 8 } * (num_threads + 1)) + + phase_one_prologue_bytes; + const size_t available_budget = + (BATCH_MEM_BUDGET > fixed_overhead) ? (BATCH_MEM_BUDGET - fixed_overhead) : size_t{ 0 }; + const size_t windows_per_batch = (per_window_bytes_lo == 0 || available_budget == 0) + ? std::max(1, sched.num_windows) + : std::min(std::max(1, available_budget / per_window_bytes_lo), + static_cast(sched.num_windows)); + + auto align_up = [](size_t off, size_t align) -> size_t { return (off + align - 1) & ~(align - 1); }; + auto layout_add = [&](size_t& off, size_t bytes, size_t align) { off = align_up(off, align) + bytes; }; + auto bump_fits = [&](size_t count, + size_t size, + size_t align, + size_t& cursor, + size_t bound, + size_t base_offset, + size_t base_misalign) { + const size_t cur_addr_mod = (base_misalign + base_offset + cursor) & (align - 1); + const size_t align_delta = (cur_addr_mod == 0) ? size_t{ 0 } : (align - cur_addr_mod); + const size_t aligned_local = cursor + align_delta; + const size_t bytes = count * size; + if (aligned_local + bytes > bound) { + return false; + } + cursor = aligned_local + bytes; + return true; + }; + + for (size_t base_misalign = 0; base_misalign < alignof(AffineElement); ++base_misalign) { + size_t arena_cursor = 0; + if (!bump_fits(n, sizeof(uint8_t), alignof(uint8_t), arena_cursor, arena_capacity, 0, base_misalign)) { + return false; + } + if (!bump_fits(profile_threads, + sizeof(std::array), + alignof(std::array), + arena_cursor, + arena_capacity, + 0, + base_misalign)) { + return false; + } + if (use_glv) { + if (!bump_fits( + n, sizeof(ScalarField), alignof(ScalarField), arena_cursor, arena_capacity, 0, base_misalign)) { + return false; + } + if (inline_glv_double && + !bump_fits( + n, sizeof(AffineElement), alignof(AffineElement), arena_cursor, arena_capacity, 0, base_misalign)) { + return false; + } + } + const size_t bytes_P_prefix = arena_cursor; + + const rpd::PerWorkerArenaLayout worker_layout(chunk_capacity, + global_max_overflow_per_window, + dedup_active, + phase_a_cluster_members_cap, + phase_a_cluster_offsets_cap, + windows_per_batch, + dense_stride_est); + constexpr size_t WORKER_SLAB_ALIGN = rpd::PerWorkerArenaLayout::WORKER_SLAB_ALIGN; + const size_t per_worker_bytes = worker_layout.per_worker_bytes; + + size_t bytes_P_extra_layout = 0; + layout_add(bytes_P_extra_layout, sizeof(Element) * rpd::VAR_WINDOW_MAX_WINDOWS, alignof(Element)); + if (dedup_active) { + layout_add(bytes_P_extra_layout, sizeof(uint32_t) * n, alignof(uint32_t)); + layout_add(bytes_P_extra_layout, sizeof(AffineElement) * rpd::DEDUP_MAX_CLUSTERS, alignof(AffineElement)); + } + const size_t bytes_P_min = align_up(bytes_P_prefix, alignof(Element)) + bytes_P_extra_layout; + const size_t bytes_P = align_up(bytes_P_min + base_misalign, WORKER_SLAB_ALIGN) - base_misalign; + const size_t bytes_W = per_worker_bytes * worker_total; + if (bytes_P + bytes_W > arena_capacity) { + return false; + } + const size_t bytes_S_total = arena_capacity - bytes_P - bytes_W; + size_t zone_S_cursor = 0; + const size_t zone_S_base = bytes_P + bytes_W; + + const size_t schedule_total = windows_per_batch * capacity_lo; + if (!bump_fits(schedule_total, + sizeof(uint32_t), + alignof(uint32_t), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign)) { + return false; + } + const size_t hist_h_bytes_total = size_t{ 4 } * windows_per_batch * num_threads * B_eff; + size_t o_layout_cur = 0; + o_layout_cur = align_up(o_layout_cur, alignof(rpd::ChunkOutput)); + o_layout_cur += sizeof(rpd::ChunkOutput) * windows_per_batch * num_threads; + o_layout_cur = align_up(o_layout_cur, alignof(Element)); + o_layout_cur += sizeof(Element) * num_threads * windows_per_batch; + const size_t hist_slot_cells = + (std::max(hist_h_bytes_total, o_layout_cur) + sizeof(AffineElement) - 1) / sizeof(AffineElement); + const size_t dense_slot_cells = + ((size_t{ 65 } * windows_per_batch * bucket_partials_per_window_max) + sizeof(AffineElement) - 1) / + sizeof(AffineElement); + if (!bump_fits(hist_slot_cells, + sizeof(AffineElement), + alignof(AffineElement), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign) || + !bump_fits(dense_slot_cells, + sizeof(AffineElement), + alignof(AffineElement), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign) || + !bump_fits(windows_per_batch * (B_eff + 1), + sizeof(size_t), + alignof(size_t), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign) || + !bump_fits(windows_per_batch * (num_threads + 1), + sizeof(size_t), + alignof(size_t), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign) || + !bump_fits(windows_per_batch * (num_threads + 1), + sizeof(size_t), + alignof(size_t), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign) || + !bump_fits(windows_per_batch * num_threads, + sizeof(size_t), + alignof(size_t), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign) || + !bump_fits((num_threads * windows_per_batch) + 1, + sizeof(size_t), + alignof(size_t), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign) || + !bump_fits(num_threads + 1, + sizeof(size_t), + alignof(size_t), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign) || + !bump_fits(windows_per_batch * num_threads, + sizeof(size_t), + alignof(size_t), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign) || + !bump_fits(windows_per_batch * num_threads, + sizeof(size_t), + alignof(size_t), + zone_S_cursor, + bytes_S_total, + zone_S_base, + base_misalign)) { + return false; + } + } + return true; +} } // namespace template class ScalarMultiplicationTest : public ::testing::Test { @@ -31,18 +306,12 @@ template class ScalarMultiplicationTest : public ::testing::Test { static constexpr size_t kMaxBatchMSMs = 32; static constexpr size_t kMaxBatchPointsPerMSM = 400; - // Used by test_consume_point_batch{,_and_accumulate}, which read generators[0..kMaxBucketTestPoints). - static constexpr size_t kMaxBucketTestPoints = 30071; - // Pinning invariants: these tests walk generators[]/scalars[] without bounds checks beyond an // occasional runtime ASSERT_LT. Pin the relationships at compile time so changing any one of // these constants in isolation cannot regress into an out-of-bounds walk. static_assert(kMaxBatchMSMs * kMaxBatchPointsPerMSM < num_points, "test_batch_multi_scalar_mul can exceed num_points; " "raise num_points or lower kMaxBatchMSMs / kMaxBatchPointsPerMSM"); - static_assert(kMaxBucketTestPoints <= num_points, - "test_consume_point_batch* reads past end of generators; " - "raise num_points or lower kMaxBucketTestPoints"); static inline std::vector generators{}; static inline std::vector scalars{}; @@ -76,6 +345,15 @@ template class ScalarMultiplicationTest : public ::testing::Test { return AffineElement(expected_acc); } + static std::vector make_repeated_test_points(size_t num_pts) + { + std::vector points(num_pts); + for (size_t i = 0; i < num_pts; ++i) { + points[i] = generators[i % generators.size()]; + } + return points; + } + static void SetUpTestSuite() { generators.resize(num_points); @@ -93,218 +371,6 @@ template class ScalarMultiplicationTest : public ::testing::Test { // ======================= Test Methods ======================= - void test_get_scalar_slice() - { - constexpr uint32_t fr_size = 254; - constexpr uint32_t slice_bits = 7; - constexpr uint32_t num_slices = (fr_size + 6) / 7; - constexpr uint32_t last_slice_bits = fr_size - ((num_slices - 1) * slice_bits); - - for (size_t x = 0; x < 100; ++x) { - uint256_t input_u256 = engine.get_random_uint256(); - input_u256.data[3] = input_u256.data[3] & 0x3FFFFFFFFFFFFFFF; // 254 bits - while (input_u256 > ScalarField::modulus) { - input_u256 -= ScalarField::modulus; - } - std::vector slices(num_slices); - - uint256_t acc = input_u256; - for (uint32_t i = 0; i < num_slices; ++i) { - uint32_t mask = ((1U << slice_bits) - 1U); - uint32_t shift = slice_bits; - if (i == 0) { - mask = ((1U << last_slice_bits) - 1U); - shift = last_slice_bits; - } - slices[num_slices - 1 - i] = static_cast((acc & mask).data[0]); - acc = acc >> shift; - } - - ScalarField input(input_u256); - input.self_from_montgomery_form_reduced(); - - ASSERT_EQ(input.data[0], input_u256.data[0]); - ASSERT_EQ(input.data[1], input_u256.data[1]); - ASSERT_EQ(input.data[2], input_u256.data[2]); - ASSERT_EQ(input.data[3], input_u256.data[3]); - - for (uint32_t i = 0; i < num_slices; ++i) { - uint32_t result = scalar_multiplication::MSM::get_scalar_slice(input, i, slice_bits); - EXPECT_EQ(result, slices[i]); - } - } - } - - void test_consume_point_batch() - { - const size_t total_points = kMaxBucketTestPoints; - const size_t num_buckets = 128; - - std::vector input_point_schedule; - for (size_t i = 0; i < total_points; ++i) { - uint64_t bucket = static_cast(engine.get_random_uint8()) & 0x7f; - uint64_t schedule = static_cast(bucket) + (static_cast(i) << 32); - input_point_schedule.push_back(schedule); - } - typename scalar_multiplication::MSM::AffineAdditionData affine_data; - typename scalar_multiplication::MSM::BucketAccumulators bucket_data(num_buckets); - scalar_multiplication::MSM::batch_accumulate_points_into_buckets( - input_point_schedule, generators, affine_data, bucket_data); - - std::vector expected_buckets(num_buckets); - for (auto& e : expected_buckets) { - e.self_set_infinity(); - } - for (size_t i = 0; i < total_points; ++i) { - uint64_t bucket = input_point_schedule[i] & 0xFFFFFFFF; - EXPECT_LT(static_cast(bucket), num_buckets); - expected_buckets[static_cast(bucket)] += generators[i]; - } - for (size_t i = 0; i < num_buckets; ++i) { - if (!expected_buckets[i].is_point_at_infinity()) { - AffineElement expected(expected_buckets[i]); - EXPECT_EQ(expected, bucket_data.buckets[i]); - } else { - EXPECT_FALSE(bucket_data.bucket_exists.get(i)); - } - } - } - - void test_consume_point_batch_and_accumulate() - { - const size_t total_points = kMaxBucketTestPoints; - const size_t num_buckets = 128; - - std::vector input_point_schedule; - for (size_t i = 0; i < total_points; ++i) { - uint64_t bucket = static_cast(engine.get_random_uint8()) & 0x7f; - uint64_t schedule = static_cast(bucket) + (static_cast(i) << 32); - input_point_schedule.push_back(schedule); - } - typename scalar_multiplication::MSM::AffineAdditionData affine_data; - typename scalar_multiplication::MSM::BucketAccumulators bucket_data(num_buckets); - scalar_multiplication::MSM::batch_accumulate_points_into_buckets( - input_point_schedule, generators, affine_data, bucket_data); - - Element result = scalar_multiplication::MSM::accumulate_buckets(bucket_data); - - Element expected_acc; - expected_acc.self_set_infinity(); - size_t num_threads = get_num_cpus(); - std::vector expected_accs(num_threads); - size_t range_per_thread = (total_points + num_threads - 1) / num_threads; - parallel_for(num_threads, [&](size_t thread_idx) { - Element expected_thread_acc; - expected_thread_acc.self_set_infinity(); - size_t start = thread_idx * range_per_thread; - size_t end = (thread_idx == num_threads - 1) ? total_points : (thread_idx + 1) * range_per_thread; - bool skip = start >= total_points; - if (!skip) { - for (size_t i = start; i < end; ++i) { - ScalarField scalar = input_point_schedule[i] & 0xFFFFFFFF; - expected_thread_acc += generators[i] * scalar; - } - } - expected_accs[thread_idx] = expected_thread_acc; - }); - - for (size_t i = 0; i < num_threads; ++i) { - expected_acc += expected_accs[i]; - } - AffineElement expected(expected_acc); - EXPECT_EQ(AffineElement(result), expected); - } - - void test_radix_sort_count_zero_entries() - { - const size_t total_points = 30071; - - std::vector input_point_schedule; - for (size_t i = 0; i < total_points; ++i) { - uint64_t bucket = static_cast(engine.get_random_uint8()) & 0x7f; - uint64_t schedule = static_cast(bucket) + (static_cast(i) << 32); - input_point_schedule.push_back(schedule); - } - - size_t result = scalar_multiplication::sort_point_schedule_and_count_zero_buckets( - &input_point_schedule[0], input_point_schedule.size(), 7); - - // Verify zero entry count is correct - size_t expected = 0; - for (size_t i = 0; i < total_points; ++i) { - expected += static_cast((input_point_schedule[i] & 0xFFFFFFFF) == 0); - } - EXPECT_EQ(result, expected); - - // Verify the array is sorted by bucket index (lower 32 bits) - for (size_t i = 1; i < total_points; ++i) { - uint32_t prev_bucket = static_cast(input_point_schedule[i - 1]); - uint32_t curr_bucket = static_cast(input_point_schedule[i]); - EXPECT_LE(prev_bucket, curr_bucket) << "Array not sorted at index " << i; - } - } - - // Regression test: radix sort zero-counting bug for bucket_index_bits > 16 (3+ recursion levels). - // The recursive call passes `keys` instead of `top_level_keys`, causing num_zero_entries to be - // overwritten by non-zero-bucket counts when the MSD radix sort recurses 3+ levels deep. - void test_radix_sort_count_zero_entries_wide_buckets() - { - // Use bucket_index_bits = 17, which pads to 24 bits → 3 recursion levels (shift: 16→8→0). - // At the 3rd level, the top_level_keys bug causes zero-counting to fire for every - // level-0 bucket's sub-bucket-0, not just the bucket-0 chain. - constexpr uint32_t bucket_index_bits = 17; - constexpr size_t num_entries = 1000; - - std::vector schedule(num_entries); - - // Place some entries with bucket_index = 0 (true zero-bucket entries) - const size_t num_true_zeros = 10; - for (size_t i = 0; i < num_true_zeros; ++i) { - schedule[i] = static_cast(i) << 32; // point_index=i, bucket_index=0 - } - - // Place entries with bucket_index = 65536 (= 1 << 16). These have bits [0:16) all zero, - // so the buggy code counts them as zero-bucket entries after the final recursion level - // overwrites num_zero_entries from the level-0 bucket 1 path. - const size_t num_false_zeros = 20; - for (size_t i = 0; i < num_false_zeros; ++i) { - size_t idx = num_true_zeros + i; - schedule[idx] = (static_cast(idx) << 32) | 65536ULL; - } - - // Fill remaining entries with random non-zero bucket indices that won't confuse the count - for (size_t i = num_true_zeros + num_false_zeros; i < num_entries; ++i) { - uint32_t bucket = (engine.get_random_uint32() % ((1U << bucket_index_bits) - 1)) + 1; - // Avoid bucket_index values with all lower 16 bits zero (i.e., multiples of 65536) - if ((bucket & 0xFFFF) == 0) { - bucket |= 1; - } - schedule[i] = (static_cast(i) << 32) | static_cast(bucket); - } - - size_t result = scalar_multiplication::sort_point_schedule_and_count_zero_buckets( - schedule.data(), num_entries, bucket_index_bits); - - // Count actual zero-bucket entries after sort - size_t expected = 0; - for (size_t i = 0; i < num_entries; ++i) { - if ((schedule[i] & scalar_multiplication::BUCKET_INDEX_MASK) == 0) { - expected++; - } - } - - EXPECT_EQ(result, expected) << "Zero-bucket count is wrong for bucket_index_bits=" << bucket_index_bits - << ". Got " << result << ", expected " << expected - << " (likely overwritten by count from a non-zero bucket)"; - - // Also verify the array is sorted - for (size_t i = 1; i < num_entries; ++i) { - uint32_t prev = static_cast(schedule[i - 1]); - uint32_t curr = static_cast(schedule[i]); - EXPECT_LE(prev, curr) << "Array not sorted at index " << i; - } - } - void test_pippenger_low_memory() { std::span test_scalars(&scalars[0], num_points); @@ -322,30 +388,30 @@ template class ScalarMultiplicationTest : public ::testing::Test { std::vector expected(num_msms); std::vector> batch_scalars_copies(num_msms); - std::vector> batch_points_span; - std::vector> batch_scalars_spans; + std::vector start_indices(num_msms); + std::vector> batch_scalars_spans; size_t vector_offset = 0; for (size_t k = 0; k < num_msms; ++k) { const size_t num_pts = static_cast(engine.get_random_uint16()) % kMaxBatchPointsPerMSM; ASSERT_LT(vector_offset + num_pts, num_points); - std::span batch_points(&generators[vector_offset], num_pts); batch_scalars_copies[k].resize(num_pts); for (size_t i = 0; i < num_pts; ++i) { batch_scalars_copies[k][i] = scalars[vector_offset + i]; } + start_indices[k] = vector_offset; + batch_scalars_spans.emplace_back(vector_offset, std::span(batch_scalars_copies[k])); vector_offset += num_pts; - batch_points_span.push_back(batch_points); - batch_scalars_spans.push_back(batch_scalars_copies[k]); - expected[k] = naive_msm(batch_scalars_spans[k], batch_points_span[k]); + std::span batch_points(&generators[start_indices[k]], num_pts); + expected[k] = naive_msm(batch_scalars_copies[k], batch_points); } std::vector result = - scalar_multiplication::MSM::batch_multi_scalar_mul(batch_points_span, batch_scalars_spans); + scalar_multiplication::MSM::batch_multi_scalar_mul(generators, batch_scalars_spans); EXPECT_EQ(result, expected); } @@ -356,8 +422,7 @@ template class ScalarMultiplicationTest : public ::testing::Test { std::vector expected(num_msms); std::vector> batch_scalars(num_msms); - std::vector> batch_points_span; - std::vector> batch_scalars_spans; + std::vector> batch_scalars_spans; for (size_t k = 0; k < num_msms; ++k) { const size_t num_pts = 33; @@ -367,7 +432,7 @@ template class ScalarMultiplicationTest : public ::testing::Test { size_t fixture_offset = k * num_pts; - std::span batch_points(&generators[fixture_offset], num_pts); + std::span batch_points(&generators[fixture_offset], num_pts); for (size_t i = 0; i < 13; ++i) { test_scalars[i] = 0; } @@ -377,18 +442,81 @@ template class ScalarMultiplicationTest : public ::testing::Test { for (size_t i = 23; i < num_pts; ++i) { test_scalars[i] = 0; } - batch_points_span.push_back(batch_points); - batch_scalars_spans.push_back(batch_scalars[k]); + batch_scalars_spans.emplace_back(fixture_offset, std::span(batch_scalars[k])); expected[k] = naive_msm(batch_scalars[k], batch_points); } std::vector result = - scalar_multiplication::MSM::batch_multi_scalar_mul(batch_points_span, batch_scalars_spans); + scalar_multiplication::MSM::batch_multi_scalar_mul(generators, batch_scalars_spans); EXPECT_EQ(result, expected); } + // Larger workload that crosses the batched dispatcher's `total_nonzero > 4096` eligibility + // threshold so the multi-MSM Phases 1-6b pipeline (REBALANCE path) is exercised, not the + // per-MSM delegation fallback. + void test_batch_multi_scalar_mul_large_dense() + { + constexpr size_t num_msms = 4; + constexpr size_t per_msm_n = 1 << 13; // 8192 points per MSM, total = 32768 + + std::vector expected(num_msms); + std::vector> batch_scalars(num_msms); + std::vector> batch_scalars_spans; + + for (size_t k = 0; k < num_msms; ++k) { + batch_scalars[k].resize(per_msm_n); + for (size_t i = 0; i < per_msm_n; ++i) { + batch_scalars[k][i] = scalars[k * per_msm_n + i]; + } + std::span pts(&generators[0], per_msm_n); + batch_scalars_spans.emplace_back(0, std::span(batch_scalars[k])); + expected[k] = naive_msm(batch_scalars[k], pts); + } + + std::vector result = + scalar_multiplication::MSM::batch_multi_scalar_mul(generators, batch_scalars_spans); + + for (size_t k = 0; k < num_msms; ++k) { + EXPECT_EQ(result[k], expected[k]) << "MSM " << k << " mismatched"; + } + } + + // Ragged batch with mixed densities — the workload pattern for translator wires + databus. + // K=5 MSMs of varying sizes, varying zero density, all sharing the same SRS prefix. + void test_batch_multi_scalar_mul_ragged() + { + const std::vector sizes = { 16384, 4096, 8192, 1024, 12000 }; + const size_t num_msms = sizes.size(); + + std::vector expected(num_msms); + std::vector> batch_scalars(num_msms); + std::vector> batch_scalars_spans; + + for (size_t k = 0; k < num_msms; ++k) { + const size_t n = sizes[k]; + batch_scalars[k].resize(n); + for (size_t i = 0; i < n; ++i) { + if ((k == 1 || k == 3) && (i % 4 != 0)) { + batch_scalars[k][i] = ScalarField::zero(); + } else { + batch_scalars[k][i] = scalars[(k * 17 + i) % num_points]; + } + } + std::span pts(&generators[0], n); + batch_scalars_spans.emplace_back(0, std::span(batch_scalars[k])); + expected[k] = naive_msm(batch_scalars[k], pts); + } + + std::vector result = + scalar_multiplication::MSM::batch_multi_scalar_mul(generators, batch_scalars_spans); + + for (size_t k = 0; k < num_msms; ++k) { + EXPECT_EQ(result[k], expected[k]) << "MSM " << k << " (n=" << sizes[k] << ") mismatched"; + } + } + void test_msm() { const size_t start_index = 1234; @@ -453,8 +581,7 @@ template class ScalarMultiplicationTest : public ::testing::Test { std::vector> batch_scalars(num_msms); std::vector> scalars_copies(num_msms); - std::vector> batch_points; - std::vector> batch_scalar_spans; + std::vector> batch_scalar_spans; for (size_t k = 0; k < num_msms; ++k) { batch_scalars[k].resize(num_pts); @@ -465,11 +592,10 @@ template class ScalarMultiplicationTest : public ::testing::Test { scalars_copies[k][i] = batch_scalars[k][i]; } - batch_points.push_back(std::span(&generators[k * num_pts], num_pts)); - batch_scalar_spans.push_back(batch_scalars[k]); + batch_scalar_spans.emplace_back(k * num_pts, std::span(batch_scalars[k])); } - scalar_multiplication::MSM::batch_multi_scalar_mul(batch_points, batch_scalar_spans); + scalar_multiplication::MSM::batch_multi_scalar_mul(generators, batch_scalar_spans); for (size_t k = 0; k < num_msms; ++k) { for (size_t i = 0; i < num_pts; ++i) { @@ -629,33 +755,488 @@ template class ScalarMultiplicationTest : public ::testing::Test { AffineElement expected = naive_msm(test_scalars, points); EXPECT_EQ(AffineElement(result), expected); } + + /** + * @brief Validate that a non-zero start_index in the PolynomialSpan is honoured. + * + * `pippenger`/`pippenger_unsafe` index the `points` argument from `start_index`, + * so `points.size()` must cover `[start_index, start_index + n_used)`. The + * scalars span starts at `start_index` with `n_used` elements. + */ + void test_offset_span(size_t n_total, size_t start_index, size_t n_used, uint64_t seed) + { + auto& rng = numeric::get_debug_randomness(true, seed); + std::vector test_scalars(n_total); + std::vector input_points(start_index + n_used); + for (size_t i = 0; i < n_total; ++i) { + test_scalars[i] = ScalarField::random_element(&rng); + } + for (size_t i = 0; i < input_points.size(); ++i) { + input_points[i] = AffineElement(Element::random_element(&rng)); + } + + PolynomialSpan scalar_span{ + start_index, std::span{ test_scalars.data() + start_index, n_used } + }; + + Element actual = scalar_multiplication::pippenger_unsafe(scalar_span, input_points); + + Element expected; + expected.self_set_infinity(); + for (size_t i = 0; i < n_used; ++i) { + expected += input_points[start_index + i] * test_scalars[start_index + i]; + } + EXPECT_EQ(AffineElement(actual), AffineElement(expected)) + << "Offset MSM mismatch at n_total=" << n_total << " start_index=" << start_index << " n_used=" << n_used; + } + + /** + * @brief Coverage at very large N (exercises the non-GLV path on WASM, where + * n_input > 2^16 disables the GLV decomposition). + */ + void test_large_n_non_glv() + { + const size_t num_pts = scalar_multiplication::round_parallel_detail::GLV_SMALL_N_THRESHOLD + 31; + auto& rng = numeric::get_debug_randomness(true, 0x5eedu + 35); + std::vector points(num_pts); + std::vector test_scalars(num_pts); + for (size_t i = 0; i < num_pts; ++i) { + points[i] = AffineElement(Element::random_element(&rng)); + test_scalars[i] = ScalarField::random_element(&rng); + } + + PolynomialSpan scalar_span(0, test_scalars); + AffineElement result = scalar_multiplication::MSM::msm(points, scalar_span); + AffineElement expected = naive_msm(test_scalars, points); + EXPECT_EQ(result, expected); + } + + /** + * @brief Force every Pippenger window to contain a single mega-run of one digit. + * + * Setting every input scalar to the same value means each window's signed-digit + * recoding is the same for all points. The Stage 6a schedule for any window is + * therefore a single contiguous run of that digit across all N entries — far + * longer than SUBCHUNK_ENTRIES_CAP, so each thread's slice gets split into many + * sub-chunks all targeting the same bucket slot. This exercises the + * seam-overflow merge path: the first sub-chunk writes the dense slot and every + * subsequent sub-chunk routes its partial through the per-window overflow buffer, + * which is folded back into the slot at end-of-window. + */ + void test_msm_single_digit_mega_run() + { + const size_t num_pts = 100000; + auto& rng = numeric::get_debug_randomness(true, 0x5eedu + 36); + std::vector points(num_pts); + for (size_t i = 0; i < num_pts; ++i) { + points[i] = AffineElement(Element::random_element(&rng)); + } + std::vector uniform_scalars(num_pts, ScalarField(7)); + PolynomialSpan scalar_span(0, uniform_scalars); + + AffineElement result = scalar_multiplication::MSM::msm(points, scalar_span); + AffineElement expected = + naive_msm(std::span(uniform_scalars), std::span(points)); + EXPECT_EQ(result, expected); + } + + /** + * @brief Stress-test the dedup pass's worst-case caps and the split-cluster carry. + * + * Inputs: `num_pts` (default 50 000) scalars all equal to a single + * dedup-eligible value (`msb >= c`) — i.e. one mega-cluster. With the dedup + * caps `MAX_CLUSTERS = 16 384`, `MAX_MEMBERS = 32 768`, `MAX_CHUNK_MEMBERS = 8 192`, + * this exercises: + * 1. The MAX_MEMBERS cap: only the first 32 K duplicates are recorded; the + * remaining ~17.5 K fall through to the standard pippenger path. + * 2. The split-cluster carry in the chunked tree-reduce: the 32 K-member + * cluster is split across 4 chunks of 8 K, with the partial sum carried + * into the next chunk as that cluster's first member. + * + * Activation is forced via the explicit `dedup_hint` parameter on + * `MSM::msm`. Validation: result equals the naive MSM regardless + * of which scalars dedup picked up. + */ + void test_msm_dedup_cap_and_carry() + { + const size_t num_pts = 50000; + // Pick a dedup-eligible scalar: msb >= c (c ≈ 11 for n ≈ 50 000), so any value + // ≥ 2^11 works. Use 2^200 so msb is firmly large for any c the dispatch picks. + const ScalarField val = ScalarField(uint256_t(0, 0, 0, uint64_t{ 1 } << (200 - 192))); // 2^200 + std::vector uniform_scalars(num_pts, val); + std::vector points = make_repeated_test_points(num_pts); + PolynomialSpan scalar_span(0, uniform_scalars); + + AffineElement result = scalar_multiplication::MSM::msm( + points, scalar_span, /*handle_edge_cases=*/false, /*dedup_hint=*/true); + + AffineElement expected = + naive_msm(std::span(uniform_scalars), std::span(points)); + EXPECT_EQ(result, expected); + } + + /** + * @brief Stress-test dedup cap fallback across many small clusters. + * + * This shape opens more clusters than can fit in the flattened member slab: + * 12K distinct scalar values, each repeated 3 times, produce 36K potential + * cluster members against the 32K member cap. Clusters that do not fit must + * remain unpublished and fall through the ordinary Pippenger path. + */ + void test_msm_dedup_many_small_clusters_cap() + { + constexpr size_t NUM_CLUSTERS = 12000; + constexpr size_t CLUSTER_SIZE = 3; + const size_t num_pts = NUM_CLUSTERS * CLUSTER_SIZE; + + std::vector scalars; + scalars.reserve(num_pts); + const uint256_t high_bit(0, 0, 0, uint64_t{ 1 } << (200 - 192)); + for (size_t i = 0; i < NUM_CLUSTERS; ++i) { + const ScalarField val = ScalarField(high_bit + uint256_t(i + 1)); + for (size_t j = 0; j < CLUSTER_SIZE; ++j) { + scalars.push_back(val); + } + } + + std::vector points = make_repeated_test_points(num_pts); + PolynomialSpan scalar_span(0, scalars); + + AffineElement result = + scalar_multiplication::MSM::msm(points, scalar_span, /*handle_edge_cases=*/false, true); + AffineElement expected = naive_msm(std::span(scalars), std::span(points)); + EXPECT_EQ(result, expected); + } + + // ============================================================================ + // Dispatch-coverage tests for `pippenger_round_parallel`. + // + // The function has several branches that need to all be exercised: + // * `n_input == 0` → infinity + // * `pts_per_thread < MIN_PTS_PER_THREAD_FOR_PIPPENGER` → trivial_msm_threaded + // (single-thread → trivial_msm, otherwise straus_msm per worker) + // * Otherwise → main pippenger pipeline + // - use_glv=true (n_input ≤ GLV_SMALL_N_THRESHOLD) + // - use_glv=false (n_input > GLV_SMALL_N_THRESHOLD; only on huge N) + // * `external_glv_doubled` provided vs not (drives one of the GLV-split branches) + // + // Each test below restores `bb::set_parallel_for_concurrency` to its original + // value before returning, even if the assertion fails, so subsequent tests are + // unaffected. + // ============================================================================ + + /// RAII helper to scope a `bb::set_parallel_for_concurrency` change to one test. + class ConcurrencyScope { + size_t prev_; + + public: + explicit ConcurrencyScope(size_t n) + : prev_(bb::get_num_cpus()) + { + bb::set_parallel_for_concurrency(n); + } + ~ConcurrencyScope() { bb::set_parallel_for_concurrency(prev_); } + ConcurrencyScope(const ConcurrencyScope&) = delete; + ConcurrencyScope& operator=(const ConcurrencyScope&) = delete; + ConcurrencyScope(ConcurrencyScope&&) = delete; + ConcurrencyScope& operator=(ConcurrencyScope&&) = delete; + }; + + /// Run pippenger_round_parallel at the given size and validate it equals + /// the naive MSM. `start_index` shifts the (scalars, points) slice in the input + /// arrays. This is the workhorse used by all dispatch tests below. + void check_internal_against_naive(size_t n, size_t start_index, const char* label) + { + ASSERT_LE(start_index + n, num_points) << label; + + std::span scalar_subspan(&scalars[start_index], n); + std::span point_subspan(&generators[0], start_index + n); + PolynomialSpan scalar_span{ start_index, scalar_subspan }; + + Element actual = scalar_multiplication::pippenger_round_parallel(scalar_span, point_subspan); + + Element expected; + expected.self_set_infinity(); + for (size_t i = 0; i < n; ++i) { + expected += point_subspan[start_index + i] * scalar_subspan[i]; + } + + EXPECT_EQ(AffineElement(actual), AffineElement(expected)) + << label << " (n=" << n << ", start_index=" << start_index << ")"; + } + + /// Single-thread (`bb::set_parallel_for_concurrency(1)`) — every dispatch path + /// must still produce a correct answer. Tests across N from 1 up past + /// MIN_PTS_PER_THREAD_FOR_PIPPENGER and into the affine pippenger range. + void test_pippenger_internal_single_thread() + { + ConcurrencyScope scope(1); + // n_input == 0: infinity short-circuit. + { + std::span empty_points; + std::span empty_scalars; + PolynomialSpan empty_span{ 0, empty_scalars }; + Element r = scalar_multiplication::pippenger_round_parallel(empty_span, empty_points); + EXPECT_TRUE(r.is_point_at_infinity()); + } + // Walk N across all dispatch boundaries with a single thread. With 1 thread, + // pts_per_thread == n; the trivial dispatch fires up to N=23, falls through + // at N=24+. The fall-through path then runs the affine pippenger with + // num_threads=1. + for (size_t n : { size_t{ 1 }, + size_t{ 2 }, + size_t{ 3 }, + size_t{ 4 }, + size_t{ 23 }, + size_t{ 24 }, + size_t{ 25 }, + size_t{ 32 }, + size_t{ 64 }, + size_t{ 100 }, + size_t{ 192 }, + size_t{ 1000 } }) { + check_internal_against_naive(n, 0, "single_thread"); + } + } + + /// Specifically the case the user called out: single thread, + /// n = MIN_PTS_PER_THREAD_FOR_PIPPENGER + 1. Was where the old assert tripped. + void test_pippenger_internal_single_thread_at_dispatch_threshold_plus_one() + { + ConcurrencyScope scope(1); + constexpr size_t kThreshold = scalar_multiplication::MIN_PTS_PER_THREAD_FOR_PIPPENGER; + check_internal_against_naive(kThreshold + 1, 0, "single_thread n=Threshold+1"); + // Also exercise N just below where `chunk_len = n / num_threads = n / 1 = n` + // approaches MIN_BATCH_CAPACITY=32 — the (now-removed) brittle fallback used + // to fire here; we want the affine path to still run and produce correct + // output even with very small chunks. + for (size_t n : { kThreshold + 1, size_t{ 32 }, size_t{ 33 }, size_t{ 50 }, size_t{ 100 } }) { + check_internal_against_naive(n, 0, "single_thread small-chunk"); + } + } + + /// Walk N across the dispatch threshold for HARDWARE_CONCURRENCY=2,4,8,16. At + /// each thread count the dispatch fires when `pts_per_thread < 24`; we test + /// just-below, exactly-at, and just-above the boundary, plus a midrange value. + void test_pippenger_internal_dispatch_threshold_per_thread_count() + { + constexpr size_t kThreshold = scalar_multiplication::MIN_PTS_PER_THREAD_FOR_PIPPENGER; + for (size_t threads : { size_t{ 2 }, size_t{ 4 }, size_t{ 8 }, size_t{ 16 } }) { + ConcurrencyScope scope(threads); + // Dispatch boundary is at n = threads * kThreshold (= pts_per_thread = 24). + const size_t boundary = threads * kThreshold; + for (size_t n : { boundary - 1, boundary, boundary + 1 }) { + check_internal_against_naive(n, 0, "dispatch_boundary"); + } + } + } + + /// Same dispatch coverage but with a non-zero start_index — make sure the + /// PolynomialSpan offset is honoured in both the dispatch (small-N → trivial) + /// and fall-through (affine pippenger) paths. + void test_pippenger_internal_offset_span_dispatch() + { + ConcurrencyScope scope(8); + // Small N (will dispatch to trivial_msm_threaded). + check_internal_against_naive(/*n=*/64, /*start_index=*/17, "offset small-N"); + // Just above dispatch threshold (8 threads → boundary at 192). + check_internal_against_naive(/*n=*/200, /*start_index=*/13, "offset just-above-boundary"); + // Mid-N falls through into pippenger. + check_internal_against_naive(/*n=*/1024, /*start_index=*/41, "offset mid-N"); + } + + /// Test a scalar layout that is sometimes problematic: all-zero scalars. + /// The result must be infinity. Exercises both the trivial path (small N) and + /// the affine path (mid N). + void test_pippenger_internal_all_zero_scalars() + { + ConcurrencyScope scope(8); + // Save and restore the global scalars buffer. + std::vector saved(scalars.begin(), scalars.begin() + 1024); + for (size_t i = 0; i < saved.size(); ++i) { + scalars[i] = ScalarField::zero(); + } + for (size_t n : { size_t{ 1 }, size_t{ 24 }, size_t{ 100 }, size_t{ 1000 } }) { + std::span sub(&scalars[0], n); + std::span pts(&generators[0], n); + PolynomialSpan sp{ 0, sub }; + Element r = scalar_multiplication::pippenger_round_parallel(sp, pts); + EXPECT_TRUE(r.is_point_at_infinity()) << "all-zero n=" << n; + } + // Restore. + for (size_t i = 0; i < saved.size(); ++i) { + scalars[i] = saved[i]; + } + } + + /// Mix of zero and non-zero scalars. The result should equal the naive sum + /// excluding the zero terms. Tests the trivial path (small N) and affine path. + void test_pippenger_internal_mixed_zero_scalars() + { + ConcurrencyScope scope(8); + std::vector saved(scalars.begin(), scalars.begin() + 1024); + // Zero out every other scalar. + for (size_t i = 0; i < 1024; i += 2) { + scalars[i] = ScalarField::zero(); + } + for (size_t n : { size_t{ 24 }, size_t{ 100 }, size_t{ 1024 } }) { + check_internal_against_naive(n, 0, "mixed-zero"); + } + // Restore. + for (size_t i = 0; i < saved.size(); ++i) { + scalars[i] = saved[i]; + } + } + + /// Test scalars that exercise the endomorphism k2-overflow fix (k2 + r path). + /// Picking scalar = 1 and scalar = -1 ensures we hit at least one of the + /// boundary corrections. Plus randoms at fixed seeds. + void test_pippenger_internal_extreme_scalars() + { + ConcurrencyScope scope(8); + std::vector saved(scalars.begin(), scalars.begin() + 256); + + // Scalar = 1 + for (auto& s : saved) { + (void)s; + } + for (size_t i = 0; i < 256; ++i) { + scalars[i] = ScalarField::one(); + } + check_internal_against_naive(256, 0, "scalar=1"); + + // Scalar = -1 + for (size_t i = 0; i < 256; ++i) { + scalars[i] = -ScalarField::one(); + } + check_internal_against_naive(256, 0, "scalar=-1"); + + // Restore. + for (size_t i = 0; i < saved.size(); ++i) { + scalars[i] = saved[i]; + } + } + + /// Direct calls to `trivial_msm_threaded` at a range of (thread_count, n) pairs. + /// Verifies the per-worker straus split produces the same result as a naive sum + /// regardless of slice_n. n=1 hits the `num_threads <= 1 → trivial_msm` early-out. + void test_trivial_msm_threaded_per_worker_paths() + { + for (size_t threads : { size_t{ 1 }, size_t{ 2 }, size_t{ 4 }, size_t{ 8 } }) { + ConcurrencyScope scope(threads); + for (size_t n : { size_t{ 1 }, size_t{ 2 }, size_t{ 8 }, size_t{ 32 }, size_t{ 80 }, size_t{ 160 } }) { + std::span sub(&scalars[0], n); + std::span pts(&generators[0], n); + PolynomialSpan sp{ 0, sub }; + Element actual = scalar_multiplication::trivial_msm_threaded(sp, pts); + Element expected; + expected.self_set_infinity(); + for (size_t i = 0; i < n; ++i) { + expected += pts[i] * sub[i]; + } + EXPECT_EQ(AffineElement(actual), AffineElement(expected)) + << "trivial_msm_threaded threads=" << threads << " n=" << n; + } + } + } + + /// Large-N coverage: GLV boundary on WASM is 2^16; on native 2^13. Test crossing + /// the boundary in both directions to exercise the use_glv=true and use_glv=false + /// pipelines. Native already has `LargeNNonGLV` for the false case; we add the + /// just-above and just-below GLV-boundary tests here. + void test_pippenger_internal_glv_boundary() + { + ConcurrencyScope scope(8); +#ifdef __wasm__ + constexpr size_t glv_threshold = size_t{ 1 } << 16; +#else + constexpr size_t glv_threshold = size_t{ 1 } << 13; +#endif + if (glv_threshold >= num_points) { + GTEST_SKIP() << "GLV threshold " << glv_threshold << " not exercisable with " << num_points + << " precomputed points"; + } + // Just below threshold: use_glv=true. + check_internal_against_naive(glv_threshold - 1, 0, "glv-boundary minus-1 (use_glv=true)"); + // Exactly at threshold: use_glv=true (≤ comparison). + check_internal_against_naive(glv_threshold, 0, "glv-boundary exact (use_glv=true)"); + // Just above: use_glv=false. + check_internal_against_naive(glv_threshold + 1, 0, "glv-boundary plus-1 (use_glv=false)"); + } + + /// Regression test for the arena allocator's alignment handling. + /// `make_unique_for_overwrite` returns a buffer at default-new + /// alignment (16 on x86_64); Element is alignas(32) and AffineElement is + /// alignas(64). If the arena allocator only aligns the byte offset rather + /// than the absolute address, allocations land on a 16-byte-aligned but + /// 32-byte-misaligned address, and the AVX `vmovdqa` clang lowers + /// `std::fill_n(window_partial_sums, …, point_at_infinity)` into raises + /// #GP -> SIGSEGV. This test deliberately passes an external arena whose + /// base is 16-byte aligned but not 32-byte aligned to make the failure + /// mode reproducible regardless of system allocator behaviour. + void test_pippenger_internal_misaligned_external_arena() + { + ConcurrencyScope scope(1); + constexpr size_t kThreshold = scalar_multiplication::MIN_PTS_PER_THREAD_FOR_PIPPENGER; + for (size_t n : { kThreshold + 1, size_t{ 50 }, size_t{ 100 }, size_t{ 256 } }) { + std::span scalar_subspan(&scalars[0], n); + std::span point_subspan(&generators[0], n); + PolynomialSpan scalar_span{ 0, scalar_subspan }; + + constexpr size_t kArenaCapacity = size_t{ 64 } * 1024 * 1024; + std::vector raw(kArenaCapacity + 64); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + const auto base = reinterpret_cast(raw.data()); + const uintptr_t aligned32 = (base + 31) & ~uintptr_t{ 31 }; + std::byte* misaligned = raw.data() + (aligned32 - base) + 16; + ASSERT_EQ(reinterpret_cast(misaligned) % 32, size_t{ 16 }); + std::span external_arena(misaligned, kArenaCapacity); + + Element actual = scalar_multiplication::pippenger_round_parallel( + scalar_span, point_subspan, /*dedup_hint=*/false, {}, external_arena); + + Element expected; + expected.self_set_infinity(); + for (size_t i = 0; i < n; ++i) { + expected += point_subspan[i] * scalar_subspan[i]; + } + EXPECT_EQ(AffineElement(actual), AffineElement(expected)) << "misaligned external arena (n=" << n << ")"; + } + } }; using CurveTypes = ::testing::Types; TYPED_TEST_SUITE(ScalarMultiplicationTest, CurveTypes); -// ======================= Test Wrappers ======================= - -TYPED_TEST(ScalarMultiplicationTest, GetScalarSlice) -{ - this->test_get_scalar_slice(); -} -TYPED_TEST(ScalarMultiplicationTest, ConsumePointBatch) -{ - this->test_consume_point_batch(); -} -TYPED_TEST(ScalarMultiplicationTest, ConsumePointBatchAndAccumulate) -{ - this->test_consume_point_batch_and_accumulate(); -} -TYPED_TEST(ScalarMultiplicationTest, RadixSortCountZeroEntries) -{ - this->test_radix_sort_count_zero_entries(); -} -TYPED_TEST(ScalarMultiplicationTest, RadixSortCountZeroEntriesWideBuckets) +TEST(ScalarMultiplicationArenaTest, LargeBn254RecursionVkShapeFitsComputedArena) { - this->test_radix_sort_count_zero_entries_wide_buckets(); + const size_t saved_threads = bb::get_num_cpus(); + + // CI regression from HonkRecursionConstraintTestWithoutPredicate/2.GenerateVKFromConstraints: + // Zone S attempted a uint32_t schedule allocation whose aligned end was 26,454,272 + // bytes after the computed arena left only 25,505,329 bytes in Zone S. The log does + // not expose windows_per_batch, so cover every plausible n_input divisor for that + // schedule size. + constexpr size_t schedule_slots = size_t{ 26454272 } / sizeof(uint32_t); + constexpr std::array candidate_window_batches{ 1, 2, 4, 8, 13, 16, 26, 32 }; + for (const size_t threads : { size_t{ 4 }, size_t{ 32 } }) { + bb::set_parallel_for_concurrency(threads); + for (const size_t windows_per_batch : candidate_window_batches) { + const size_t n = schedule_slots / windows_per_batch; + for (size_t effective_num_bits = 1; effective_num_bits <= 254; ++effective_num_bits) { + EXPECT_TRUE(pippenger_bn254_arena_layout_fits_for_test( + n, /*external_glv_provided=*/false, /*dedup_active=*/false, effective_num_bits)) + << "threads=" << threads << " windows_per_batch=" << windows_per_batch << " n=" << n + << " effective_num_bits=" << effective_num_bits; + } + } + } + + bb::set_parallel_for_concurrency(saved_threads); } + +// ======================= Test Wrappers ======================= + TYPED_TEST(ScalarMultiplicationTest, PippengerLowMemory) { this->test_pippenger_low_memory(); @@ -668,6 +1249,14 @@ TYPED_TEST(ScalarMultiplicationTest, BatchMultiScalarMulSparse) { this->test_batch_multi_scalar_mul_sparse(); } +TYPED_TEST(ScalarMultiplicationTest, BatchMultiScalarMulLargeDense) +{ + this->test_batch_multi_scalar_mul_large_dense(); +} +TYPED_TEST(ScalarMultiplicationTest, BatchMultiScalarMulRagged) +{ + this->test_batch_multi_scalar_mul_ragged(); +} TYPED_TEST(ScalarMultiplicationTest, MSM) { this->test_msm(); @@ -720,151 +1309,308 @@ TYPED_TEST(ScalarMultiplicationTest, PippengerUnsafeFreeFunction) { this->test_pippenger_unsafe_free_function(); } +TYPED_TEST(ScalarMultiplicationTest, OffsetSpan) +{ + this->test_offset_span(/*n_total=*/4096, /*start_index=*/7, /*n_used=*/512, 0x5eedu + 33); + this->test_offset_span(/*n_total=*/8192, /*start_index=*/4097, /*n_used=*/2048, 0x5eedu + 34); +} +TYPED_TEST(ScalarMultiplicationTest, LargeNNonGLV) +{ +#ifdef __wasm__ + GTEST_SKIP() << "Large synthetic MSM coverage is native-only; WASM coverage comes from integration flows."; +#endif + this->test_large_n_non_glv(); +} +TYPED_TEST(ScalarMultiplicationTest, MSMSingleDigitMegaRun) +{ +#ifdef __wasm__ + GTEST_SKIP() << "Large synthetic MSM coverage is native-only; WASM coverage comes from integration flows."; +#endif + this->test_msm_single_digit_mega_run(); +} +TYPED_TEST(ScalarMultiplicationTest, MSMDedupCapAndCarry) +{ +#ifdef __wasm__ + GTEST_SKIP() << "Large synthetic MSM coverage is native-only; WASM coverage comes from integration flows."; +#endif + this->test_msm_dedup_cap_and_carry(); +} +TYPED_TEST(ScalarMultiplicationTest, MSMDedupManySmallClustersCap) +{ +#ifdef __wasm__ + GTEST_SKIP() << "Large synthetic MSM coverage is native-only; WASM coverage comes from integration flows."; +#endif + this->test_msm_dedup_many_small_clusters_cap(); +} -// Curve-independent unit tests for the work-unit partitioner. -// partition_by_weight is the load-bearing balancing logic in get_work_units; pinning its -// behavior with synthetic weights makes regressions in the partition algorithm visible -// without needing a full MSM run. -namespace { +// Dispatch-coverage tests for `pippenger_round_parallel`. +TYPED_TEST(ScalarMultiplicationTest, PippengerInternalSingleThread) +{ + this->test_pippenger_internal_single_thread(); +} +TYPED_TEST(ScalarMultiplicationTest, PippengerInternalSingleThreadAtDispatchThresholdPlusOne) +{ + this->test_pippenger_internal_single_thread_at_dispatch_threshold_plus_one(); +} +TYPED_TEST(ScalarMultiplicationTest, PippengerInternalDispatchThresholdPerThreadCount) +{ + this->test_pippenger_internal_dispatch_threshold_per_thread_count(); +} +TYPED_TEST(ScalarMultiplicationTest, PippengerInternalOffsetSpanDispatch) +{ + this->test_pippenger_internal_offset_span_dispatch(); +} +TYPED_TEST(ScalarMultiplicationTest, PippengerInternalAllZeroScalars) +{ + this->test_pippenger_internal_all_zero_scalars(); +} +TYPED_TEST(ScalarMultiplicationTest, PippengerInternalMixedZeroScalars) +{ + this->test_pippenger_internal_mixed_zero_scalars(); +} +TYPED_TEST(ScalarMultiplicationTest, PippengerInternalExtremeScalars) +{ + this->test_pippenger_internal_extreme_scalars(); +} +TYPED_TEST(ScalarMultiplicationTest, TrivialMsmThreadedPerWorkerPaths) +{ + this->test_trivial_msm_threaded_per_worker_paths(); +} +TYPED_TEST(ScalarMultiplicationTest, PippengerInternalGlvBoundary) +{ + this->test_pippenger_internal_glv_boundary(); +} +TYPED_TEST(ScalarMultiplicationTest, PippengerInternalMisalignedExternalArena) +{ + this->test_pippenger_internal_misaligned_external_arena(); +} -using PartitionMSM = scalar_multiplication::MSM; -using WorkUnit = PartitionMSM::MSMWorkUnit; +// NOTE: the curve-independent `PartitionByWeight` unit tests that previously lived here +// exercised `MSM<>::MSMWorkUnit` / `MSM<>::partition_by_weight` from the OLD radix-sort + +// bucket-accumulator pippenger. Both were removed in the round-parallel refactor; the +// equivalent multi-MSM work-unit balancing logic has not yet been built (will live in +// `pippenger_round_parallel_batched` once Phases 1-6b are complete). The tests are left +// out for now and will be rewritten against the batched dispatcher's partitioner. + +// Variable-c (split-c) Pippenger dispatch — synthetic distributions per spec §"Validation". +// These force SPLIT to fire (cliff / decaying / half-zero / all-large) or to fall through +// (uniform-random / all-zero) and validate the result against `naive_msm`. +template class VariableWindowSplitDispatchTest : public ::testing::Test { + public: + using Group = typename Curve::Group; + using Element = typename Curve::Element; + using AffineElement = typename Curve::AffineElement; + using ScalarField = typename Curve::ScalarField; -// Total weight assigned to a thread (sum of WorkUnit sizes weighted by the input vector). -size_t thread_weight(const std::vector& units, const std::vector>& weights) -{ - size_t total = 0; - for (const auto& u : units) { - for (size_t k = 0; k < u.size; ++k) { - total += weights[u.batch_msm_index][u.start_index + k]; - } + static AffineElement naive_msm(std::span input_scalars, std::span input_points) + { + return ScalarMultiplicationTest::naive_msm(input_scalars, input_points); } - return total; -} -} // namespace + static std::vector make_points(size_t n) + { + std::vector pts(n); + parallel_for_range(n, [&](size_t s, size_t e) { + for (size_t i = s; i < e; ++i) { + pts[i] = Group::one * Curve::ScalarField::random_element(&engine); + } + }); + return pts; + } -TEST(PartitionByWeight, NoMsmsReturnsEmptyThreads) -{ - auto units = PartitionMSM::partition_by_weight({}, 8); - ASSERT_EQ(units.size(), 8U); - for (const auto& t : units) { - EXPECT_TRUE(t.empty()); + static ScalarField scalar_below_2pow(size_t bits) + { + // Random scalar with canonical-form msb < `bits`. We pull a random ScalarField + // (Montgomery), reduce to canonical, mask the canonical representation, and + // reconstruct via the canonical-uint256_t constructor (which re-Montgomery-encodes). + // Masking the .data field directly would mask the Montgomery form, producing garbage. + if (bits >= 254) { + return ScalarField::random_element(&engine); + } + ScalarField r = ScalarField::random_element(&engine); + ScalarField canonical = r.from_montgomery_form_reduced(); + auto& d = canonical.data; + size_t bits_remaining = bits; + for (size_t l = 0; l < 4; ++l) { + const size_t take = std::min(64, bits_remaining); + const uint64_t mask = (take == 64) ? ~uint64_t{ 0 } + : (take == 0) ? uint64_t{ 0 } + : ((uint64_t{ 1 } << take) - 1); + d[l] &= mask; + if (bits_remaining > take) { + bits_remaining -= take; + } else { + bits_remaining = 0; + } + } + return ScalarField(uint256_t(d[0], d[1], d[2], d[3])); } -} -TEST(PartitionByWeight, AllEmptyMsmsReturnsEmptyThreads) -{ - std::vector> weights{ {}, {}, {} }; - auto units = PartitionMSM::partition_by_weight(weights, 4); - ASSERT_EQ(units.size(), 4U); - for (const auto& t : units) { - EXPECT_TRUE(t.empty()); + static void check_against_naive(std::span scalars, std::span points) + { + AffineElement expected = naive_msm(scalars, points); + AffineElement actual = scalar_multiplication::MSM::msm(points, PolynomialSpan(0, scalars)); + EXPECT_EQ(actual, expected); } -} -TEST(PartitionByWeight, SingleThreadGetsEverything) -{ - std::vector> weights{ { 5, 5, 5, 5, 5 } }; - auto units = PartitionMSM::partition_by_weight(weights, 1); - ASSERT_EQ(units.size(), 1U); - ASSERT_EQ(units[0].size(), 1U); - EXPECT_EQ(units[0][0].batch_msm_index, 0U); - EXPECT_EQ(units[0][0].start_index, 0U); - EXPECT_EQ(units[0][0].size, 5U); -} + static constexpr size_t kN = 131072; -TEST(PartitionByWeight, EvenSplitAcrossThreads) -{ - // 8 weights of 5 => total 40, target 10 per thread (4 threads), so 2 weights per thread. - std::vector> weights{ { 5, 5, 5, 5, 5, 5, 5, 5 } }; - auto units = PartitionMSM::partition_by_weight(weights, 4); - ASSERT_EQ(units.size(), 4U); - for (size_t t = 0; t < 4; ++t) { - ASSERT_EQ(units[t].size(), 1U) << "thread " << t; - EXPECT_EQ(units[t][0].size, 2U) << "thread " << t; - EXPECT_EQ(thread_weight(units[t], weights), 10U) << "thread " << t; + void test_cliff() + { + // All scalars < 2^30 plus 16 large scalars (full 254-bit). SPLIT must fire. + constexpr size_t large_count = 16; + auto pts = make_points(kN); + std::vector ss(kN); + for (size_t i = 0; i < kN - large_count; ++i) { + ss[i] = scalar_below_2pow(30); + } + for (size_t i = kN - large_count; i < kN; ++i) { + ss[i] = ScalarField::random_element(&engine); + } + check_against_naive(ss, pts); } -} -TEST(PartitionByWeight, HeavyFirstWeightClosesFirstThreadEarly) -{ - // First weight alone exceeds the per-thread target; remainder is evenly split. - std::vector> weights{ { 100, 5, 5, 5, 5 } }; - auto units = PartitionMSM::partition_by_weight(weights, 4); - ASSERT_EQ(units.size(), 4U); - // Thread 0 should close after the heavy weight. - ASSERT_FALSE(units[0].empty()); - EXPECT_EQ(units[0][0].start_index, 0U); - EXPECT_EQ(units[0][0].size, 1U); - // Total assigned across all threads must equal n. - size_t total_assigned = 0; - for (const auto& t : units) { - for (const auto& u : t) { - total_assigned += u.size; + void test_decaying() + { + // Half below-128 + half below-160. + auto pts = make_points(kN); + std::vector ss(kN); + for (size_t k = 0; k < kN / 2; ++k) { + ss[k] = scalar_below_2pow(128); + } + for (size_t k = kN / 2; k < kN; ++k) { + ss[k] = scalar_below_2pow(160); } + check_against_naive(ss, pts); } - EXPECT_EQ(total_assigned, 5U); -} -TEST(PartitionByWeight, BoundaryStraddlesMsm) -{ - // Two MSMs of 4 weights of 5 each => total 40, 4 threads, target 10. - // Boundary should land mid-MSM if weights cross between MSMs. - std::vector> weights{ { 5, 5, 5, 5 }, { 5, 5, 5, 5 } }; - auto units = PartitionMSM::partition_by_weight(weights, 4); - ASSERT_EQ(units.size(), 4U); - size_t total_assigned = 0; - for (const auto& t : units) { - for (const auto& u : t) { - total_assigned += u.size; + void test_uniform_random() + { + // Standard random scalars — must hit the NO_SPLIT fall-through. + auto pts = make_points(kN); + std::vector ss(kN); + for (size_t k = 0; k < kN; ++k) { + ss[k] = ScalarField::random_element(&engine); } + check_against_naive(ss, pts); } - EXPECT_EQ(total_assigned, 8U); - // Each thread should carry exactly weight 10. - for (size_t t = 0; t < 4; ++t) { - EXPECT_EQ(thread_weight(units[t], weights), 10U) << "thread " << t; + + void test_all_zero() + { + auto pts = make_points(kN); + std::vector ss(kN, ScalarField::zero()); + AffineElement actual = + scalar_multiplication::MSM::msm(pts, PolynomialSpan(0, std::span(ss))); + EXPECT_TRUE(actual.is_point_at_infinity()); } -} -TEST(PartitionByWeight, LastThreadAbsorbsRemainder) -{ - // weights {7,7,1}, num_threads=3 => total 15, target = ceil(15/3) = 5. - // Walk: T0 closes after weight 7, T1 closes after weight 7, then weight 1 trails. - // Without the "current_thread_idx < num_threads - 1" guard the partitioner would - // refuse to close T2 (running weight 1 < target 5) and the trailing weight would - // be lost. The guard makes T2 absorb it via the post-loop push. - std::vector> weights{ { 7, 7, 1 } }; - auto units = PartitionMSM::partition_by_weight(weights, 3); - ASSERT_EQ(units.size(), 3U); - size_t total_assigned = 0; - for (const auto& t : units) { - for (const auto& u : t) { - total_assigned += u.size; + void test_half_zero() + { + // Half zero, half full-random. + auto pts = make_points(kN); + std::vector ss(kN, ScalarField::zero()); + for (size_t k = 0; k < kN / 2; ++k) { + ss[k] = ScalarField::random_element(&engine); } + check_against_naive(ss, pts); } - EXPECT_EQ(total_assigned, 3U); - ASSERT_EQ(units[2].size(), 1U); - EXPECT_EQ(units[2][0].start_index, 2U); - EXPECT_EQ(units[2][0].size, 1U); - EXPECT_EQ(thread_weight(units[2], weights), 1U); -} -TEST(PartitionByWeight, MoreThreadsThanScalars) -{ - // 3 weights of 5 => total 15, 8 threads, target ceil(15/8)=2. - // Each weight (5) immediately crosses target => first 3 threads each get one scalar. - std::vector> weights{ { 5, 5, 5 } }; - auto units = PartitionMSM::partition_by_weight(weights, 8); - ASSERT_EQ(units.size(), 8U); - for (size_t t = 0; t < 3; ++t) { - ASSERT_EQ(units[t].size(), 1U) << "thread " << t; - EXPECT_EQ(units[t][0].size, 1U); + void test_all_large() + { + // Every scalar full-range — NO_SPLIT (Guard A rejects). + auto pts = make_points(kN); + std::vector ss(kN); + for (size_t k = 0; k < kN; ++k) { + ss[k] = ScalarField::random_element(&engine); + } + check_against_naive(ss, pts); + } + + // Synthetic minimal repro for the SPLIT bookkeeping bug: + // half scalars with msb < 64, half full-range. SPLIT may fire (set VAR_WINDOW_FORCE_SPLIT to be sure). + void test_mid_distribution() + { + auto pts = make_points(kN); + std::vector ss(kN); + for (size_t k = 0; k < kN / 2; ++k) { + ss[k] = scalar_below_2pow(60); + } + for (size_t k = kN / 2; k < kN; ++k) { + ss[k] = ScalarField::random_element(&engine); + } + check_against_naive(ss, pts); } - for (size_t t = 3; t < 8; ++t) { - EXPECT_TRUE(units[t].empty()) << "thread " << t; + + // All scalars with canonical msb < 192. Triggers GLV path's regular (non-shortcut) lattice + // reduction for inputs that fit in 192 bits but not 128 — exposing whether scalars + // strictly below the 128-bit shortcut threshold but with non-trivial msb cause a SPLIT + // bookkeeping bug. + void test_below_192() + { + auto pts = make_points(kN); + std::vector ss(kN); + for (size_t k = 0; k < kN; ++k) { + ss[k] = scalar_below_2pow(192); + } + check_against_naive(ss, pts); } + + // Pin-style bitwise-identity check: with VAR_WINDOW_FORCE_SPLIT setting window_bits_lo == window_bits_hi == + // window_bits_unsplit and b_star at a clean multiple of window_bits_unsplit, the SPLIT path's window decomposition + // is structurally identical to NO_SPLIT. Any divergence in the resulting MSM points to a bookkeeping bug + // (per-region driver, schedule layout, idx_large gating in upper region). + void test_force_split_bitwise_identity() + { + auto pts = make_points(kN); + std::vector ss(kN); + for (size_t k = 0; k < kN; ++k) { + ss[k] = scalar_below_2pow(160); + } + check_against_naive(ss, pts); + } +}; + +#ifndef __wasm__ +using VariableWindowCurveTypes = ::testing::Types; +TYPED_TEST_SUITE(VariableWindowSplitDispatchTest, VariableWindowCurveTypes); + +TYPED_TEST(VariableWindowSplitDispatchTest, Cliff) +{ + this->test_cliff(); +} +TYPED_TEST(VariableWindowSplitDispatchTest, Decaying) +{ + this->test_decaying(); +} +TYPED_TEST(VariableWindowSplitDispatchTest, UniformRandom) +{ + this->test_uniform_random(); +} +TYPED_TEST(VariableWindowSplitDispatchTest, AllZero) +{ + this->test_all_zero(); +} +TYPED_TEST(VariableWindowSplitDispatchTest, HalfZero) +{ + this->test_half_zero(); +} +TYPED_TEST(VariableWindowSplitDispatchTest, AllLarge) +{ + this->test_all_large(); +} +TYPED_TEST(VariableWindowSplitDispatchTest, MidDistribution) +{ + this->test_mid_distribution(); +} +TYPED_TEST(VariableWindowSplitDispatchTest, Below192) +{ + this->test_below_192(); +} +TYPED_TEST(VariableWindowSplitDispatchTest, ForceSplitBitwiseIdentity) +{ + this->test_force_split_bitwise_identity(); } +#endif // Non-templated test for explicit small inputs TEST(ScalarMultiplication, SmallInputsExplicit) diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication_fast.cpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication_fast.cpp new file mode 100644 index 000000000000..12d0842f20ed --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication_fast.cpp @@ -0,0 +1,3005 @@ +#include "./scalar_multiplication_fast.hpp" + +#include "./pippenger_arena_layout.hpp" +#include "./pippenger_constantine.hpp" +#include "./pippenger_dedup.hpp" +#include "barretenberg/common/assert.hpp" +#include "barretenberg/common/thread.hpp" +#include "barretenberg/ecc/curves/bn254/bn254.hpp" +#include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" +#include "barretenberg/ecc/groups/element_impl.hpp" +#include "barretenberg/numeric/bitop/get_msb.hpp" +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __wasm_simd128__ +#include +#endif + +namespace bb::scalar_multiplication { + +size_t window_bits_tuning_oversub_factor(size_t n_input) +{ +#ifdef __wasm__ + if (n_input <= (size_t{ 1 } << 11)) { + return 1; + } + if (n_input <= (size_t{ 1 } << 15)) { + return 2; + } + return 4; +#else + static_cast(n_input); + return 4; +#endif +} + +namespace round_parallel_detail { + +// Anonymous namespace gives all TU-private helpers in `round_parallel_detail` internal +// linkage (clang-tidy `misc-use-anonymous-namespace`). It is briefly closed and reopened +// around `pippenger_round_parallel_jacobian_fast`, which has external linkage via +// `extern template` declarations in the header. +namespace { + +// Bulk-copy a 64-byte affine point (BN254 / Grumpkin layout: 8 × uint64_t). +// On wasm, V8 TurboFan compiles the default struct copy to 8 i64 loads/stores; explicit +// v128 loads/stores halve that and roughly double throughput on random-gather access. +// On native, std::memcpy of a constant-size struct already lowers to 4 × movdqu. +template +[[gnu::always_inline]] inline void copy_affine64(AffineElement& dst, const AffineElement& src) noexcept +{ + static_assert(sizeof(AffineElement) == 64, "copy_affine64 requires 64-byte affine point"); + static_assert(std::is_trivially_copyable_v, + "AffineElement must be trivially copyable for memcpy / SIMD bulk copy " + "(also required by the bulk std::memcpy of reduce_chunk output into " + "ThreadScratch::window_pts in recursive_affine_bucket_reduce_strided's caller)"); +#ifdef __wasm_simd128__ + const auto* s = reinterpret_cast(&src); + auto* d = reinterpret_cast(&dst); + const v128_t a = wasm_v128_load(s + 0); + const v128_t b = wasm_v128_load(s + 1); + const v128_t c = wasm_v128_load(s + 2); + const v128_t e = wasm_v128_load(s + 3); + wasm_v128_store(d + 0, a); + wasm_v128_store(d + 1, b); + wasm_v128_store(d + 2, c); + wasm_v128_store(d + 3, e); +#else + std::memcpy(&dst, &src, sizeof(AffineElement)); +#endif +} + +// Constantine signed-Booth window recoder (scalar + SIMD x4 paths) lives in +// pippenger_constantine.hpp. + +// `choose_window_bits` and `build_var_window_schedule` are defined inline in +// `pippenger_arena_layout.hpp` so the test suite can build identical schedules. +// `VAR_WINDOW_MAX_WINDOWS` and `VariableWindowSchedule` likewise live there. + +// Sentinel value for `msb_per_scalar[i]` when scalar i is zero. uint8_t fits the 254 valid msb +// positions (0..253) plus this sentinel; matching `msb_hist` bin layout uses bin 0 = zero count +// so callers index via `msb + 1` (with -1 → bin 0 for the zero case). +inline constexpr uint8_t MSB_ZERO_SENTINEL = 255; + +// Batched-affine drain trigger. `tree_reduce_in_place` accumulates same-bucket pair +// candidates into the per-thread `points_to_add` / `pair_dest` scratch and drains via a +// single inversion + N-pair add when the queue hits this size. Sizing trade-off: +// - higher = larger inversion amortisation = lower per-pair cost, +// - lower = smaller scratch / less L1 pressure but more drain calls. +// 256 was chosen empirically: keeps `points_to_add` (256 × 64 B = 16 KB) inside L1, is +// well above the ~32-pair amortisation breakeven, and is the value the per-OS-thread +// scratch buffers (`points_to_add`, `inversion_scratch`, `pair_dest`) are sized for. +// +// Deliberately a compile-time constant rather than a per-call parameter: the only sites +// that ever passed a different value were chunks shorter than 256, where the early-drain +// branch never fires anyway (the end-of-loop drain catches the residue). Keeping it +// constexpr lets the compiler turn the per-iter `if (pair_count >= BATCH_CAPACITY)` into +// a compare-against-immediate and fold the drain-trigger condition into the loop shape. +// `BATCH_CAPACITY` is defined in `pippenger_arena_layout.hpp` so the layout struct can +// reference it without depending on this TU. + +inline int msb_of_2limb(uint64_t lo, uint64_t hi) noexcept +{ + if (hi != 0) { + return 64 + 63 - __builtin_clzll(hi); + } + if (lo != 0) { + return 63 - __builtin_clzll(lo); + } + return -1; +} + +// Accepts the raw `uint64_t[4]` `.data` of `uint256_t` / field elements directly. +inline int msb_of_4limb(const uint64_t (&d)[4]) noexcept // NOLINT(cppcoreguidelines-avoid-c-arrays) +{ + if (d[3] != 0) { + return 192 + 63 - __builtin_clzll(d[3]); + } + if (d[2] != 0) { + return 128 + 63 - __builtin_clzll(d[2]); + } + if (d[1] != 0) { + return 64 + 63 - __builtin_clzll(d[1]); + } + if (d[0] != 0) { + return 63 - __builtin_clzll(d[0]); + } + return -1; +} + +inline void record_msb(int msb, uint8_t& dst, std::array& th_hist) noexcept +{ + dst = (msb < 0) ? MSB_ZERO_SENTINEL : static_cast(msb); + ++th_hist[static_cast(msb) + 1]; +} + +/** + * @brief Build a uniform window schedule. + */ +// `AffineBucketChunkInfo` is defined in `pippenger_arena_layout.hpp` (included above). + +/** + * @brief Per-thread scratch: VIEWS into the per-MSM_fast arena. Each `std::span` is rebound at + * the start of every `pippenger_round_parallel` call to point into a slice + * of the static arena. The struct never owns storage. + * + * The pippenger_fast function pre-sizes each span large enough for that call's worst + * case so kernels can read/write without bounds checks on the hot path. + */ +template struct ThreadScratch { + using AffineElement = typename Curve::AffineElement; + using Element = typename Curve::Element; + using BaseField = typename Curve::BaseField; + + // reduce_chunk's tree-reduce buffer. Per level the inner loop walks with a read cursor + // `i` and a write cursor `next_len ≤ i`, compacting in-place; the next level re-enters + // the same buffer without a swap. + std::span curr_pts; + std::span curr_buckets; + + // reduce_chunk's batch-affine scratch. + std::span points_to_add; + std::span inversion_scratch; + std::span pair_dest; + + size_t result_len = 0; + + // Stage 6a seam-overflow buffer: when a sub-chunk emits a partial for a slot whose + // dense bucket entry is already populated (i.e. the digit's run was split across two + // sub-chunks), the partial is deferred here and merged at end-of-window via a single + // Montgomery-batched tree reduce. Reset to length 0 between windows. + std::span overflow_slots; + std::span overflow_pts; + size_t overflow_len = 0; + + // Recursive affine bucket reduction scratch (cross-window batched, sparse-aware). + // `dense_buckets` holds W chunks worth of dense AffineElement arrays back-to-back. + // Layout: dense_buckets[w * affine_bucket_stride + i] for window w and 0-indexed slot i. + // `is_present` is a parallel uint8_t array marking non-identity slots (0 = empty, 1 = present). + // `affine_bucket_pairs` is the scratch buffer for the real-pairs list (single pass: filtered + // inline as candidates are generated, no intermediate candidate buffer). + // `affine_bucket_indices` is the scratch index buffer for the doubling kernel. + // `affine_bucket_inversion_scratch` is reused for the indexed batch-affine kernels. + std::span dense_buckets; + std::span is_present; + std::span> affine_bucket_pairs; + std::span affine_bucket_indices; + std::span affine_bucket_inversion_scratch; + size_t affine_bucket_stride = 0; + // Per-window metadata consumed by recursive_affine_bucket_reduce_strided (lo, hi, buckets_padded, + // empty per window). Filled in the lambda before the call. + std::span chunk_infos; +}; + +struct MsmArena { + std::unique_ptr local_owner; // NOLINT(cppcoreguidelines-avoid-c-arrays) + std::byte* data = nullptr; + uintptr_t base_addr = 0; + size_t capacity = 0; + size_t cursor = 0; + + MsmArena(size_t required_bytes, std::span external_arena) + { + if (!external_arena.empty() && required_bytes <= external_arena.size()) { + data = external_arena.data(); + capacity = external_arena.size(); + } else { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) + local_owner = std::make_unique_for_overwrite(required_bytes); + data = local_owner.get(); + capacity = required_bytes; + } + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + base_addr = reinterpret_cast(data); + } + + template std::span alloc(size_t count) { return bump_alloc(count, cursor, capacity, 0); } + + template std::span bump_alloc(size_t count, size_t& local_cursor, size_t bound, size_t base_offset) + { + const size_t align = alignof(T); + const uintptr_t cur_addr = base_addr + base_offset + local_cursor; + const uintptr_t aligned_addr = (cur_addr + align - 1) & ~(uintptr_t{ align } - 1); + const size_t aligned_local = static_cast(aligned_addr - (base_addr + base_offset)); + const size_t bytes = count * sizeof(T); + BB_ASSERT_LTE(aligned_local + bytes, bound); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + T* p = reinterpret_cast(data + base_offset + aligned_local); + local_cursor = aligned_local + bytes; + return std::span{ p, count }; + } +}; + +template inline void drain_batch(ThreadScratch& s, size_t pair_count) noexcept +{ + if (pair_count == 0) { + return; + } + bb::group_elements::batch_affine_add_interleaved( + s.points_to_add.data(), 2 * pair_count, s.inversion_scratch.data()); + // In-place compaction: each `pair_dest[i]` is the `next_len` value at the moment the + // pair was queued, which is < the read cursor `i_outer` and < the current `next_len` + // — so writing back into curr_pts at `pair_dest[i]` lands on a slot that is already + // past the read cursor. See reduce_chunk for the full invariant. + for (size_t i = 0; i < pair_count; ++i) { + s.curr_pts[s.pair_dest[i]] = s.points_to_add[pair_count + i]; + } +} + +/** + * @brief In-place tree reduce on (curr_pts, curr_buckets)[0..initial_len). + * + * Per level, walks the buffer with a read cursor `i` and write cursor + * `next_len ≤ i`, pairing consecutive same-bucket entries and batching their adds via + * Montgomery's trick. The invariant `next_len ≤ i` holds throughout (singletons + * advance both by 1; pairs advance i by 2 and next_len by 1) — so every + * compacted-write `curr_pts[next_len] = curr_pts[i]` and every drain-write at + * `pair_dest[k] = next_len_at_record < i` lands on a slot that has already been + * read this level. After the inner pass, the level's output occupies + * curr_pts[0..next_len) and we re-enter the loop without needing a swap. + * + * On return, `s.result_len` holds the deduplicated count and the surviving + * (point, bucket) pairs occupy curr_pts/curr_buckets[0..result_len). + */ +template void tree_reduce_in_place(ThreadScratch& s, size_t initial_len) noexcept +{ + size_t curr_len = initial_len; + + while (true) { + size_t i = 0; + size_t next_len = 0; + size_t pair_count = 0; + bool made_pair = false; + + while (i < curr_len) { + if (i + 1 < curr_len && s.curr_buckets[i] == s.curr_buckets[i + 1]) { + const size_t slot = 2 * pair_count; + s.points_to_add[slot] = s.curr_pts[i]; + s.points_to_add[slot + 1] = s.curr_pts[i + 1]; + s.curr_buckets[next_len] = s.curr_buckets[i]; + s.pair_dest[pair_count] = static_cast(next_len); + ++next_len; + ++pair_count; + i += 2; + made_pair = true; + + if (pair_count >= BATCH_CAPACITY) { + drain_batch(s, pair_count); + pair_count = 0; + } + } else { + s.curr_pts[next_len] = s.curr_pts[i]; + s.curr_buckets[next_len] = s.curr_buckets[i]; + ++next_len; + ++i; + } + } + + drain_batch(s, pair_count); + + if (!made_pair) { + break; + } + + curr_len = next_len; + } + + s.result_len = curr_len; +} + +/** + * @brief Merge per-thread Stage 6a seam-overflow partials back into the per-window dense + * bucket buffer. + * + * When a sub-chunk emits a partial for a slot whose dense entry is already + * populated (the slot's bucket run was split across sub-chunks), the partial is + * appended to (overflow_slots, overflow_pts) instead of overwriting. This routine + * builds a tree-reduce input by, for each unique slot, emitting the existing + * dense entry as the head followed by all overflow partials for that slot, then + * runs `tree_reduce_in_place` once across all affected slots — sharing one + * Montgomery batch per tree level instead of one inversion per partial. + * + * Overflow entries arrive in slot-sorted order by construction (sub-chunks process + * buckets in sorted order), so no sort is needed. On return, `overflow_len` is + * reset to 0. + */ +template +void merge_overflow(ThreadScratch& s, typename Curve::AffineElement* dst_dense) noexcept +{ + if (s.overflow_len == 0) { + return; + } + + size_t merge_len = 0; + size_t i = 0; + while (i < s.overflow_len) { + const uint32_t slot = s.overflow_slots[i]; + s.curr_buckets[merge_len] = slot; + s.curr_pts[merge_len] = dst_dense[slot]; + ++merge_len; + while (i < s.overflow_len && s.overflow_slots[i] == slot) { + s.curr_buckets[merge_len] = slot; + s.curr_pts[merge_len] = s.overflow_pts[i]; + ++merge_len; + ++i; + } + } + + tree_reduce_in_place(s, merge_len); + + for (size_t k = 0; k < s.result_len; ++k) { + dst_dense[s.curr_buckets[k]] = s.curr_pts[k]; + } + + s.overflow_len = 0; +} + +/** + * @brief Tree-reduce one thread's bucket-aligned slice of the bucket-partitioned schedule, + * emitting directly into `curr_pts / curr_buckets` for the caller's running-sum pass. + */ +template +void reduce_chunk(ThreadScratch& s, + const uint32_t* schedule, + const size_t* bucket_start, + size_t chunk_lo, + size_t chunk_hi, + size_t& bucket_cursor, + size_t chunk_bucket_hi, + std::span points, + std::span dedup_extra_points = {}) noexcept +{ + const size_t chunk_len = chunk_hi - chunk_lo; + if (chunk_len == 0) { + s.result_len = 0; + return; + } + + BB_ASSERT_LTE(chunk_len, s.curr_pts.size()); + static_assert(BATCH_CAPACITY <= 4096, "BATCH_CAPACITY must fit in pair_dest scratch"); + + // Compact entries while loading: dedup non-rep entries (DEDUP_SKIP_BIT set in the + // schedule entry) carry no contribution — their points are already accumulated + // into the cluster's combined `extra_points[cid]` emitted at the rep's slot. Skip + // them to avoid double-counting and to shrink the tree-reduce input. + size_t valid_len = 0; + size_t bucket = bucket_cursor; + size_t pos = chunk_lo; + while (bucket <= chunk_bucket_hi && pos < chunk_hi) { + const size_t run_lo = std::max(pos, bucket_start[bucket]); + const size_t run_hi = std::min(chunk_hi, bucket_start[bucket + 1]); + if (run_lo >= run_hi) { + ++bucket; + continue; + } + + const uint32_t bucket_u32 = static_cast(bucket); + for (size_t i = run_lo; i < run_hi; ++i) { + const uint32_t e = schedule[i]; + if ((e & DEDUP_SKIP_BIT) != 0) { + continue; // non-rep: skip, don't consume a curr_pts slot + } + const uint32_t raw_idx = e & SCHEDULE_INDEX_MASK; + const bool neg = (e & SCHEDULE_SIGN_BIT) != 0; + s.curr_buckets[valid_len] = bucket_u32; + // SIMD-widened gather: 4 × v128.load on WASM (2× faster than the + // default 8 × i64.load struct copy on V8 TurboFan); 4 × movdqu on + // native (already optimal). The conditional negation runs after the + // copy because Fq::operator-() is a modular subtract, not a bit flip, + // so it can't be folded into the SIMD load lanes. + auto& dst_pt = s.curr_pts[valid_len]; + // Dedup redirect: if the redirect bit is set, fetch from the dedup + // extra-points buffer (combined point for a cluster of duplicate scalars) + // instead of the original points span. The branch is always-not-taken when + // dedup is inactive (`dedup_extra_points` empty) and predictably-mostly-taken-or-not + // when active, since cluster-rep scheduling is uniform per MSM_fast. + if ((e & DEDUP_REDIRECT_BIT) != 0) { + copy_affine64(dst_pt, dedup_extra_points[raw_idx]); + } else { + copy_affine64(dst_pt, points[raw_idx]); + } + if (neg) { + dst_pt.y = -dst_pt.y; + } + ++valid_len; + } + pos = run_hi; + if (pos < chunk_hi) { + ++bucket; + } + } + bucket_cursor = bucket; + + tree_reduce_in_place(s, valid_len); +} + +// `ChunkOutput` (Stage 6 per-chunk bucket-reduce output) is defined in +// `pippenger_arena_layout.hpp` so the test suite can size the Zone S slot the +// same way the live allocator does. + +// `AffineBucketChunkInfo` is defined in `pippenger_arena_layout.hpp` (forward declared +// above at line ~674 for ThreadScratch). It describes one chunk's contribution to the +// cross-window recursive affine bucket reduction (lo/hi digit bounds, buckets_padded, +// empty flag). + +/** + * @brief Inline filter for one (dst, src) candidate pair, called from each phase's + * candidate-emission loop. Replaces the previous two-pass `filter_and_batch_add` + * function (which gathered all candidates into a buffer and then filtered into a + * real-pairs buffer): the inline version handles each candidate as it's generated, + * so we never materialise the candidate buffer at all. Single memory pass per phase + * iter, half the buffer scratch. + * + * Identity / coincidence cases per candidate (handled inline, no batch dispatch): + * - src is identity: skip entirely (dst unchanged). + * - dst is identity, src is not: copy src into dst (set is_present[dst]=1). No add issued. + * - dst and src share the same affine point (Phase A's "copy through empty slots" can + * leave two adjacent slots with identical values): issue a Jacobian doubling out-of-band. + * - dst and src are inverses (same x, opposite y): result is identity; clear dst. + * Otherwise emits a "real pair" to `real_pairs[*real_count]` for later batch-affine dispatch. + * + * Caller is responsible for invoking `batch_affine_add_indexed_impl` once on the accumulated + * `real_pairs` array after the candidate-emission loop completes. + */ +template +[[gnu::always_inline]] inline void try_filter_pair(typename Curve::AffineElement* buckets, + uint8_t* is_present, + uint32_t dst_idx, + uint32_t src_idx, + std::pair* real_pairs, + size_t& real_count) noexcept +{ + using Element = typename Curve::Element; + using AffineElement = typename Curve::AffineElement; + + if (is_present[src_idx] == 0) { + return; // src is identity → no-op + } + if (is_present[dst_idx] == 0) { + buckets[dst_idx] = buckets[src_idx]; // dst was identity → just copy + is_present[dst_idx] = 1; + return; + } + // Edge case: dst.x == src.x. Since both points are on-curve, this means either + // dst == src (doubling case) or dst == -src (inverse case, result is identity). + // batch_affine_add_indexed_impl would invert zero here, so handle out-of-band. + if (buckets[dst_idx].x == buckets[src_idx].x) { + if (buckets[dst_idx].y == buckets[src_idx].y) { + // dst == src → result is 2 * dst. + Element doubled = Element(buckets[dst_idx]); + doubled.self_dbl(); + buckets[dst_idx] = AffineElement{ doubled }; + } else { + // dst == -src → result is identity. + buckets[dst_idx].self_set_infinity(); + is_present[dst_idx] = 0; + } + return; + } + real_pairs[real_count++] = { dst_idx, src_idx }; +} + +/** + * @brief Inline filter for one doubling candidate. If the slot is populated, append its + * index to `real_indices`; otherwise skip silently. The caller invokes + * `batch_affine_double_indexed_impl` on the accumulated `real_indices` array. + */ +[[gnu::always_inline]] inline void try_filter_idx(const uint8_t* is_present, + uint32_t idx, + uint32_t* real_indices, + size_t& real_count) noexcept +{ + if (is_present[idx] != 0) { + real_indices[real_count++] = idx; + } +} + +/** + * @brief Recursive affine bucket reduction (Mitschabaude et al., ZPrize 2022). Pairs + * adjacent non-empty buckets and applies one batched-affine round per recursion + * level; the per-thread cross-window bucket accumulation step. + * + * For each chunk c (one per window in the batch), computes + * R_c = Σ_d B_{c,d} (simple sum) + * L_c = Σ_d (d - lo_c + 1) · B_{c,d} (weighted sum) + * and writes them into `outputs[w].R / .L`. The caller's `chunk_contribution` then + * recovers Σ_d d · B_{c,d} as `L_c + (lo_c - 1) · R_c`. + * + * Algorithm — 4 phases: + * A: per-sub-partition suffix sums. Slot `d*L0 + i` ends up holding + * Σ_{j=i..L0-1} buckets[w][d*L0 + j]. + * B: log-recombine sub-partition sums (slot 0, L0, 2L0, ...) into slot 0, + * producing R_c. R_c is captured to outputs at this point. + * C: doubling pass. Each slot d*L0 (d ≥ 1) is doubled (c0 + level_of_d) times, + * where level_of_d is the highest j with 2^j·L0 | d*L0. Combined with the + * suffix-sum structure from A, this primes phase D. + * D: flat tree-add over the buckets_padded slots; slot 0 ends up holding L_c. + * + * The is_present[] mask filters identity slots out of every batch-affine call, so + * sparse chunks don't waste inversions on dead points. Pairs from all windows_in_batch + * chunks share the same batch-affine inversion at every phase step (with D + * sub-partitions, phase A batches up to windows_in_batch · D pairs). + * + * @param s per-thread scratch buffers. + * @param chunk_infos per-window metadata (lo, hi, buckets_padded, empty); caller-set. + * @param windows_in_batch number of chunks in this batch. + * @param outputs_base per-window output cells (strided by output_stride); .R / .L are + * written in place. + * + * @note Caller must have densified buckets at `s.dense_buckets[w*stride + i]`, set + * is_present[w*stride + i] for populated slots, and called + * s.ensure_affine_bucket_capacity(windows_in_batch, stride) with + * stride = max_w(buckets_padded_w). + */ +template +void recursive_affine_bucket_reduce_strided(ThreadScratch& s, + const AffineBucketChunkInfo* chunk_infos, + size_t windows_in_batch, + ChunkOutput* outputs_base, + size_t output_stride) noexcept +{ + using AffineElement = typename Curve::AffineElement; + using Element = typename Curve::Element; + + auto out_at = [outputs_base, output_stride](size_t w) -> ChunkOutput& { + return outputs_base[w * output_stride]; + }; + + if (windows_in_batch == 0) { + return; + } + + // Stride is the caller's pre-sized layout width (`s.affine_bucket_stride`, set via + // `ensure_affine_bucket_capacity`). The densification step in the caller scattered buckets at + // `w * s.affine_bucket_stride + i`, so we MUST use the same value for our own indexing — any + // re-derivation that disagrees with the layout would index neighbouring windows. The + // pre-size already enforces `stride ≥ max_w(buckets_padded_w)` AND `stride ≥ 2` AND + // `stride is a power of two`, so the trivial-stride fast path and the 4-phase math + // both stay valid here. Per-window buckets_padded controls how many slots each window walks + // and is bounded by `stride` — verified below in debug. + const size_t stride = s.affine_bucket_stride; + bool any_nonempty = false; + for (size_t w = 0; w < windows_in_batch; ++w) { + if (chunk_infos[w].empty == 0) { + any_nonempty = true; + BB_ASSERT_LTE(chunk_infos[w].buckets_padded, stride); + } + } + if (!any_nonempty) { + for (size_t w = 0; w < windows_in_batch; ++w) { + out_at(w).R = Curve::Group::point_at_infinity; + out_at(w).L = Curve::Group::point_at_infinity; + } + return; + } + + AffineElement* const buckets = s.dense_buckets.data(); + uint8_t* const is_present = s.is_present.data(); + + // Pick L0 (the leaf-partition size). c0 = floor(log2(stride) / 2) + // gives L0 ≈ sqrt(stride) — balances Phase A batch size (W·D) vs Phase A iter count + // (L0 - 1). Both L0 and D = stride / L0 must be powers of two. + BB_ASSERT_GT(stride, size_t{ 0 }); + const size_t c_log = static_cast(std::countr_zero(stride)); + BB_ASSERT_EQ(static_cast(1) << c_log, stride); + // Trivial-stride fast paths. The 4-phase algorithm requires c_log ≥ 2 (so we can pick + // c0 ∈ [1, c_log - 1]) — fall back to direct computation for stride ∈ {1, 2}. + if (stride <= 2) { + for (size_t w = 0; w < windows_in_batch; ++w) { + if (chunk_infos[w].empty != 0) { + out_at(w).R = Curve::Group::point_at_infinity; + out_at(w).L = Curve::Group::point_at_infinity; + continue; + } + // Walk the (up to two) populated slots directly. + const size_t base = w * stride; + Element R = Curve::Group::point_at_infinity; + Element L = Curve::Group::point_at_infinity; + for (size_t i = 0; i < chunk_infos[w].buckets_padded; ++i) { + if (is_present[base + i] == 0) { + continue; + } + R += Element(buckets[base + i]); + L += Element(buckets[base + i]); // weight 1 + if (i == 1) { + L += Element(buckets[base + i]); // weight 2 for i=1 + } + } + out_at(w).R = R; + out_at(w).L = L; + } + return; + } + + // Choose c0 = floor(c_log / 2), clamped so that 1 ≤ c0 ≤ c_log - 1. + size_t c0 = c_log / 2; + if (c0 == 0) { + c0 = 1; + } + if (c0 >= c_log) { + c0 = c_log - 1; + } + const size_t L0 = static_cast(1) << c0; + const size_t D = stride >> c0; // == stride / L0 + BB_ASSERT_EQ(L0 * D, stride); + BB_ASSERT_GTE(L0, size_t{ 2 }); + BB_ASSERT_GTE(D, size_t{ 2 }); + + auto* const reals = s.affine_bucket_pairs.data(); + auto* const dbl_reals = s.affine_bucket_indices.data(); + auto* const inv_scratch = s.affine_bucket_inversion_scratch.data(); + + // Phase A: per-sub-partition running-sum (suffix sums). + // For each window w and each sub-partition d, walk slots from L0-1 down to 1 within the + // sub-partition, accumulating buckets[w*stride + d*L0 + l - 1] += buckets[... l]. All + // (w, d, l) triples for a fixed l share one batch-affine inversion (up to windows_in_batch + // · D pairs). Short windows (my_M_w < L0) are treated as a single sub-partition of length + // my_M_w to skip dead candidates; effective per-(w, d) length is min(L0, my_M_w - d·L0). + { + for (size_t l = L0 - 1; l >= 1; --l) { + size_t real_count = 0; + for (size_t w = 0; w < windows_in_batch; ++w) { + if (chunk_infos[w].empty != 0) { + continue; + } + const size_t my_M_w = chunk_infos[w].buckets_padded; + const size_t base = w * stride; + if (my_M_w < L0) { + // Short window: single sub-partition of effective length `my_M_w`. + if (l >= my_M_w) { + continue; // l is in the empty-padding region, skip + } + const uint32_t src = static_cast(base + l); + const uint32_t dst = static_cast(base + l - 1); + try_filter_pair(buckets, is_present, dst, src, reals, real_count); + } else { + const size_t my_D = my_M_w >> c0; // ≥ 1 + for (size_t d = 0; d < my_D; ++d) { + const uint32_t src = static_cast(base + (d * L0) + l); + const uint32_t dst = static_cast(base + (d * L0) + l - 1); + try_filter_pair(buckets, is_present, dst, src, reals, real_count); + } + } + } + if (real_count > 0) { + bb::group_elements::batch_affine_add_indexed_impl( + buckets, reals, real_count, inv_scratch); + } + } + } + + // After Phase A, each window's slot 0 holds the simple sum of its sub-partition 0, + // and slot d*L0 (d ≥ 1) holds the simple sum of sub-partition d. The other slots within + // each sub-partition hold suffix sums that Phase D will combine. + + // Phase B: log-recombine sub-partition simple sums into slot 0. + // For L1 = L0, 2*L0, 4*L0, ..., stride/2: pair (slot 2d*L1, slot (2d+1)*L1). + { + size_t L1 = L0; + while (L1 < stride) { + size_t real_count = 0; + const size_t step = 2 * L1; + for (size_t w = 0; w < windows_in_batch; ++w) { + if (chunk_infos[w].empty != 0) { + continue; + } + const size_t my_M = chunk_infos[w].buckets_padded; + if (step > my_M) { + continue; + } + const size_t base = w * stride; + const size_t num_pairs_w = my_M / step; + for (size_t d = 0; d < num_pairs_w; ++d) { + const uint32_t dst = static_cast(base + ((2 * d) * L1)); + const uint32_t src = static_cast(base + (((2 * d) + 1) * L1)); + try_filter_pair(buckets, is_present, dst, src, reals, real_count); + } + } + if (real_count > 0) { + bb::group_elements::batch_affine_add_indexed_impl( + buckets, reals, real_count, inv_scratch); + } + L1 *= 2; + } + } + + // After Phase B, each window's slot 0 holds Σ_d B_{c,d} = R_c. Save R_c into outputs + // before Phase D's tree-add overwrites slot 0. + for (size_t w = 0; w < windows_in_batch; ++w) { + if (chunk_infos[w].empty != 0) { + out_at(w).R = Curve::Group::point_at_infinity; + continue; + } + const AffineElement& slot0 = buckets[w * stride]; + if (is_present[w * stride] == 0) { + out_at(w).R = Curve::Group::point_at_infinity; + } else { + out_at(w).R = Element(slot0); + } + } + + // Phase C: doublings. + // The candidate index list for the initial pass is constant across all c0 iters — + // every slot d*L0 for d ∈ [1, my_D - 1] in every non-empty window. Build the empty- + // filtered list once and chain c0 doublings on it instead of filtering c0 times. + // Subsequent levels (L1 = 2*L0, 4*L0, ...) do one doubling per level on level-specific + // index sets handled separately below. + { + size_t real_count = 0; + for (size_t w = 0; w < windows_in_batch; ++w) { + if (chunk_infos[w].empty != 0) { + continue; + } + const size_t my_M_w = chunk_infos[w].buckets_padded; + const size_t my_D = (my_M_w >= L0) ? (my_M_w >> c0) : size_t{ 0 }; + const size_t base = w * stride; + for (size_t d = 1; d < my_D; ++d) { + try_filter_idx(is_present, static_cast(base + (d * L0)), dbl_reals, real_count); + } + } + // c0 chained doublings on the same real list. + if (real_count > 0) { + for (size_t j = 0; j < c0; ++j) { + bb::group_elements::batch_affine_double_indexed_impl( + buckets, dbl_reals, real_count, inv_scratch); + } + } + } + // Successive: at L1 = 2*L0, 4*L0, ..., stride/2: every d ≥ 1 in the sub-partition + // grid of size `stride / L1` gets one more doubling. + { + size_t L1 = 2 * L0; + while (L1 < stride) { + size_t real_count = 0; + for (size_t w = 0; w < windows_in_batch; ++w) { + if (chunk_infos[w].empty != 0) { + continue; + } + const size_t my_M = chunk_infos[w].buckets_padded; + if (L1 >= my_M) { + continue; // this window has no sub-partitions at this hierarchy + } + const size_t my_D1 = my_M / L1; + const size_t base = w * stride; + for (size_t d = 1; d < my_D1; ++d) { + try_filter_idx(is_present, static_cast(base + (d * L1)), dbl_reals, real_count); + } + } + if (real_count > 0) { + bb::group_elements::batch_affine_double_indexed_impl( + buckets, dbl_reals, real_count, inv_scratch); + } + L1 *= 2; + } + } + + // Phase D: flat tree-add over the buckets_padded slots. For m = 1, 2, 4, ..., + // buckets_padded/2: pair (slot pos, slot pos+m) for pos = 0, 2m, 4m, ... + // Once the level's candidate count drops below BATCH_AFFINE_BREAKEVEN, the per-batch + // inversion overhead exceeds the projective per-add cost; bail and finish in Jacobian. + constexpr size_t BATCH_AFFINE_BREAKEVEN = 32; + size_t m = 1; + while (m < stride) { + // Live-slot count after this iter: stride / (2m) per window worst-case. + // Decision: would this iter's batch be too small? Estimate as + // `windows_in_batch * stride / (2m)` (upper bound on candidates). + const size_t est_cands_this_iter = windows_in_batch * (stride / (2 * m)); + if (est_cands_this_iter < BATCH_AFFINE_BREAKEVEN) { + break; + } + size_t real_count = 0; + const size_t step = 2 * m; + for (size_t w = 0; w < windows_in_batch; ++w) { + if (chunk_infos[w].empty != 0) { + continue; + } + const size_t my_M = chunk_infos[w].buckets_padded; + if (m >= my_M) { + continue; + } + const size_t base = w * stride; + for (size_t pos = 0; pos + m < my_M; pos += step) { + try_filter_pair(buckets, + is_present, + static_cast(base + pos), + static_cast(base + pos + m), + reals, + real_count); + } + } + if (real_count > 0) { + bb::group_elements::batch_affine_add_indexed_impl( + buckets, reals, real_count, inv_scratch); + } + m *= 2; + } + + // Write L_c. After Phase D's loop, `m` is the level NOT performed (or `stride` if all + // levels ran). The "live" slots — those holding cumulative tree-sums of consecutive m + // original buckets each — are {0, m, 2m, 3m, ...} ∩ [0, my_M): + // - loop completed (m == stride): only slot 0 is live; it holds the final L. + // - loop broke at level m: sum the live slots in Jacobian (live_step = m). + // - loop broke at m == 1: every original bucket is still live, sum them all. + // The Jacobian sum recovers what the unfinished levels would have computed in the + // batch-affine inner loop. + for (size_t w = 0; w < windows_in_batch; ++w) { + if (chunk_infos[w].empty != 0) { + out_at(w).L = Curve::Group::point_at_infinity; + continue; + } + const size_t base = w * stride; + const size_t my_M = chunk_infos[w].buckets_padded; + Element L = Curve::Group::point_at_infinity; + const size_t live_step = m; // distance between live slots after the affine phase + for (size_t pos = 0; pos < my_M; pos += live_step) { + if (is_present[base + pos] != 0) { + L += Element(buckets[base + pos]); + } + } + out_at(w).L = L; + } +} + +/** + * @brief Single chunk's contribution to its window's weighted bucket sum. + * + * The local Horner pre-bakes weight `(d - lo_t + 1)` onto each bucket-d in chunk t's `L`. + * Adding the global offset `(lo_t - 1) · R_t` then yields total weight `d` on bucket d, i.e. + * exactly the window's contribution restricted to the chunk. The window sum is therefore a + * trivial reduction `Σ_t (L_t + (lo_t - 1) · R_t)` over non-empty chunks — no inter-chunk + * suffix sum needed. (factor · R) uses double-and-add since `lo_t - 1 < 2^c ≤ 2^20`. + * + * Algebraic proof: substituting `H_i = total_running − Σ_{s +[[gnu::always_inline]] inline typename Curve::Element chunk_contribution(const ChunkOutput& chunk) noexcept +{ + using Element = typename Curve::Element; + if (chunk.empty != 0) { + return Curve::Group::point_at_infinity; + } + const uint32_t k = chunk.lo - 1; + Element acc = chunk.L; + if (k != 0) { + Element p = chunk.R; + uint32_t kk = k; + while (kk != 0) { + if ((kk & 1U) != 0) { + acc += p; + } + kk >>= 1; + if (kk != 0) { + p.self_dbl(); + } + } + } + return acc; +} + +} // namespace +// `pippenger_round_parallel_jacobian_fast` has external linkage via the `extern template` +// declarations in the header (used by the batched driver). Defined at namespace scope. + +/** + * @brief Small-N fast-path: per-thread Jacobian Pippenger over a partition of the input. + * + * Bypasses the round-parallel scaffolding (biased recoding, count histogram, prefix sum, + * scatter, partition, recursive affine bucket reduction) entirely. Caller must have already + * converted `scalars` from Montgomery form to standard form. Each thread runs a textbook + * Pippenger over its slice of the input, with the result summed across threads at the end. + * + * Per round (high-bit slice → low-bit slice): + * - Reset `present` bitmap. + * - For each point in the thread's range, extract the window_bits-wide scalar slice; if + * non-zero, either ASSIGN the bucket (Z = 1) on first hit or `Element += AffineElement` + * (mixed Jacobian-affine, 7M+4S, no inversion) on subsequent hits. + * - Running suffix sum over populated buckets only. + * - Double the running result by `window_bits` bits (or `remainder` for the last round) + * and add the bucket sum. + * + * No batched-affine path, no modular inversions, no count-sort. + */ +template +[[gnu::noinline]] typename Curve::Element pippenger_round_parallel_jacobian_fast( + std::span scalars, + std::span points, + size_t min_pts_per_thread_override) noexcept +{ + using Element = typename Curve::Element; + using ScalarField = typename Curve::ScalarField; + using BaseField = typename Curve::BaseField; + + const size_t n = scalars.size(); + if (n == 0) { + return Curve::Group::point_at_infinity; + } + + constexpr size_t NUM_BITS = ScalarField::modulus.get_msb() + 1; + + // Cost-model window-size selection (mirrors MSM_fast::get_optimal_log_num_buckets, + // with BUCKET_ACCUMULATION_COST = 5 = J-J-add-equiv-muls / J-A-add-equiv-muls ≈ 16/11 + // rounded up). We do NOT delegate to the public method — keeping it self-contained + // avoids dragging the AffineAddition / AFFINE_TRICK_THRESHOLD machinery in here. + constexpr size_t BUCKET_ACCUMULATION_COST = 5; + constexpr uint32_t MAX_C = 18; + auto cost = [n](uint32_t bits) -> size_t { + size_t rounds = (NUM_BITS + bits - 1) / bits; + size_t buckets = size_t{ 1 } << bits; + return rounds * (n + buckets * BUCKET_ACCUMULATION_COST); + }; + uint32_t window_bits = 1; + size_t best_cost = cost(1); + for (uint32_t b = 2; b <= MAX_C; ++b) { + const size_t this_cost = cost(b); + if (this_cost < best_cost) { + best_cost = this_cost; + window_bits = b; + } + } + const size_t num_buckets = size_t{ 1 } << window_bits; + const uint32_t num_rounds = static_cast((NUM_BITS + window_bits - 1) / window_bits); + const uint32_t last_round_bits = + static_cast(NUM_BITS - (static_cast(num_rounds - 1) * window_bits)); + + // Each thread owns a num_buckets-sized scratch slice and runs num_rounds passes; below + // ~256 points per thread the parallel_for wakeup + per-call bucket reset dominate. + // wasm is forced single-threaded — its barrier cost is much higher than native. +#ifdef __wasm__ + constexpr size_t MIN_PTS_PER_THREAD_DEFAULT = SIZE_MAX; +#else + constexpr size_t MIN_PTS_PER_THREAD_DEFAULT = 256; +#endif + const size_t MIN_PTS_PER_THREAD = + (min_pts_per_thread_override == 0) ? MIN_PTS_PER_THREAD_DEFAULT : min_pts_per_thread_override; + const size_t max_threads = get_num_cpus(); + size_t num_threads = std::min(std::max(1, n / MIN_PTS_PER_THREAD), max_threads); + if (num_threads == 0) { + num_threads = 1; + } + + // Allocate the per-thread bucket + presence scratch ONCE, indexed by tid inside the + // parallel_for. Allocating inside the lambda body would re-malloc on every call (and + // on WASM the malloc cost is non-trivial relative to the arithmetic work at small n). + std::vector per_thread_results(num_threads); + std::vector all_buckets(num_threads * num_buckets); + std::vector all_present(num_threads * num_buckets); + + auto thread_body = [&](size_t tid) { + BB_BENCH_NAME("MSM_fast::jacobian_fast/worker"); + const size_t lo = (tid * n) / num_threads; + const size_t hi = ((tid + 1) * n) / num_threads; + + Element* const buckets = all_buckets.data() + (tid * num_buckets); + uint8_t* const present = all_present.data() + (tid * num_buckets); + + Element result = Curve::Group::point_at_infinity; + + for (uint32_t round = 0; round < num_rounds; ++round) { + std::memset(present, 0, num_buckets); + + const size_t hi_bit = NUM_BITS - (static_cast(round) * window_bits); + const size_t lo_bit = (hi_bit < window_bits) ? size_t{ 0 } : (hi_bit - window_bits); + const size_t actual_size = hi_bit - lo_bit; + const size_t start_limb = lo_bit >> 6; + const size_t end_limb = hi_bit >> 6; + const size_t lo_off = lo_bit & 63; + const size_t lo_bits = (64 - lo_off < actual_size) ? (64 - lo_off) : actual_size; + const size_t hi_bits = actual_size - lo_bits; + const uint64_t lo_mask = (lo_bits == 64) ? ~uint64_t{ 0 } : ((uint64_t{ 1 } << lo_bits) - 1); + const uint64_t hi_mask = (hi_bits == 0) ? uint64_t{ 0 } : ((uint64_t{ 1 } << hi_bits) - 1); + + for (size_t i = lo; i < hi; ++i) { + const uint64_t s_lo = (scalars[i].data[start_limb] >> lo_off) & lo_mask; + const uint64_t s_hi = (start_limb != end_limb) ? (scalars[i].data[end_limb] & hi_mask) : uint64_t{ 0 }; + const uint32_t slice = static_cast(s_lo | (s_hi << lo_bits)); + if (slice == 0) { + continue; + } + if (present[slice] == 0) { + buckets[slice].x = points[i].x; + buckets[slice].y = points[i].y; + buckets[slice].z = BaseField::one(); + present[slice] = 1; + } else { + buckets[slice] += points[i]; + } + } + + // Running suffix sum over populated buckets only. + // acc = Σ_{j ≥ i, present[j]} bucket[j] + // bucket_sum = Σ_{i in [first_pop_low, top]} acc(i) = Σ_k k * bucket[k] + // Bucket 0 carries no contribution and is never added. + std::ptrdiff_t top = static_cast(num_buckets) - 1; + while (top >= 1 && present[static_cast(top)] == 0) { + --top; + } + Element bucket_sum = Curve::Group::point_at_infinity; + if (top >= 1) { + Element acc = buckets[static_cast(top)]; + bucket_sum = acc; + for (std::ptrdiff_t i = top - 1; i >= 1; --i) { + if (present[static_cast(i)] != 0) { + acc += buckets[static_cast(i)]; + } + bucket_sum += acc; + } + } + + const uint32_t doublings = (round == num_rounds - 1) ? last_round_bits : window_bits; + for (uint32_t d = 0; d < doublings; ++d) { + result.self_dbl(); + } + result += bucket_sum; + } + + per_thread_results[tid] = result; + }; + + if (num_threads == 1) { + thread_body(0); + } else { + bb::parallel_for(num_threads, thread_body); + } + + Element total = per_thread_results[0]; + for (size_t t = 1; t < num_threads; ++t) { + total += per_thread_results[t]; + } + return total; +} + +// PerWorkerArenaLayout (and its dependencies BATCH_CAPACITY, DEDUP_MAX_CHUNK_MEMBERS, +// AffineBucketChunkInfo) lives in `pippenger_arena_layout.hpp`. Used by the sizer +// below, the live allocator in `pippenger_round_parallel`, and the arena-layout +// regression test. +} // namespace round_parallel_detail + +/** + * @brief Round-parallel Pippenger MSM_fast. Windows process sequentially (high-to-low) but each + * window is fully parallel across threads. Windows are processed in batches of + * `windows_in_batch` to amortise parallel_for barriers; the batch count is sized at + * runtime to fit BATCH_MEM_BUDGET (~32 MiB). + * + * Per batch the stages are: + * 1. digit extraction — decode signed digits, tally per-(thread, window) digit counts + * 2. bucket histogram — Σ_t counts[w][t][d] + within-digit offsets + * 3. bucket offsets — per-window prefix sum + * 4. digit scatter — write into digit-partitioned schedules + * 5. chunk partition — pick per-thread digit-aligned chunk boundaries + * 6a. bucket partials — each thread reduces its chunk into a dense bucket buffer + * 6b. cross-thread merge — reduce per-thread partials across threads via the + * recursive affine bucket reduction + * 7. cross-window combine — Horner over per-window partials into the final point. + */ +#include "./pippenger_fallbacks.hpp" + +// Compute the exact arena bytes a single MSM_fast of `n_input` points will need. +// Mirrors the inline budget calculation inside `pippenger_round_parallel`. +// Returns 0 when N is small enough that we'll fall back to the Jacobian fast path +// (no affine arena needed). Exposed (declared in `scalar_multiplication_fast.hpp`) +// so the test suite can exercise the same sizer the live allocator uses. +template +size_t compute_arena_bytes_for_msm(size_t n_input, bool external_glv_provided, bool dedup_active) noexcept +{ + using ScalarField = typename Curve::ScalarField; + constexpr size_t FULL_NUM_BITS = ScalarField::modulus.get_msb() + 1; + + if (n_input < 4) { + return 0; // trivial path + } + + const bool use_glv = external_glv_provided || (n_input <= round_parallel_detail::GLV_SMALL_N_THRESHOLD); + const size_t n = use_glv ? 2 * n_input : n_input; + const size_t NUM_BITS = use_glv ? size_t{ 128 } : FULL_NUM_BITS; + BB_ASSERT_LTE(n, + size_t{ round_parallel_detail::SCHEDULE_INDEX_MASK } + 1, + "working scalar indices must fit in the 29-bit schedule payload"); + + using round_parallel_detail::BATCH_MEM_BUDGET; + using round_parallel_detail::MIN_AFFINE_THREAD_RATIO; + using round_parallel_detail::MIN_BATCH_CAPACITY; + using round_parallel_detail::SUBCHUNK_ENTRIES_CAP; + + // window-bits selection uses the ideal per-window oversubscription factor (not the dispatch lmul). + const size_t num_logical_threads_for_c = bb::get_num_cpus() * window_bits_tuning_oversub_factor(n_input); + const size_t window_bits = + round_parallel_detail::choose_window_bits(n, NUM_BITS, n_input, num_logical_threads_for_c); + const size_t num_windows = (NUM_BITS + 2 + window_bits - 1) / window_bits; + const size_t num_buckets = (size_t{ 1 } << (window_bits - 1)) + 1; + + const size_t desired_threads = std::max(1, bb::get_num_cpus()); + const size_t max_threads_for_min_batch = n / MIN_BATCH_CAPACITY; + const size_t min_threads_allowed = + std::max(1, (desired_threads + MIN_AFFINE_THREAD_RATIO - 1) / MIN_AFFINE_THREAD_RATIO); + + if (max_threads_for_min_batch < min_threads_allowed) { + return 0; // jacobian-fast fallback, no affine arena + } + + const size_t num_threads = std::min(desired_threads, std::max(1, max_threads_for_min_batch)); + + // num_threads sizes the per-task arrays; worker_total sizes the per-OS-thread scratch + // (FIFO-shared by every task that lands on that OS thread). + const size_t worker_total_for_budget = num_threads; + const size_t dense_stride_est = round_parallel_detail::compute_dense_stride(num_buckets, num_threads); + + // Pre-schedule conservative per-window cost: uses `num_buckets` (= 2^(c-1)+1) as the + // B upper bound. The lambda below recomputes once the actual schedule is built. + const size_t per_window_bytes = round_parallel_detail::compute_per_window_bytes( + num_threads, num_buckets, n, dense_stride_est, worker_total_for_budget); + + const size_t global_max_overflow_per_window = + round_parallel_detail::compute_global_max_overflow_per_window(n, num_threads, SUBCHUNK_ENTRIES_CAP); + + const bool inline_glv_double = use_glv && !external_glv_provided; + const size_t profile_threads = std::max(1, bb::get_num_cpus()); + const size_t phase_one_prologue_bytes = + round_parallel_detail::compute_phase_one_prologue_bytes(n, use_glv, inline_glv_double, profile_threads); + + const auto phase_a_caps = round_parallel_detail::compute_phase_a_caps(n, num_threads); + const size_t phase_a_cluster_members_cap = phase_a_caps.members_cap; + const size_t phase_a_cluster_offsets_cap = phase_a_caps.offsets_cap; + + // Zone W per-worker UNION via the canonical layout walk. Stage 6a, Stage 6b, and + // Phase A overlay the same per-worker bytes; the struct returns the max-of-layouts + // (the Stage 6 wpb-dependent tail is added below once `windows_per_batch` is known). + // Passing `windows_per_batch = 0` here skips the tail — we only need the union bytes + // for the fixed_overhead → wpb solve. + const round_parallel_detail::PerWorkerArenaLayout union_layout(/*chunk_capacity=*/SUBCHUNK_ENTRIES_CAP, + global_max_overflow_per_window, + dedup_active, + phase_a_cluster_members_cap, + phase_a_cluster_offsets_cap, + /*windows_per_batch=*/0, + /*dense_stride_est=*/0); + const size_t worker_union_bytes = union_layout.per_worker_union_bytes; + + const size_t fixed_overhead = (worker_union_bytes * worker_total_for_budget) + + (size_t{ 96 } * round_parallel_detail::VAR_WINDOW_MAX_WINDOWS) // window_sums_storage + + (size_t{ 8 } * (num_threads + 1)) // rebalanced_bucket_lo_partition + + phase_one_prologue_bytes; + + // wpb fallback when fixed_overhead has eaten the BATCH_MEM_BUDGET headroom: the inline + // `solve_wpb` in `pippenger_round_parallel` returns `W_R` (the whole region) — running + // every window in a single batch — when `available_budget == 0`. Previously the sizer + // returned `wpb = 1` and relied on a `worst_case_arena = BATCH_MEM_BUDGET + 32K` floor; + // that floor failed for large num_threads where fixed_overhead alone exceeds the budget. + const size_t available_budget_outer = + (BATCH_MEM_BUDGET > fixed_overhead) ? (BATCH_MEM_BUDGET - fixed_overhead) : size_t{ 0 }; + const size_t windows_per_batch = + round_parallel_detail::solve_wpb(per_window_bytes, available_budget_outer, num_windows); + // Dedup state lives in the arena (allocated post-Phase-1, retained through Stage 6a). + // Worst-case sizes: redirect_lookup is one uint32 per working scalar (4n bytes); + // extra_points is the fixed DEDUP_MAX_CLUSTERS cap (≈1 MB) regardless of n. + const size_t dedup_bytes = dedup_active ? ((size_t{ 4 } * n) + (size_t{ sizeof(typename Curve::AffineElement) } * + round_parallel_detail::DEDUP_MAX_CLUSTERS)) + : size_t{ 0 }; + auto arena_bytes_for_window_layout = [&](size_t bit_budget) { + const size_t wb = round_parallel_detail::choose_window_bits(n, bit_budget, n_input, num_logical_threads_for_c); + const auto layout_sched = round_parallel_detail::build_var_window_schedule(bit_budget, wb); + size_t B_eff_layout = (size_t{ 1 } << (wb - 1)) + 1; + for (size_t w = 0; w < layout_sched.num_windows; ++w) { + B_eff_layout = std::max(B_eff_layout, static_cast(layout_sched.num_buckets[w])); + } + const size_t dense_stride_layout = round_parallel_detail::compute_dense_stride(B_eff_layout, num_threads); + const size_t per_window_bytes_layout = round_parallel_detail::compute_per_window_bytes( + num_threads, B_eff_layout, n, dense_stride_layout, worker_total_for_budget); + + const size_t available_budget = + (BATCH_MEM_BUDGET > fixed_overhead) ? (BATCH_MEM_BUDGET - fixed_overhead) : size_t{ 0 }; + const size_t wpb = round_parallel_detail::solve_wpb( + per_window_bytes_layout, available_budget, static_cast(layout_sched.num_windows)); + return fixed_overhead + (wpb * per_window_bytes_layout) + 32768 + dedup_bytes; + }; + + // Tight return: the arena holds `fixed_overhead + wpb · per_window_bytes` of typed + // buffers plus a 32 KiB alignment pad and the dedup state (when active). Sizing + // tightly — rather than padding up to BATCH_MEM_BUDGET — matters for many-MSM_fast flows + // (e.g. PerMsmChonk's 256 separate per-circuit MSMs) where every per-MSM_fast + // `make_unique_for_overwrite` mmap/munmaps the buffer above glibc's + // M_MMAP_THRESHOLD; a 32 MiB floor here would tax every MSM_fast with the page-fault + // first-touch cost regardless of how much of the arena the small MSM_fast actually uses. + size_t arena_bytes = fixed_overhead + (windows_per_batch * per_window_bytes) + 32768 + dedup_bytes; + + // The live pipeline shrinks NUM_BITS to the observed max scalar bit before choosing + // window_bits. GLV MSMs and large non-GLV MSMs can therefore select a different + // schedule/zone layout than the full-bit pre-sizer. Keep the common Chonk wire/IPA + // non-GLV sizes on the original tight path. + if (use_glv || n_input >= (size_t{ 1 } << 17)) { + for (size_t bit_budget = 1; bit_budget <= NUM_BITS; ++bit_budget) { + arena_bytes = std::max(arena_bytes, arena_bytes_for_window_layout(bit_budget)); + } + } + return arena_bytes; +} + +// Round-parallel Pippenger MSM_fast. +// `external_glv_doubled` — optional caller-supplied [P_0, φP_0, …, P_{n-1}, φP_{n-1}] +// buffer (length 2·n_input). When non-empty, forces use_glv=true and skips the +// internal doubling pass. The interleaved layout means longer-prefix aliasing +// (length 2·Nmax) is valid for any n ≤ Nmax with no copy. +// `external_arena` — optional caller-supplied scratch buffer ≥ this MSM_fast's required +// bytes. When empty, allocate per-MSM_fast via make_unique_for_overwrite and free at +// return. The batched driver supplies a single arena sized to the largest member. +template +// NOLINTNEXTLINE(readability-function-size, readability-function-cognitive-complexity, +// google-readability-function-size) +typename Curve::Element pippenger_round_parallel(PolynomialSpan scalars_span, + std::span all_points, + bool dedup_hint, + std::span external_glv_doubled, + std::span external_arena) noexcept +{ + using Element = typename Curve::Element; + using AffineElement = typename Curve::AffineElement; + using ScalarField = typename Curve::ScalarField; + using BaseField = typename Curve::BaseField; + + const size_t n_input = scalars_span.size(); + if (n_input == 0) { + return Curve::Group::point_at_infinity; + } + + // Bail to trivial_msm_threaded when each worker would own fewer than + // MIN_PTS_PER_THREAD_FOR_PIPPENGER points — pippenger_fast's per-window scaffolding loses + // to straus_msm at this density. Caller-supplied GLV doubling is wasted at this size, + // but the overhead is negligible. + { + const size_t max_threads = bb::get_num_cpus(); + const size_t num_threads_dispatch = std::max(1, std::min(n_input, max_threads)); + const size_t pts_per_thread = (n_input + num_threads_dispatch - 1) / num_threads_dispatch; + if (pts_per_thread < MIN_PTS_PER_THREAD_FOR_PIPPENGER) { + return trivial_msm_threaded(scalars_span, all_points); + } + } + + BB_ASSERT_GTE(all_points.size(), scalars_span.start_index + n_input); + std::span input_points(&all_points[scalars_span.start_index], n_input); + + constexpr size_t FULL_NUM_BITS = ScalarField::modulus.get_msb() + 1; + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + ScalarField* scalar_ptr = const_cast(&scalars_span[scalars_span.start_index]); + std::span input_scalars(scalar_ptr, n_input); + + // GLV: split k ≡ k1 − k2·λ (mod r), giving 2n pairs at NUM_BITS=128. Halves num_windows; + // costs an extra n point doubles. Applied only below GLV_SMALL_N_THRESHOLD where the + // win-on-windows beats the lose-on-doubled-scan, OR forced on by the batched dispatcher + // supplying `external_glv_doubled` (it amortises the doubling across the whole batch). + // Empirical crossover (best-of-3 sweep at HC=16, P ∈ {4, 8, 16}): wasmtime keeps GLV up + // to n=2^16; native to n=2^13 (clang's branchless bias-decode is fast enough that the 2× + // point-count cost dominates above that). Threshold is platform-conditional in the + // hoisted GLV_SMALL_N_THRESHOLD declaration. + const bool external_glv_provided = !external_glv_doubled.empty(); + const bool use_glv = external_glv_provided || n_input <= round_parallel_detail::GLV_SMALL_N_THRESHOLD; + + // Stage 6 splits into 6a (per-thread bucket partials over the contiguous-by-schedule- + // index partition) and 6b (cross-thread bucket reduction over a uniform-width digit + // slice). Small MSMs short-circuit to trivial_msm_threaded above this point. + + // n is the working scalar/point count (GLV doubles it); NUM_BITS is the post-recoding + // window-bit budget (128 for GLV, FULL_NUM_BITS otherwise). + const size_t n = use_glv ? (2 * n_input) : n_input; + const size_t NUM_BITS = use_glv ? size_t{ 128 } : FULL_NUM_BITS; + BB_ASSERT_LTE(n, + size_t{ round_parallel_detail::SCHEDULE_INDEX_MASK } + 1, + "working scalar indices must fit in the 29-bit schedule payload"); + std::span scalars; + std::span points; + const bool inline_glv_double = use_glv && !external_glv_provided; + + // Activation gate: caller-supplied hint opts this MSM_fast into the dedup pre-pass. + // Hint-driven so polynomials with low duplicate density (PC counters, range checks) + // skip the O(n) tagging cost. The small-n bail above (pts_per_thread < + // MIN_PTS_PER_THREAD_FOR_PIPPENGER) already shed every case where dedup wouldn't fit + // — n ≥ MIN_PTS_PER_THREAD_FOR_PIPPENGER * 1 = 24 here. + const bool dedup_active = dedup_hint; + + // --------------------------------------------------------------------------------------- + // Arena setup (pre-Phase-1). + // + // The per-MSM_fast arena is allocated BEFORE Phase 1 so the Phase 1 prologue (msb_per_scalar, + // glv_*_storage, per_thread_msb_hist) lives inside the arena instead of on the heap. + // Once Phase 1 finishes and the window schedule is known (T, B_eff, dense_stride, wpb), + // we partition the remaining capacity into three named zones + // (Zone P / Zone W / Zone S) — see the "Arena zone layout" block after the wpb solve. + // + // We size the buffer using `compute_arena_bytes_for_msm`, whose conservative bound + // dominates the inline-tight (P + W + S) sum for any wpb we choose below. + // --------------------------------------------------------------------------------------- + const size_t arena_total_bytes = compute_arena_bytes_for_msm(n_input, external_glv_provided, dedup_active); + round_parallel_detail::MsmArena arena(arena_total_bytes, external_arena); + + // --------------------------------------------------------------------------------------- + // Phase 1 — convert scalars from Montgomery, optionally GLV-split, populate msb buffer. + // The msb_per_scalar buffer feeds max-msb num_windows selection; + // per-thread msb_hist counts (bin 0 = zero, bin k+1 = msb == k) feed the n_active gate + // and the active-scalar gate. + // + // When dedup is active the per-scalar dedup work (hash + linear-probe shared atomic + // table, per-thread dup_pair recording) is fused into the same per-thread loop so + // scalars stay hot in L1 between from-Mont and the hash. The post-pass (sort, cluster + // build, chunked tree-reduce, redirect_lookup) runs sequentially after the parallel_for + // — see `dedup_finalize_parallel`. + // --------------------------------------------------------------------------------------- + using round_parallel_detail::MSB_ZERO_SENTINEL; + const size_t profile_threads = std::max(1, bb::get_num_cpus()); + auto msb_per_scalar = arena.template alloc(n); + auto per_thread_msb_hist = arena.template alloc>(profile_threads); + // MsmArena::alloc returns uninitialised memory; the histograms must be zero-initialised so + // record_msb's increments land on a clean slate. + std::fill_n(per_thread_msb_hist.data(), profile_threads, std::array{}); + + // GLV storage (optional). `glv_scalars_storage` is the GLV-split working scalar buffer; + // `glv_points_storage` is the inline-doubled point buffer (skipped when the caller + // supplied an external doubled buffer). Both span empty when `use_glv` is false. + std::span glv_scalars_storage; + std::span glv_points_storage; + if (use_glv) { + glv_scalars_storage = arena.template alloc(n); + if (inline_glv_double) { + glv_points_storage = arena.template alloc(n); + } else { + BB_ASSERT_EQ(external_glv_doubled.size(), n); + } + } + + if (use_glv) { + // Convert each input scalar from-Mont into a stack local, GLV-split it, store both + // 128-bit halves and their msb into the profile buffer. input_scalars is read-only on + // this path so the user's buffer is preserved (no Montgomery restore needed). Inline + // path additionally GLV-doubles the points in the same parallel pass; external path + // aliases the caller-supplied doubled buffer. + const BaseField beta = inline_glv_double ? BaseField::cube_root_of_unity() : BaseField{}; + bb::parallel_for(bb::get_num_cpus(), [&](const ThreadChunk& chunk) { + BB_BENCH_NAME("MSM_fast::glv_from_mont_split/worker"); + auto& th_hist = per_thread_msb_hist[chunk.thread_index]; + for (size_t i : chunk.range(n_input)) { + const ScalarField canonical = input_scalars[i].from_montgomery_form_reduced(); + const auto split = ScalarField::split_into_endomorphism_scalars(canonical); + const auto& k1 = split.first; + const auto& k2 = split.second; + glv_scalars_storage[2 * i].data[0] = k1[0]; + glv_scalars_storage[2 * i].data[1] = k1[1]; + glv_scalars_storage[2 * i].data[2] = 0; + glv_scalars_storage[2 * i].data[3] = 0; + glv_scalars_storage[(2 * i) + 1].data[0] = k2[0]; + glv_scalars_storage[(2 * i) + 1].data[1] = k2[1]; + glv_scalars_storage[(2 * i) + 1].data[2] = 0; + glv_scalars_storage[(2 * i) + 1].data[3] = 0; + if (inline_glv_double) { + glv_points_storage[2 * i] = input_points[i]; + glv_points_storage[(2 * i) + 1].x = input_points[i].x * beta; + glv_points_storage[(2 * i) + 1].y = -input_points[i].y; + } + round_parallel_detail::record_msb( + round_parallel_detail::msb_of_2limb(k1[0], k1[1]), msb_per_scalar[2 * i], th_hist); + round_parallel_detail::record_msb( + round_parallel_detail::msb_of_2limb(k2[0], k2[1]), msb_per_scalar[(2 * i) + 1], th_hist); + } + }); + points = + inline_glv_double ? std::span(glv_points_storage.data(), n) : external_glv_doubled; + scalars = glv_scalars_storage; + } else { + // Non-GLV path: in-place from-Mont (later restored in the Stage-7 epilogue). + bb::parallel_for(bb::get_num_cpus(), [&](const ThreadChunk& chunk) { + BB_BENCH_NAME("MSM_fast::from_montgomery/worker"); + auto& th_hist = per_thread_msb_hist[chunk.thread_index]; + for (size_t i : chunk.range(n_input)) { + input_scalars[i].self_from_montgomery_form_reduced(); + round_parallel_detail::record_msb( + round_parallel_detail::msb_of_4limb(input_scalars[i].data), msb_per_scalar[i], th_hist); + } + }); + scalars = input_scalars; + points = input_points; + } + + std::array msb_hist{}; + for (size_t t = 0; t < profile_threads; ++t) { + for (size_t b = 0; b < 256; ++b) { + msb_hist[b] += per_thread_msb_hist[t][b]; + } + } + const size_t n_active_early = n - static_cast(msb_hist[0]); + + // --------------------------------------------------------------------------------------- + // Phase 2 — bail to trivial_msm_threaded when n_active is too small to amortise pippenger_fast's + // per-window scaffolding. trivial_msm_threaded -> straus_msm wants Montgomery scalars, so + // re-Mont-form them in parallel before dispatching. + // --------------------------------------------------------------------------------------- + { + const size_t max_threads_dispatch = bb::get_num_cpus(); + const size_t threads_for_dispatch = std::max(1, std::min(n_active_early, max_threads_dispatch)); + const size_t pts_per_thread = (n_active_early + threads_for_dispatch - 1) / threads_for_dispatch; + if (pts_per_thread < MIN_PTS_PER_THREAD_FOR_PIPPENGER) { + bb::parallel_for(bb::get_num_cpus(), [&](const ThreadChunk& chunk) { + BB_BENCH_NAME("MSM_fast::trivial_msm_to_montgomery/worker"); + for (size_t i : chunk.range(n)) { + scalars[i].self_to_montgomery_form(); + } + }); + std::span scalars_const(scalars.data(), n); + PolynomialSpan ps(0, scalars_const); + return trivial_msm_threaded(ps, points); + } + } + + // --------------------------------------------------------------------------------------- + // Phase 3 — pick the window layout, build the schedule, run the pipeline, sum into the result. + // --------------------------------------------------------------------------------------- + const size_t num_logical_threads_for_c = bb::get_num_cpus() * window_bits_tuning_oversub_factor(n_input); + + // Shrink the bit budget to the highest non-empty msb_hist bin so num_windows is determined + // by the actual data, not the conservative GLV / FULL_NUM_BITS bound. + size_t effective_num_bits = 0; + for (size_t bin = 256; bin > 1;) { + --bin; + if (msb_hist[bin] != 0) { + effective_num_bits = bin; + break; + } + } + if (effective_num_bits == 0 || effective_num_bits > NUM_BITS) { + effective_num_bits = NUM_BITS; + } + const size_t window_bits = + round_parallel_detail::choose_window_bits(n, effective_num_bits, n_input, num_logical_threads_for_c); + const size_t num_buckets = (size_t{ 1 } << (window_bits - 1)) + 1; + + // Schedule-based dedup state. The two arrays are allocated from the per-MSM_fast arena + // *from the arena after Phase 1. + // Until then, both spans are empty. + // Lifetimes: + // redirect_lookup — written by Phase A; read by Stage 4b's dedup_patch_schedule per batch + // extra_points — written by Phase A; read by Stage 6a's reduce_chunk per batch + // Both must survive until the last Stage 6a, so they sit in the arena (which is freed + // when this function returns). + round_parallel_detail::DedupResult dedup_state; + + // Variable-window split was removed from the production path after Chonk traces showed + // it regressing this rewrite. Keep the schedule uniform and run one region over all + // non-zero scalars. + const auto sched = round_parallel_detail::build_var_window_schedule(effective_num_bits, window_bits); + BB_ASSERT_LTE(sched.num_windows, + round_parallel_detail::VAR_WINDOW_MAX_WINDOWS, + "window schedule exceeds compile-time max window count"); + + using round_parallel_detail::BATCH_CAPACITY; + using round_parallel_detail::BATCH_MEM_BUDGET; + using round_parallel_detail::MIN_BATCH_CAPACITY; + using round_parallel_detail::SUBCHUNK_ENTRIES_CAP; + + // Thread count: aim for `lmul × physical_cpus` logical tasks so the rpmsm pool can + // FIFO-balance heterogeneous P/E cores; cap at `n / MIN_BATCH_CAPACITY` so each chunk + // can saturate the batched-affine drains. `bb::get_num_cpus() <= 1` is the chonk + // batch-verifier's signal that outer parallelism owns all cores — run sequentially. + const size_t desired_threads = std::max(1, bb::get_num_cpus()); + const size_t max_threads_for_min_batch = std::max(1, n / MIN_BATCH_CAPACITY); + const size_t num_threads = std::min(desired_threads, max_threads_for_min_batch); + + // Stage 6's tree-reduce splits each thread's chunk into sub-chunks of at most + // SUBCHUNK_ENTRIES_CAP entries before calling reduce_chunk, bounding per-thread scratch + // independent of n. 2048 keeps level-0 saturated (≥ 4 BATCH_CAPACITY drains at typical + // c=16) while the deepest level still hits BATCH_AFFINE_BREAKEVEN (~32 pairs); halving + // breaks the deep levels and doubling wastes memory. + // Pick windows_in_batch so per-MSM_fast working set fits in ~32 MB. Empirically 32 MB + // performs as well as 128 MB on the WASM grid (the recursive affine bucket reduction + // recovers most of the small-batch loss). + // The per_window_bytes / fixed_overhead formulas below mirror this enum of allocations + // exactly. Anyone adding an arena buffer must update both the alloc and the corresponding + // term in those formulas, otherwise windows_per_batch drifts off the BATCH_MEM_BUDGET. + + // Per-(w, t) slot stride must fit the widest schedule window. + size_t B_eff = num_buckets; + for (size_t w = 0; w < sched.num_windows; ++w) { + B_eff = std::max(B_eff, static_cast(sched.num_buckets[w])); + } + + const size_t worker_total_for_budget = num_threads; + const size_t dense_stride_est = round_parallel_detail::compute_dense_stride(B_eff, num_threads); + const size_t bucket_partials_per_window_max = + round_parallel_detail::compute_bucket_partials_max(B_eff, num_threads); + const size_t per_window_bytes_lo = round_parallel_detail::compute_per_window_bytes( + num_threads, B_eff, n, dense_stride_est, worker_total_for_budget); + + const size_t global_max_overflow_per_window_for_budget = + round_parallel_detail::compute_global_max_overflow_per_window(n, num_threads, SUBCHUNK_ENTRIES_CAP); + + const size_t phase_one_prologue_bytes = + round_parallel_detail::compute_phase_one_prologue_bytes(n, use_glv, inline_glv_double, profile_threads); + + const auto phase_a_caps = round_parallel_detail::compute_phase_a_caps(n, num_threads); + const size_t phase_a_cluster_members_cap = phase_a_caps.members_cap; + const size_t phase_a_cluster_offsets_cap = phase_a_caps.offsets_cap; + + // Zone W per-worker UNION via the canonical layout walk. The wpb-dependent Stage 6 + // tail is added separately after `windows_per_batch` is solved; here we only need + // the union bytes for the fixed_overhead → wpb budget. + const round_parallel_detail::PerWorkerArenaLayout budget_layout( + /*chunk_capacity=*/SUBCHUNK_ENTRIES_CAP, + global_max_overflow_per_window_for_budget, + dedup_active, + phase_a_cluster_members_cap, + phase_a_cluster_offsets_cap, + /*windows_per_batch=*/0, + /*dense_stride_est=*/0); + const size_t worker_union_bytes_for_budget = budget_layout.per_worker_union_bytes; + + const size_t fixed_overhead = (worker_union_bytes_for_budget * worker_total_for_budget) + + (size_t{ 96 } * round_parallel_detail::VAR_WINDOW_MAX_WINDOWS) // window_sums_storage + + (size_t{ 8 } * (num_threads + 1)) // rebalanced_bucket_lo_partition + + phase_one_prologue_bytes; + + // Solve `wpb · per_window_bytes ≤ BATCH_MEM_BUDGET − fixed_overhead`. + const size_t available_budget = + (BATCH_MEM_BUDGET > fixed_overhead) ? (BATCH_MEM_BUDGET - fixed_overhead) : size_t{ 0 }; + const size_t windows_per_batch = + round_parallel_detail::solve_wpb(per_window_bytes_lo, available_budget, sched.num_windows); + + // Per-thread chunk-capacity scratch sizing. A thread's per-window slice is split into + // sub-chunks of at most SUBCHUNK_ENTRIES_CAP entries. Worst-case overflow per + // (thread, window) is one partial per sub-chunk boundary that lands mid-run, bounded + // above by `ceil(max_chunk_len / SUBCHUNK_ENTRIES_CAP)` where max_chunk_len ≤ n/T. + // The Stage 6a end-of-window overflow merge runs tree_reduce on `2 × overflow` entries + // (each affected slot contributes a dense head + ≥1 overflow entry). Tree-reduce + // scratch must fit either a sub-chunk's reduce_chunk input (up to SUBCHUNK_ENTRIES_CAP) + // or a full overflow merge — take the max. + const size_t global_max_chunk_len = (n + num_threads - 1) / num_threads; + const size_t global_max_overflow_per_window = + (global_max_chunk_len + SUBCHUNK_ENTRIES_CAP - 1) / SUBCHUNK_ENTRIES_CAP; + const size_t chunk_capacity = std::max(SUBCHUNK_ENTRIES_CAP, 2 * global_max_overflow_per_window); + + // Per-OS-thread scratch. The rpmsm pool dispatches `num_threads` logical tasks across + // `worker_total = num_threads = physical_cpus` OS threads. Tasks on the same + // OS thread run sequentially (FIFO claim), so they share scratch — every field in + // ThreadScratch is overwritten fresh at task start, never read across tasks. Indexing + // by `worker_id` (rather than `tid`) keeps memory linear in physical_cpus instead of + // num_threads = lmul × physical_cpus. + const size_t worker_total = num_threads; + std::vector> thread_scratch(worker_total); + std::vector> phase_a_scratch; + if (dedup_active) { + phase_a_scratch.resize(worker_total); + } + + // --------------------------------------------------------------------------------------- + // Arena zone layout — set up after Phase 1 and schedule selection (see + // https://gist.github.com/AztecBot/7c5ef0581350f6fdb9711679552fd86f §1, §4, §5). + // + // [0 .. bytes_P) Zone P — whole-MSM_fast permanent + // msb_per_scalar (already alloc'd above) + // glv_scalars / glv_points (already alloc'd above) + // per_thread_msb_hist (already alloc'd above) + // window_sums (Stage 7 accumulator) + // redirect_lookup, extra_points (dedup, if active) + // [bytes_P .. bytes_P + bytes_W) Zone W — per-worker union slab × T + // Stage 6a/6b ThreadScratch fields and PhaseA + // scratch overlay the same per-worker bytes; the + // wpb-dependent Stage 6 fields sit immediately + // after the union. Stage 6a, Stage 6b, and Phase A + // run in distinct parallel_for invocations and + // never co-exist on a worker. + // [bytes_P + bytes_W .. arena.capacity) + // Zone S — per-batch swing region (schedule, HIST slot, + // DENSE slot, partition metadata). + // HIST slot overlays H ↔ O on one byte slab: + // H (S1-S4): digit_cursors + // O (S6b-S7): chunk_outputs/window_partial_sums + // Slot per-window = max(H, O). At chonk this is + // H-bound (~256 KiB/window). + // DENSE slot is dedicated for D (S6a-S6b): + // bucket_partials_dense / _present + // (~135 KiB/window at chonk). The D-class was + // moved out of the HIST slot to eliminate L1 + // cache aliasing on the Stage 6a scatter writes + // (+1.29% regression observed when D was overlaid + // at the HIST offset). + // + // wpb solve: BATCH_MEM_BUDGET - bytes_P - bytes_W_fixed - bytes_S_shared - 32 KiB pad, + // divided by (bytes_S_per_window + bytes_W_per_wpb). per_window_bytes_shared accounts + // for HIST + DENSE as two separate slots. + // --------------------------------------------------------------------------------------- + + // Freeze Zone P prefix at the post-Phase-1 cursor — everything allocated so far + // (msb_per_scalar, glv storage, per_thread_msb_hist) is Zone P permanent state. + const size_t bytes_P_prefix = arena.cursor; + + // Per-worker fixed-bytes "union": ThreadScratch's wpb-independent fields overlay the + // PhaseAScratch fields. Compute each layout's strict byte requirement (including the + // alignment slop a bump cursor would consume), then take the max. + auto align_up = [](size_t off, size_t align) -> size_t { return (off + align - 1) & ~(align - 1); }; + auto layout_add = [&](size_t& off, size_t bytes, size_t align) { off = align_up(off, align) + bytes; }; + + // Per-worker layout via the canonical walk (single source of truth shared with + // `compute_arena_bytes_for_msm`). Pre-wpb-solve usage there passes wpb=0; here we + // pass the actual windows_per_batch so the Stage 6 wpb-dependent tail is included. + const round_parallel_detail::PerWorkerArenaLayout worker_layout(chunk_capacity, + global_max_overflow_per_window, + dedup_active, + phase_a_cluster_members_cap, + phase_a_cluster_offsets_cap, + windows_per_batch, + dense_stride_est); + constexpr size_t WORKER_SLAB_ALIGN = round_parallel_detail::PerWorkerArenaLayout::WORKER_SLAB_ALIGN; + const size_t per_worker_union_bytes = worker_layout.per_worker_union_bytes; + const size_t per_worker_bytes = worker_layout.per_worker_bytes; + + // Zone P extra (post-decision permanent state): window_sums + dedup state. Sized + // with the strict alignment a bump cursor would apply. + constexpr size_t VAR_WINDOW_WINDOW_SUMS_CAP = round_parallel_detail::VAR_WINDOW_MAX_WINDOWS; + size_t bytes_P_extra_layout = 0; + layout_add(bytes_P_extra_layout, sizeof(Element) * VAR_WINDOW_WINDOW_SUMS_CAP, alignof(Element)); + if (dedup_active) { + layout_add(bytes_P_extra_layout, sizeof(uint32_t) * n, alignof(uint32_t)); + layout_add(bytes_P_extra_layout, + sizeof(AffineElement) * round_parallel_detail::DEDUP_MAX_CLUSTERS, + alignof(AffineElement)); + } + + // Zone sizes. The Zone W slab uses `MsmArena::bump_alloc` which aligns in ABSOLUTE address + // space (the arena buffer base is only `__STDCPP_DEFAULT_NEW_ALIGNMENT__`-aligned, but + // AffineElement is alignas(64)). To make the per-worker layout match the layout-only + // calc (which assumes the slab starts on a 64-byte boundary), bias bytes_P so the + // absolute address `arena.data + bytes_P` is 64-aligned. + const size_t arena_base_misalign = static_cast(arena.base_addr & (WORKER_SLAB_ALIGN - 1)); + const size_t bytes_P_min = align_up(bytes_P_prefix, alignof(Element)) + bytes_P_extra_layout; + const size_t bytes_P = align_up(bytes_P_min + arena_base_misalign, WORKER_SLAB_ALIGN) - arena_base_misalign; + // bytes_W: per_worker_bytes is already rounded to WORKER_SLAB_ALIGN, so consecutive + // slabs stay aligned once the first slab is aligned. + const size_t bytes_W = per_worker_bytes * worker_total; + + // Sanity: zones must fit. The conservative `compute_arena_bytes_for_msm` upper bound + // sized the buffer to `BATCH_MEM_BUDGET + 32K + dedup_bytes` at worst, which dominates + // every reachable (P + W + S) sum at the inline-tight wpb chosen above. + BB_ASSERT_LTE(bytes_P + bytes_W, arena.capacity); + const size_t bytes_S_total = arena.capacity - bytes_P - bytes_W; + + // Per-zone bump cursors. Zone P continues from `bytes_P_prefix`; Zones W and S start + // fresh at their zone base. Zone P's bound is `bytes_P` so the bump cursor stays inside + // its slot even if the extra slabs alignment-slop a hair. + size_t zone_P_cursor = bytes_P_prefix; + size_t zone_S_cursor = 0; + auto zone_P_alloc = [&](size_t count) -> std::span { + return arena.template bump_alloc(count, zone_P_cursor, bytes_P, 0); + }; + auto zone_S_alloc = [&](size_t count) -> std::span { + return arena.template bump_alloc(count, zone_S_cursor, bytes_S_total, bytes_P + bytes_W); + }; + // Zone W is carved into per-worker slabs directly via `MsmArena::bump_alloc` below — each + // worker gets its own (cursor, bound) pair, so a single zone-wide allocator would not + // capture the per-worker discipline. + // The pre-Phase-1 `MsmArena::alloc` cursor is retired here — every subsequent allocation + // routes through `zone_P_alloc`, the per-worker Zone W allocators, or `zone_S_alloc`. + + // Zone W: per-worker union slab — Stage6a/6b ThreadScratch and PhaseA fields overlay the + // same per-worker bytes, with the wpb-dependent Stage 6 fields immediately after. + for (size_t t = 0; t < worker_total; ++t) { + // Each worker's slab is a contiguous `per_worker_bytes` window inside Zone W. + const size_t slab_base = t * per_worker_bytes; + auto& s = thread_scratch[t]; + + // ThreadScratch fixed fields — first view into the union. Bound = union size. + size_t ts_fixed_cur = 0; + auto ts_fixed_alloc = [&](size_t count) -> std::span { + return arena.template bump_alloc(count, ts_fixed_cur, per_worker_union_bytes, bytes_P + slab_base); + }; + s.curr_pts = ts_fixed_alloc.template operator()(chunk_capacity); + s.curr_buckets = ts_fixed_alloc.template operator()(chunk_capacity); + s.points_to_add = ts_fixed_alloc.template operator()(2 * BATCH_CAPACITY); + s.inversion_scratch = ts_fixed_alloc.template operator()(BATCH_CAPACITY); + s.pair_dest = ts_fixed_alloc.template operator()(BATCH_CAPACITY); + s.overflow_slots = ts_fixed_alloc.template operator()(global_max_overflow_per_window); + s.overflow_pts = ts_fixed_alloc.template operator()(global_max_overflow_per_window); + + // PhaseA fields — second view, overlays the SAME per-worker union bytes. PhaseA's + // parallel_for never overlaps Stage 6a/6b on the same worker, so reusing the bytes is + // safe; the union's size is max(ts_fixed_layout, pa_layout) by construction. + if (dedup_active) { + size_t pa_cur = 0; + auto pa_alloc = [&](size_t count) -> std::span { + return arena.template bump_alloc(count, pa_cur, per_worker_union_bytes, bytes_P + slab_base); + }; + auto& ps = phase_a_scratch[t]; + using PWAL = round_parallel_detail::PerWorkerArenaLayout; + ps.cluster_members = pa_alloc.template operator()(phase_a_cluster_members_cap); + ps.cluster_offsets = pa_alloc.template operator()(phase_a_cluster_offsets_cap); + ps.dirty_slots = pa_alloc.template operator()(PWAL::PHASE_A_DIRTY_SLOTS_CAP); + ps.bucket_rep = pa_alloc.template operator()(PWAL::PHASE_A_BUCKET_REP_CAP); + ps.staged = pa_alloc.template operator()>(PWAL::PHASE_A_STAGED_CAP); + ps.chunk_pts = pa_alloc.template operator()(PWAL::PHASE_A_CHUNK_CAP); + ps.chunk_ids = pa_alloc.template operator()(PWAL::PHASE_A_CHUNK_CAP); + } + + // Stage 6 wpb-dependent fields — tail of the per-worker slab, BEYOND the union. Bound + // = full per-worker slab size; cursor starts at per_worker_union_bytes so we don't + // overwrite the union region. + size_t ts_tail_cur = per_worker_union_bytes; + auto ts_tail_alloc = [&](size_t count) -> std::span { + return arena.template bump_alloc(count, ts_tail_cur, per_worker_bytes, bytes_P + slab_base); + }; + const size_t dense_total = windows_per_batch * dense_stride_est; + const size_t dense_pair_max = dense_total / 2; + s.dense_buckets = ts_tail_alloc.template operator()(dense_total); + s.is_present = ts_tail_alloc.template operator()(dense_total); + s.affine_bucket_pairs = ts_tail_alloc.template operator()>(dense_pair_max); + s.affine_bucket_indices = ts_tail_alloc.template operator()(dense_pair_max); + s.affine_bucket_inversion_scratch = ts_tail_alloc.template operator()(dense_pair_max); + s.chunk_infos = + ts_tail_alloc.template operator()(windows_per_batch); + std::fill_n(s.chunk_infos.begin(), windows_per_batch, round_parallel_detail::AffineBucketChunkInfo{}); + s.affine_bucket_stride = dense_stride_est; + } + + // Zone S: per-batch swing region — schedule + HIST slot + DENSE slot + partition metadata. + const size_t schedule_total = windows_per_batch * n; + auto schedule = zone_S_alloc.template operator()(schedule_total); + + // ----- HIST slot ------------------------------------------------------------------ + // Single byte slab backing two non-coexisting lifetime classes: + // Epoch H (Stages 1-4): digit_cursors. + // Epoch O (Stages 6b-7): chunk_outputs, window_partial_sums. + // H dies before O is born (Stage 4 cursor advance ends before Stage 6b first writes + // chunk_outputs / window_partial_sums). + // + // D-class (bucket_partials_dense + bucket_partials_present) previously overlaid this + // slot too, but a 10× interleaved WASM Chonk bench showed Stage 6a regressed +1.29% + // (t=+58) because of L1 cache aliasing on the `dense[slot]/present[slot]` scatter + // writes when D sat at the HIST-overlaid offset. D-class now has its own dedicated + // Zone-S DENSE slot below — see "DENSE slot" comment block. + // + // Phase 4: `digit_cursors` is dual-role within epoch H. After Stage 1 it holds + // per-(w, t) counts of digit d; Stage 2 walks each (w, d) column from t = 0..T-1 + // reading the count from slot k and writing back the exclusive prefix-sum offset + // (the count is consumed into `running` BEFORE the slot is overwritten, so the + // in-place transform is mathematically identical to the previous out-of-place + // version). Stage 4 then advances each (w, t) slice as a per-thread cursor. + // Strict aliasing: every access goes through a std::span obtained by + // reinterpret_cast(hist_slot.data() + offset) + // which is well-defined because std::byte is allowed by [basic.lval] to alias any + // POD type. All overlaid types (uint32_t, size_t, Element, ChunkOutput) are + // trivially copyable / standard layout so the two epochs do not require construction + // or destruction calls when the role of the bytes changes. + static_assert(alignof(Element) <= 32, "HIST slot O layout assumes alignof(Element) <= 32"); + static_assert(alignof(round_parallel_detail::ChunkOutput) <= 32, + "HIST slot O layout assumes alignof(ChunkOutput) <= 32"); + + auto align_up_local = [](size_t off, size_t a) -> size_t { return (off + a - 1) & ~(a - 1); }; + + // Exact byte requirements for each epoch (matches the budget formula above). + const size_t hist_h_bytes_total = (size_t{ 4 } * windows_per_batch * num_threads * B_eff); // digit_cursors + + // O epoch layout — chunk_outputs first, then window_partial_sums. Both are alignof + // <= 32; align each up to its own alignment. + size_t o_layout_cur = 0; + o_layout_cur = align_up_local(o_layout_cur, alignof(round_parallel_detail::ChunkOutput)); + const size_t off_chunk_outputs = o_layout_cur; + o_layout_cur += sizeof(round_parallel_detail::ChunkOutput) * windows_per_batch * num_threads; + o_layout_cur = align_up_local(o_layout_cur, alignof(typename Curve::Element)); + const size_t off_window_partial_sums = o_layout_cur; + o_layout_cur += sizeof(typename Curve::Element) * num_threads * windows_per_batch; + const size_t hist_o_bytes_total = o_layout_cur; + + const size_t hist_slot_bytes_total = std::max(hist_h_bytes_total, hist_o_bytes_total); + // Round up to AffineElement size so the bump allocator below treats the slot as a + // whole number of 64-byte alignas(64) cells. Allocate via AffineElement to force the + // slot base to be 64-byte aligned in absolute address space — sufficient for the + // H-epoch uint32 digit_cursors span (alignof 4) and the O-epoch ChunkOutput/Element + // spans (alignof ≤ 32). + const size_t hist_slot_cells = (hist_slot_bytes_total + sizeof(AffineElement) - 1) / sizeof(AffineElement); + auto hist_slot_cells_span = zone_S_alloc.template operator()(hist_slot_cells); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + std::byte* const hist_slot_bytes = reinterpret_cast(hist_slot_cells_span.data()); + + // H-epoch view — live S1..S4. `digit_cursors[(w*T + t) * stride + d]` holds three + // distinct meanings depending on stage: + // * After Stage 1: per-(w, t) count of digit d's occurrences in thread t's slice. + // * After Stage 2: per-(w, t) exclusive prefix-sum offset (cursor base) for the + // bucket-d run inside that window's schedule slot. + // * After Stage 4: offset + count (final cursor end-state); dead from then on. + // Stage 2 reads each (w, t, d) count from this buffer and writes the running prefix + // sum back to the SAME slot before advancing `running`, so the count is preserved + // long enough to feed the accumulator. Stage 4's `++` post-increment on each + // thread's slice runs without atomics because each thread owns its (w, t, *) row + // exclusively. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + auto digit_cursors = + std::span{ reinterpret_cast(hist_slot_bytes), windows_per_batch * num_threads * B_eff }; + + // O-epoch views — live S6b..S7. Backed by the SAME bytes as above; H contents are + // dead by the time these are touched. ChunkOutput and Curve::Element have + // user-defined constructors so are not formally trivially_copyable, but they are + // standard-layout PODs of fixed bytes (Element is alignas(32) over a fixed-width Fq + // field array). The existing arena pre-Phase-3 already aliases them through std::byte + // buffers via `make_unique_for_overwrite` + reinterpret_cast; the + // std::byte aliasing rule in [basic.lval] applies regardless of trivial-copyability. + auto chunk_outputs = std::span>{ + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast*>(hist_slot_bytes + off_chunk_outputs), + windows_per_batch * num_threads + }; + auto window_partial_sums = std::span{ + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast(hist_slot_bytes + off_window_partial_sums), + num_threads * windows_per_batch + }; + // window_partial_sums is reset to identity at the start of each Stage 6b worker + // (`my_partials[w] = point_at_infinity` loop), so we deliberately do NOT initialise + // it here. chunk_outputs is written unconditionally per (w, tprime) in Stage 6b + // (the empty path sets `out.empty = 1`), so no pre-init is needed either. + // ----- end HIST slot -------------------------------------------------------------- + + // ----- DENSE slot ----------------------------------------------------------------- + // Dedicated Zone-S slot for D-class (bucket_partials_dense + bucket_partials_present). + // Lifetime is Stages 6a-6b only. Isolated from the HIST slot so Stage 6a's tight + // scatter loop + // `dst_dense[slot] = pt; dst_present[slot] = 1;` + // does not L1-alias against the HIST slot's H/O bytes (the previous co-located + // layout caused a +1.29% Stage 6a regression in WASM, t=+58 across 10× interleaved + // runs). The dense ↔ present pair stays packed at fixed aligned offsets within this + // slot — they MUST stay close because Stage 6a reads `present[slot]` then writes + // `dense[slot]` / `present[slot]` in tandem in the inner loop. + static_assert(alignof(AffineElement) == 64, "DENSE slot D layout assumes alignof(AffineElement) == 64"); + const size_t bp_total = windows_per_batch * bucket_partials_per_window_max; + size_t d_layout_cur = 0; + const size_t off_dense = d_layout_cur; + d_layout_cur += sizeof(AffineElement) * bp_total; // bucket_partials_dense + const size_t off_present = d_layout_cur; + d_layout_cur += sizeof(uint8_t) * bp_total; // bucket_partials_present + const size_t dense_slot_bytes_total = d_layout_cur; + const size_t dense_slot_cells = (dense_slot_bytes_total + sizeof(AffineElement) - 1) / sizeof(AffineElement); + // Allocate via AffineElement to force 64-byte alignment for the leading + // bucket_partials_dense view. + auto dense_slot_cells_span = zone_S_alloc.template operator()(dense_slot_cells); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + std::byte* const dense_slot_bytes = reinterpret_cast(dense_slot_cells_span.data()); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + auto bucket_partials_dense = + std::span{ reinterpret_cast(dense_slot_bytes + off_dense), bp_total }; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + auto bucket_partials_present = + std::span{ reinterpret_cast(dense_slot_bytes + off_present), bp_total }; + // ----- end DENSE slot ------------------------------------------------------------- + + auto bucket_start_all = zone_S_alloc.template operator()(windows_per_batch * (B_eff + 1)); + auto chunk_start_all = zone_S_alloc.template operator()(windows_per_batch * (num_threads + 1)); + // chunk_bucket_lo_all[w*(T+1) + t] = bucket index of the first schedule entry in + // chunk t of window w. + // chunk_bucket_hi_all[w*T + t] = bucket index of the last schedule entry in chunk t. + // Chunks are partitioned by schedule index (uniform t·m/T), not by bucket boundary, so + // a bucket's run can straddle threads — both threads then carry a partial for that + // shared bucket and Stage 7's chunk_contribution sum (Σ_d d · partial_d_in_t over t) + // combines them without an explicit merge step. + auto chunk_bucket_lo_all = zone_S_alloc.template operator()(windows_per_batch * (num_threads + 1)); + auto chunk_bucket_hi_all = zone_S_alloc.template operator()(windows_per_batch * num_threads); + + // bucket_partials_offsets is the index table that maps (thread, window) -> slot + // start in bucket_partials_dense/present. Lives S5..S6b alongside chunk_start_all, + // and stays as its own Zone S allocation (separate from the DENSE slot). + auto bucket_partials_offsets = zone_S_alloc.template operator()((num_threads * windows_per_batch) + 1); + + // Stage 6b rebalanced-task partition. The bucket range [1, num_buckets) is split evenly + // across `num_threads` rebalanced tasks t'. The partition is uniform in num_buckets so + // we store T+1 boundaries (not per-window). For each window we record the half-open + // range of original threads whose chunk range intersects each task t' — usually 1-2 + // originals per task. + auto rebalanced_bucket_lo_partition = zone_S_alloc.template operator()(num_threads + 1); + auto orig_thread_lo = zone_S_alloc.template operator()(windows_per_batch * num_threads); + auto orig_thread_hi = zone_S_alloc.template operator()(windows_per_batch * num_threads); + + // Zone P: window_sums (Stage 7 accumulator — survives the whole MSM_fast). + auto window_sums = zone_P_alloc.template operator()(VAR_WINDOW_WINDOW_SUMS_CAP); + std::fill_n(window_sums.begin(), VAR_WINDOW_WINDOW_SUMS_CAP, Curve::Group::point_at_infinity); + + // Zone P: dedup state — written by Phase A and read through Stage 6a of every batch, + // so it must outlive every batch. + // - redirect_lookup: parallel-filled with DEDUP_INVALID_EXTRA below before Phase A reads it. + // - extra_points: no init needed; Phase A writes per-thread cid ranges, and consumers + // only read indices Phase A actually populated. + if (dedup_active) { + dedup_state.redirect_lookup = zone_P_alloc.template operator()(n); + dedup_state.extra_points = + zone_P_alloc.template operator()(round_parallel_detail::DEDUP_MAX_CLUSTERS); + BB_BENCH_NAME("MSM_fast::dedup/redirect_invalid_fill"); + uint32_t* const rl = dedup_state.redirect_lookup.data(); + bb::parallel_for(bb::get_num_cpus(), [&](const ThreadChunk& chunk) { + BB_BENCH_NAME("MSM_fast::dedup/redirect_invalid_fill/worker"); + for (size_t i : chunk.range(n)) { + rl[i] = round_parallel_detail::DEDUP_INVALID_EXTRA; + } + }); + } + + // BUCKET_MASK strips the sign bit off a packed (sign | bucket) digit produced by + // get_constantine_packed_digit, leaving the unsigned bucket index. + constexpr uint32_t BUCKET_MASK = (uint32_t{ 1 } << 31) - 1; + + // Phase A runs at most once per MSM_fast (not per batch). Cluster membership is determined + // by scalar value (memcmp) — independent of which window we walk — and bucket + // adjacency holds in any window's sorted schedule because true duplicates land in the + // same bucket of every window. So we Phase A on the very first batch's window-0 + // schedule, populate `dedup_state.{redirect_lookup, extra_points}` once, and reuse the + // result for every subsequent batch. + bool phase_a_done = false; + + auto run_batch = [&](size_t batch_start, size_t windows_in_batch, size_t B_R) noexcept { + // Per-(w, t) slot stride uses `B_eff` = max(num_buckets, B_lo, B_hi); each call + // iterates only the region's first B_R entries. The arena was sized for B_eff per slot. + const size_t bucket_stride = B_eff; + // Per-window slice params. The final window can be narrower when the bit budget + // does not divide evenly by the default window size; the Booth recoder must use + // that narrower width or it encroaches on bits beyond the schedule. + constexpr size_t SCALAR_UINT64_LIMBS = sizeof(ScalarField) / sizeof(uint64_t); + std::array slice_params{}; + std::array slice_params_u32{}; + std::array slice_paths{}; + std::array lo_mask_vectors{}; + std::array hi_mask_vectors{}; + std::array val_mask_vectors{}; + std::array per_window_bits{}; + constexpr size_t SCALAR_U32_LIMBS = sizeof(ScalarField) / sizeof(uint32_t); + for (size_t w = 0; w < windows_in_batch; ++w) { + const size_t global_w = batch_start + w; + const size_t window_bits_w = sched.window_bits_per_window[global_w]; + per_window_bits[w] = static_cast(window_bits_w); + slice_params[w] = round_parallel_detail::compute_constantine_slice_params( + sched.bit_base[global_w], window_bits_w, SCALAR_UINT64_LIMBS); + slice_params_u32[w] = round_parallel_detail::compute_constantine_slice_params_u32( + sched.bit_base[global_w], window_bits_w, SCALAR_U32_LIMBS); + slice_paths[w] = round_parallel_detail::classify_slice_path_u32(slice_params_u32[w]); + const uint32_t lo_mask = slice_params_u32[w].lo_mask; + const uint32_t hi_mask = slice_params_u32[w].hi_mask; + const uint32_t val_mask = (uint32_t{ 1 } << static_cast(window_bits_w)) - 1; + lo_mask_vectors[w] = round_parallel_detail::SimdU32x4{ lo_mask, lo_mask, lo_mask, lo_mask }; + hi_mask_vectors[w] = round_parallel_detail::SimdU32x4{ hi_mask, hi_mask, hi_mask, hi_mask }; + val_mask_vectors[w] = round_parallel_detail::SimdU32x4{ val_mask, val_mask, val_mask, val_mask }; + } + + constexpr size_t SIMD_BATCH = 64; + static_assert(SIMD_BATCH % 4 == 0, "SIMD_BATCH must be divisible by 4"); + constexpr size_t LIMBS_PER_SCALAR = sizeof(ScalarField) / sizeof(uint32_t); + const auto* scalars_u32 = reinterpret_cast(scalars.data()); + const round_parallel_detail::SimdU32x4 one_v = round_parallel_detail::SimdU32x4{ 1, 1, 1, 1 }; + auto fill_packed_digit_buffer = [&](size_t w, size_t i, uint32_t* packed_buf) noexcept { + const auto& sp32 = slice_params_u32[w]; + const uint32_t window_bits_w = static_cast(per_window_bits[w]); + if (slice_paths[w] == round_parallel_detail::ConstantineSlicePath::Localised) { + for (size_t k = 0; k < SIMD_BATCH; k += 4) { + round_parallel_detail::store_constantine_packed_digits_x4_localised( + packed_buf + k, + scalars_u32 + ((i + k + 0) * LIMBS_PER_SCALAR), + scalars_u32 + ((i + k + 1) * LIMBS_PER_SCALAR), + scalars_u32 + ((i + k + 2) * LIMBS_PER_SCALAR), + scalars_u32 + ((i + k + 3) * LIMBS_PER_SCALAR), + sp32.lo_limb, + sp32.lo_off, + lo_mask_vectors[w], + one_v, + val_mask_vectors[w], + window_bits_w); + } + } else if (slice_paths[w] == round_parallel_detail::ConstantineSlicePath::Bottom) { + for (size_t k = 0; k < SIMD_BATCH; k += 4) { + round_parallel_detail::store_constantine_packed_digits_x4_bottom( + packed_buf + k, + scalars_u32 + ((i + k + 0) * LIMBS_PER_SCALAR), + scalars_u32 + ((i + k + 1) * LIMBS_PER_SCALAR), + scalars_u32 + ((i + k + 2) * LIMBS_PER_SCALAR), + scalars_u32 + ((i + k + 3) * LIMBS_PER_SCALAR), + sp32.hi_limb, + sp32.lo_bits, + hi_mask_vectors[w], + one_v, + val_mask_vectors[w], + window_bits_w); + } + } else { + for (size_t k = 0; k < SIMD_BATCH; k += 4) { + round_parallel_detail::store_constantine_packed_digits_x4_boundary( + packed_buf + k, + scalars_u32 + ((i + k + 0) * LIMBS_PER_SCALAR), + scalars_u32 + ((i + k + 1) * LIMBS_PER_SCALAR), + scalars_u32 + ((i + k + 2) * LIMBS_PER_SCALAR), + scalars_u32 + ((i + k + 3) * LIMBS_PER_SCALAR), + sp32.lo_limb, + sp32.hi_limb, + sp32.lo_off, + sp32.lo_bits, + lo_mask_vectors[w], + hi_mask_vectors[w], + one_v, + val_mask_vectors[w], + window_bits_w); + } + } + }; + + // Capture the dedup state before Stage 1. The first batch must build the ordinary + // R14 schedule so Phase A can discover clusters, then patch+compact that batch. + // Later batches can schedule cluster reps directly and omit non-reps up front. + const bool phase_a_done_at_batch_start = phase_a_done; + const bool dedup_known_for_batch = + dedup_active && phase_a_done_at_batch_start && dedup_state.n_dedup_extras != 0; + + // Stage 1 (digit extraction): per-thread per-window bucket histograms. Work is + // scalar-blocked across the windows in this batch so scalars/msb/dedup metadata are + // read once per block and reused while still hot. + auto stage1_digit_extract = [&](size_t tid) noexcept { + BB_BENCH_NAME("MSM_fast::Stage1_digit_extract/worker"); + [[maybe_unused]] const uint32_t* const rl_data = dedup_state.redirect_lookup.data(); + for (size_t w = 0; w < windows_in_batch; ++w) { + uint32_t* my_counts = digit_cursors.data() + (((w * num_threads) + tid) * bucket_stride); + std::memset(my_counts, 0, B_R * sizeof(uint32_t)); + } + const size_t start = tid * n / num_threads; + const size_t end = (tid + 1) * n / num_threads; + + alignas(16) std::array packed_buf{}; + // Pack the per-block filter into a uint64 bitmask. When every scalar in the block + // is active (common in dense workloads), the inner scatter takes an all_included + // fast path that drops the per-element predicate; mixed blocks bit-scan the mask. + auto compute_include_mask = [&](size_t block_start) noexcept -> uint64_t { + uint64_t include_mask = 0; + for (size_t k = 0; k < SIMD_BATCH; ++k) { + const size_t scalar_idx = block_start + k; + const uint8_t m = msb_per_scalar[scalar_idx]; + bool include = (m != MSB_ZERO_SENTINEL); + if constexpr (DedupKnown) { + if (include) { + const uint32_t patch = rl_data[scalar_idx]; + include = (patch == round_parallel_detail::DEDUP_INVALID_EXTRA || + (patch & round_parallel_detail::DEDUP_SKIP_BIT) == 0); + } + } + include_mask |= static_cast(include) << k; + } + return include_mask; + }; + + size_t i = start; + while (i + SIMD_BATCH <= end) { + const uint64_t include_mask = compute_include_mask(i); + if (include_mask == 0) { + i += SIMD_BATCH; + continue; + } + const bool all_included = include_mask == ~uint64_t{ 0 }; + for (size_t w = 0; w < windows_in_batch; ++w) { + fill_packed_digit_buffer(w, i, packed_buf.data()); + uint32_t* my_counts = digit_cursors.data() + (((w * num_threads) + tid) * bucket_stride); + if (all_included) { + for (size_t k = 0; k < SIMD_BATCH; ++k) { + ++my_counts[packed_buf[k] & BUCKET_MASK]; + } + } else { + uint64_t scatter_mask = include_mask; + for (size_t k = 0; k < SIMD_BATCH; ++k) { + if ((scatter_mask & uint64_t{ 1 }) != 0) { + ++my_counts[packed_buf[k] & BUCKET_MASK]; + } + scatter_mask >>= 1; + } + } + } + i += SIMD_BATCH; + } + + // Tail (0..SIMD_BATCH-1 scalars). Same scalar-major loop order; per-scalar + // active check inlined since the block is short. + for (; i < end; ++i) { + const uint8_t m = msb_per_scalar[i]; + if (m == MSB_ZERO_SENTINEL) { + continue; + } + if constexpr (DedupKnown) { + const uint32_t patch = rl_data[i]; + if (patch != round_parallel_detail::DEDUP_INVALID_EXTRA && + (patch & round_parallel_detail::DEDUP_SKIP_BIT) != 0) { + continue; + } + } + for (size_t w = 0; w < windows_in_batch; ++w) { + uint32_t* my_counts = digit_cursors.data() + (((w * num_threads) + tid) * bucket_stride); + const round_parallel_detail::ConstantineSliceParams sp = slice_params[w]; + const uint32_t window_bits_w = static_cast(per_window_bits[w]); + const uint32_t packed = + round_parallel_detail::get_constantine_packed_digit(scalars[i].data, + sp.lo_limb, + sp.hi_limb, + sp.lo_off, + sp.lo_bits, + sp.lo_mask, + sp.hi_mask, + sp.slice_localised_to_one_u64, + window_bits_w); + ++my_counts[packed & BUCKET_MASK]; + } + } + }; + if (dedup_known_for_batch) { + bb::parallel_for(num_threads, [&](size_t tid) { stage1_digit_extract.template operator()(tid); }); + } else { + bb::parallel_for(num_threads, [&](size_t tid) { stage1_digit_extract.template operator()(tid); }); + } + + // Stage 2 (bucket histogram): per-window per-digit totals + per-thread within-digit + // offsets. Parallelised over digit-chunks; each worker handles its slice of 2^window_bits + // for all windows_in_batch windows. In-place exclusive prefix-sum: each slot + // `digit_cursors[(w*T + t) * stride + d]` is read for its Stage 1 count and then + // overwritten with the running prefix sum (== the cursor base Stage 4 needs). The + // count must be read BEFORE the write or `running` would skip its contribution. + // Phase 5: the per-digit total `running` is written directly into + // `bucket_start_all[w][d+1]` (one cell past where Stage 3 will read), so Stage 3 can + // prefix-sum in place without a separate `bucket_total_counts` buffer. The size_t + // bucket_start cell widens the uint32_t total implicitly. + bb::parallel_for(num_threads, [&](size_t tid) { + BB_BENCH_NAME("MSM_fast::Stage2_bucket_histogram/worker"); + const size_t d_start = tid * B_R / num_threads; + const size_t d_end = (tid + 1) * B_R / num_threads; + for (size_t w = 0; w < windows_in_batch; ++w) { + size_t* const bucket_start_w = bucket_start_all.data() + (w * (bucket_stride + 1)); + for (size_t d = d_start; d < d_end; ++d) { + if (d == 0) { + continue; + } + uint32_t running = 0; + for (size_t t = 0; t < num_threads; ++t) { + const size_t k = (((w * num_threads) + t) * bucket_stride) + d; + const uint32_t cnt = digit_cursors[k]; + digit_cursors[k] = running; + running += cnt; + } + bucket_start_w[d + 1] = running; + } + } + }); + + // Stage 3 (bucket offsets / prefix sum): per-window serial prefix sum in place. + // Stage 2 already deposited each digit's per-window total at bucket_start[d+1]; + // the loop accumulates the running prefix-sum without a separate counts buffer. + { + BB_BENCH_NAME("MSM_fast::Stage2_3_bucket_offsets"); + auto build_bucket_offsets_for_window = [&](size_t w) noexcept { + size_t* bucket_start = bucket_start_all.data() + (w * (bucket_stride + 1)); + bucket_start[0] = 0; + bucket_start[1] = 0; + for (size_t d = 1; d < B_R; ++d) { + bucket_start[d + 1] += bucket_start[d]; + } + }; + const size_t offset_threads = std::min(num_threads, windows_in_batch); + if (offset_threads <= 1) { + for (size_t w = 0; w < windows_in_batch; ++w) { + build_bucket_offsets_for_window(w); + } + } else { + bb::parallel_for(offset_threads, [&](size_t tid) { + BB_BENCH_NAME("MSM_fast::Stage2_3_bucket_offsets/worker"); + for (size_t w = tid; w < windows_in_batch; w += offset_threads) { + build_bucket_offsets_for_window(w); + } + }); + } + } + + // Stage 4 (digit scatter): scalar-cache-blocked, window-local scatter. Re-decodes each + // (point, window) signed digit via the same Constantine carry-less recoder Stage 1 used. + // Stage 4 stores only `sign | scalar_idx`; bucket magnitude is recovered later from + // bucket_start ranges. + // Stage 1 benefits from full scalar-major order because it only updates compact + // per-window histograms. Stage 4 writes large bucket schedules, so full scalar-major + // order opens too many cold write/cursor streams. Instead, process a scalar tile across + // all windows: scalar/msb/dedup metadata are reused while the tile is cache-hot, but each + // inner loop still scatters to one window's schedule at a time. + // + // First-batch Stage 4 is dedup-unaware: every scalar is emitted as + // `sched_w[idx] = sign | scalar_idx`, then Phase A + patch/compact tags cluster + // reps and removes non-reps. Later batches with known dedup state skip non-reps + // here and emit redirect reps directly. + // Splitting the dedup work out of this hot loop avoids a per-iteration + // closure-indirection chain through `dedup_state.redirect_lookup[i]` + // that the WASM JIT does not hoist (~13 ns/iter penalty observed). + auto stage4_emit = [&](size_t tid) noexcept { + [[maybe_unused]] const uint32_t* const rl_data = dedup_state.redirect_lookup.data(); + const size_t start = tid * n / num_threads; + const size_t end = (tid + 1) * n / num_threads; + std::array cursors{}; + std::array bucket_starts{}; + std::array schedules{}; + for (size_t w = 0; w < windows_in_batch; ++w) { + cursors[w] = digit_cursors.data() + (((w * num_threads) + tid) * bucket_stride); + bucket_starts[w] = bucket_start_all.data() + (w * (bucket_stride + 1)); + schedules[w] = schedule.data() + (w * n); + } + + alignas(16) std::array packed_buf{}; + constexpr size_t STAGE4_SCALAR_TILE = 2048; + std::array active_tile{}; + [[maybe_unused]] std::array out_base_tile{}; + + for (size_t tile_start = start; tile_start < end; tile_start += STAGE4_SCALAR_TILE) { + const size_t tile_end = std::min(end, tile_start + STAGE4_SCALAR_TILE); + const size_t tile_len = tile_end - tile_start; + for (size_t j = 0; j < tile_len; ++j) { + const size_t scalar_idx = tile_start + j; + const uint8_t m = msb_per_scalar[scalar_idx]; + bool include = (m != MSB_ZERO_SENTINEL); + if constexpr (DedupKnown) { + uint32_t out_base = static_cast(scalar_idx); + if (include) { + const uint32_t patch = rl_data[scalar_idx]; + if (patch != round_parallel_detail::DEDUP_INVALID_EXTRA) { + include = (patch & round_parallel_detail::DEDUP_SKIP_BIT) == 0; + out_base = patch; + } + } + out_base_tile[j] = out_base; + } + active_tile[j] = static_cast(include); + } + + for (size_t w = 0; w < windows_in_batch; ++w) { + uint32_t* my_cursor = cursors[w]; + const size_t* bucket_start = bucket_starts[w]; + uint32_t* sched_w = schedules[w]; + size_t i = tile_start; + while (i + SIMD_BATCH <= tile_end) { + const size_t rel = i - tile_start; + uint64_t include_mask = 0; + for (size_t k = 0; k < SIMD_BATCH; ++k) { + include_mask |= static_cast(active_tile[rel + k]) << k; + } + if (include_mask == 0) { + i += SIMD_BATCH; + continue; + } + fill_packed_digit_buffer(w, i, packed_buf.data()); + uint64_t scatter_mask = include_mask; + for (size_t k = 0; k < SIMD_BATCH; ++k) { + if ((scatter_mask & uint64_t{ 1 }) != 0) { + const uint32_t packed = packed_buf[k]; + const uint32_t bucket_idx = packed & BUCKET_MASK; + if (bucket_idx != 0) { + const uint32_t idx = + static_cast(bucket_start[bucket_idx]) + my_cursor[bucket_idx]++; + uint32_t out = packed & round_parallel_detail::SCHEDULE_SIGN_BIT; + if constexpr (DedupKnown) { + out |= out_base_tile[rel + k]; + } else { + out |= static_cast(i + k); + } + sched_w[idx] = out; + } + } + scatter_mask >>= 1; + } + i += SIMD_BATCH; + } + for (; i < tile_end; ++i) { + const size_t rel = i - tile_start; + if (active_tile[rel] == 0) { + continue; + } + const round_parallel_detail::ConstantineSliceParams sp = slice_params[w]; + const uint32_t packed = round_parallel_detail::get_constantine_packed_digit( + scalars[i].data, + sp.lo_limb, + sp.hi_limb, + sp.lo_off, + sp.lo_bits, + sp.lo_mask, + sp.hi_mask, + sp.slice_localised_to_one_u64, + static_cast(per_window_bits[w])); + const uint32_t bucket_idx = packed & BUCKET_MASK; + if (bucket_idx != 0) { + const uint32_t idx = + static_cast(bucket_start[bucket_idx]) + my_cursor[bucket_idx]++; + uint32_t out = packed & round_parallel_detail::SCHEDULE_SIGN_BIT; + if constexpr (DedupKnown) { + out |= out_base_tile[rel]; + } else { + out |= static_cast(i); + } + sched_w[idx] = out; + } + } + } + } + }; + + if (dedup_known_for_batch) { + bb::parallel_for(num_threads, [&](size_t tid) { + BB_BENCH_NAME("MSM_fast::Stage4_digit_scatter/worker"); + stage4_emit.template operator()(tid); + }); + } else { + bb::parallel_for(num_threads, [&](size_t tid) { + BB_BENCH_NAME("MSM_fast::Stage4_digit_scatter/worker"); + stage4_emit.template operator()(tid); + }); + } + + // Phase A: schedule-based dedup detection on window 0. Each thread owns a + // contiguous range of window 0's schedule. Detects duplicate clusters via + // consecutive-pair check (same bucket + memcmp on full scalar value), tree-reduces + // members into an aggregate, and publishes results into `dedup_state.extra_points`, + // `dedup_state.redirect_lookup`, and zeroed `msb_per_scalar` entries for non-reps. + // Per-thread cluster-id ranges keep writes disjoint — no atomics needed. + // Phase A: schedule-based dedup detection. Runs at most ONCE per MSM_fast (gated on + // `phase_a_done` from the enclosing function scope). Cluster membership is decided + // by scalar value (memcmp), so any window's bucket-sorted schedule places duplicates + // consecutively — Phase A on this first-batch's window-0 schedule produces the + // correct redirect_lookup + extra_points for all subsequent batches. We deliberately + // do not re-run Phase A per batch: the dedup_state is populated once and reused. + if (dedup_active && windows_in_batch > 0 && !phase_a_done) { + BB_BENCH_NAME("MSM_fast::PhaseA_dedup_detect"); + uint32_t* sched_w0 = schedule.data(); + // Pre-Phase-A bucket sort: Stage 4 emits each bucket's run in scalar-emit + // order, so different-value scalars that happen to share a window-0 digit + // (bucket collisions are common — c=11 → 2048 buckets vs 60-90k entries) + // interleave with same-value entries and break Phase A's consecutive-pair + // detection. Sorting each bucket's run by scalar value makes same-value + // entries adjacent so the simple consecutive-pair walk finds every cluster. + // Sort cost: per bucket of size K, ~K log K comparisons × 32-byte memcmp; + // for typical K=44 this is ~500 cycles per bucket × 2048 buckets = ~1 ms + // wall (parallelized across threads). + const uint32_t cids_per_thread = + static_cast(round_parallel_detail::DEDUP_MAX_CLUSTERS / num_threads); + // Hash-based per-bucket dedup detection: every thread owns a + // contiguous bucket range of window-0's schedule and runs an + // open-addressing hash table over that range's long-scalar entries. + // O(K) per bucket, avoids the 32-byte memcmp comparator inside any + // sort, and keeps thread balance uniform because short-scalar + // entries (the source of mega-buckets like digit_0 = 1) are skipped. + // Catches ~99.94 % of long-scalar duplicates against MSM_DUMP's + // theoretical maximum (`dup_input_extras`). + { + BB_BENCH_NAME("MSM_fast::PhaseA_dedup_detect_hash"); + const size_t* const w0_bucket_start = bucket_start_all.data(); + std::atomic dedup_cluster_count{ 0 }; + bb::parallel_for(num_threads, [&, w0_bucket_start](size_t tid) noexcept { + BB_BENCH_NAME("MSM_fast::PhaseA_dedup_detect/worker"); + const size_t b_lo = 1 + ((tid * (B_R - 1)) / num_threads); + const size_t b_hi = 1 + (((tid + 1) * (B_R - 1)) / num_threads); + const uint32_t cid_lo = static_cast(tid) * cids_per_thread; + const uint32_t cid_max = cid_lo + cids_per_thread; + const size_t local_clusters = round_parallel_detail::dedup_phase_a_worker_hash( + sched_w0, + w0_bucket_start, + b_lo, + b_hi, + std::span(scalars.data(), n), + points, + std::span(dedup_state.extra_points), + std::span(dedup_state.redirect_lookup), + msb_per_scalar.data(), + window_bits, + cid_lo, + cid_max, + phase_a_scratch[tid]); + if (local_clusters != 0) { + dedup_cluster_count.fetch_add(local_clusters, std::memory_order_relaxed); + } + }); + dedup_state.n_dedup_extras = dedup_cluster_count.load(std::memory_order_relaxed); + } + phase_a_done = true; + } + + // Schedule patch post-pass: tags cluster-member entries with SKIP/REDIRECT bits. + // Runs only for the batch that just ran Phase A: later batches with known dedup + // state skip non-reps in Stage 1/4 and emit redirect reps directly. + // Parallel by window (one window per worker) because each window's slice of the + // schedule is disjoint. Hoisting `redirect_lookup.data()` to a raw pointer outside + // the lambda + passing it by value into the inner function avoids the per-iter + // closure-indirection chain that made the inline form 3× slower per iter on WASM. + auto partition_chunks_for_window = [&](size_t w) noexcept { + const size_t* bucket_start = bucket_start_all.data() + (w * (bucket_stride + 1)); + const size_t* const bucket_start_end = bucket_start + B_R + 1; + size_t* chunk_start = chunk_start_all.data() + (w * (num_threads + 1)); + size_t* chunk_bucket_lo = chunk_bucket_lo_all.data() + (w * (num_threads + 1)); + size_t* chunk_bucket_hi = chunk_bucket_hi_all.data() + (w * num_threads); + const size_t m = bucket_start[B_R]; + const size_t* search_begin = bucket_start + 1; + size_t lo = 0; + chunk_start[0] = lo; + for (size_t t = 0; t < num_threads; ++t) { + const size_t hi = ((t + 1) == num_threads) ? m : (((t + 1) * m) / num_threads); + chunk_start[t + 1] = hi; + if (lo < hi) { + const size_t* const lo_it = std::upper_bound(search_begin, bucket_start_end, lo); + const size_t lo_bucket = static_cast(lo_it - bucket_start - 1); + const size_t* const hi_it = std::upper_bound(lo_it, bucket_start_end, hi - 1); + const size_t hi_bucket = static_cast(hi_it - bucket_start - 1); + chunk_bucket_lo[t] = lo_bucket; + chunk_bucket_hi[t] = hi_bucket; + search_begin = hi_it; + } else { + chunk_bucket_lo[t] = B_R; + chunk_bucket_hi[t] = 0; + } + lo = hi; + } + chunk_bucket_lo[num_threads] = B_R; + }; + + bool chunk_partition_done = false; + if (dedup_active && windows_in_batch > 0 && phase_a_done && !phase_a_done_at_batch_start) { + BB_BENCH_NAME("MSM_fast::dedup_patch_schedule"); + const uint32_t* const rl_data = dedup_state.redirect_lookup.data(); + const size_t bs_stride = bucket_stride + 1; + const size_t br = B_R; + const size_t cap_R = n; + bb::parallel_for(num_threads, [&, rl_data, bs_stride, br, cap_R](size_t tid) noexcept { + BB_BENCH_NAME("MSM_fast::dedup_patch_schedule/worker"); + for (size_t w = tid; w < windows_in_batch; w += num_threads) { + uint32_t* sched_w = schedule.data() + (w * cap_R); + size_t* bucket_start_w = bucket_start_all.data() + (w * bs_stride); + round_parallel_detail::dedup_patch_schedule_window(sched_w, bucket_start_w, br, rl_data); + partition_chunks_for_window(w); + } + }); + chunk_partition_done = true; + } + + // Per-window chunk partition at schedule-index granularity (chunk_start[t] = t·m/T). + // Balances across threads regardless of bucket-distribution skew. When the partition + // lands mid-bucket, both adjacent threads build their own partial into the boundary + // bucket; chunk_contribution combines them in Stage 7. + { + BB_BENCH_NAME("MSM_fast::Stage5_chunk_partition"); + if (!chunk_partition_done) { + for (size_t w = 0; w < windows_in_batch; ++w) { + partition_chunks_for_window(w); + } + } + } + + // Stage 6 bucket accumulation per thread: + // (1) For each window w: reduce_chunk emits a digit-sorted (point, digit) list, + // which we densify into a per-window dense bucket array at + // tid's affine bucket buffer + w * stride. Empty slots stay identity. + // (2) Call recursive_affine_bucket_reduce_strided once across all windows_in_batch + // chunks; it computes (R_w, L_w) for each non-empty chunk via batch-affine + // arithmetic, amortising the inversion across windows at every phase step. + // (3) chunk_contribution(out) folds L_w + (lo_w-1)·R_w into the thread's per-window + // partial. + // The Stage-6 scratch is pre-sized for every thread BEFORE entering the parallel_for + // so the per-thread vector resizes don't race the heap allocator. + auto next_pow2 = [](size_t x) -> size_t { + if (x <= 1) { + return 1; + } + size_t p = 1; + while (p < x) { + p <<= 1; + } + return p; + }; + // Drives reduce_chunk's per-thread tree-reduce buffer sizing. + size_t max_chunk_len = 0; + for (size_t t = 0; t < num_threads; ++t) { + for (size_t w = 0; w < windows_in_batch; ++w) { + const size_t* chunk_start = chunk_start_all.data() + (w * (num_threads + 1)); + const size_t entries_in_chunk = chunk_start[t + 1] - chunk_start[t]; + if (entries_in_chunk == 0) { + continue; + } + max_chunk_len = std::max(max_chunk_len, entries_in_chunk); + } + } + + // global_stride drives the per-thread `dense_buckets` layout (sized via + // `ensure_affine_bucket_capacity` below). Stage 6a writes its per-thread bucket + // partials into `bucket_partials_dense` (a separate buffer packed via + // `bucket_partials_offsets`, no power-of-two stride); Stage 6b copies them into + // `s.dense_buckets` keyed by Stage 6b's uniform bucket-index slice of width + // `buckets_per_task ≈ ⌈(num_buckets-1)/T⌉`. The recursive bucket-reduction + // algorithm (phases A-D) operates on `s.dense_buckets` with power-of-two row + // stride — that's where `next_pow2` matters. + size_t global_stride = 0; + + { + // Stage 6b's bucket-balanced partition. Uniform across windows: each rebalanced + // task t' owns active digits [d_lo'[t'], d_hi'[t']] where d_lo'[t'] = 1 + t · (B-1) / T. + const size_t active_digits = (B_R > 0) ? (B_R - 1) : 0; + for (size_t t = 0; t <= num_threads; ++t) { + rebalanced_bucket_lo_partition[t] = 1 + (t * active_digits) / num_threads; + } + rebalanced_bucket_lo_partition[num_threads] = B_R; + size_t max_buckets_per_task = 0; + for (size_t t = 0; t + 1 <= num_threads; ++t) { + const size_t hi_d = (t + 1 == num_threads) ? (B_R - 1) : (rebalanced_bucket_lo_partition[t + 1] - 1); + const size_t lo_d = rebalanced_bucket_lo_partition[t]; + if (hi_d >= lo_d) { + max_buckets_per_task = std::max(max_buckets_per_task, hi_d - lo_d + 1); + } + } + global_stride = next_pow2(max_buckets_per_task); + global_stride = std::max(global_stride, 2); + + // Per-window orig-thread contributing ranges (O(W·T·T) total — only paid for + // the rebalance path, where T is small enough that this is sub-µs). + for (size_t w = 0; w < windows_in_batch; ++w) { + const size_t* chunk_bucket_lo = chunk_bucket_lo_all.data() + (w * (num_threads + 1)); + const size_t* chunk_bucket_hi = chunk_bucket_hi_all.data() + (w * num_threads); + const size_t* chunk_start_w = chunk_start_all.data() + (w * (num_threads + 1)); + for (size_t tprime = 0; tprime < num_threads; ++tprime) { + const size_t lo_d = rebalanced_bucket_lo_partition[tprime]; + const size_t hi_d = + (tprime + 1 == num_threads) ? (B_R - 1) : (rebalanced_bucket_lo_partition[tprime + 1] - 1); + size_t lo_orig = num_threads; + size_t hi_orig = 0; + for (size_t t = 0; t < num_threads; ++t) { + const size_t entries = chunk_start_w[t + 1] - chunk_start_w[t]; + if (entries == 0) { + continue; + } + const size_t cl = chunk_bucket_lo[t]; + const size_t ch = chunk_bucket_hi[t]; + if (ch < lo_d || cl > hi_d) { + continue; + } + if (lo_orig == num_threads) { + lo_orig = t; + } + hi_orig = t; + } + orig_thread_lo[(w * num_threads) + tprime] = lo_orig; + orig_thread_hi[(w * num_threads) + tprime] = hi_orig; + } + } + + // bucket_partials_dense / _present packed via bucket_partials_offsets — each + // (thread, window) row holds exactly buckets_per_thread[t][w] AffineElements (no + // padding). The arena pre-sized to `windows_per_batch · (num_buckets - 1 + T)` + // (covers the T-1 boundary-bucket shares); only the actual prefix is touched. + size_t bucket_partials_cursor = 0; + for (size_t t = 0; t < num_threads; ++t) { + for (size_t w = 0; w < windows_in_batch; ++w) { + bucket_partials_offsets[(t * windows_in_batch) + w] = bucket_partials_cursor; + const size_t* chunk_bucket_lo_w = chunk_bucket_lo_all.data() + (w * (num_threads + 1)); + const size_t* chunk_bucket_hi_w = chunk_bucket_hi_all.data() + (w * num_threads); + const size_t* chunk_start_w = chunk_start_all.data() + (w * (num_threads + 1)); + const size_t entries = chunk_start_w[t + 1] - chunk_start_w[t]; + if (entries > 0) { + bucket_partials_cursor += chunk_bucket_hi_w[t] - chunk_bucket_lo_w[t] + 1; + } + } + } + bucket_partials_offsets[num_threads * windows_in_batch] = bucket_partials_cursor; + const size_t bucket_partials_total = bucket_partials_cursor; + BB_ASSERT_LTE(bucket_partials_total, bucket_partials_dense.size()); + std::memset(bucket_partials_present.data(), 0, bucket_partials_total); + } + + // thread_scratch is worker-indexed (one slot per OS thread, FIFO-shared by tasks); + // update the stride on each worker's slot. + for (size_t t = 0; t < worker_total; ++t) { + thread_scratch[t].affine_bucket_stride = global_stride; + } + + { + // Stage 6a — per-thread bucket partials. Each thread `tid` reduces its schedule + // slice via reduce_chunk and scatters the (digit, point) output directly into the + // per-thread dense bucket buffer at slot `(digit - chunk_bucket_lo[tid])`. Stage + // 6b then reads this buffer with O(1) slot lookup. `bucket_partials_present` is + // pre-zeroed per batch. + auto bucket_partials_per_thread_lambda = [&](size_t tid) { + BB_BENCH_NAME("MSM_fast::Stage6a_bucket_partials/worker"); + auto& s = thread_scratch[tid]; + for (size_t w = 0; w < windows_in_batch; ++w) { + const size_t* chunk_start_w = chunk_start_all.data() + (w * (num_threads + 1)); + const size_t cs_lo = chunk_start_w[tid]; + const size_t cs_hi = chunk_start_w[tid + 1]; + if (cs_lo == cs_hi) { + continue; + } + const uint32_t* sched_w = schedule.data() + (w * n); + const size_t* bucket_start = bucket_start_all.data() + (w * (bucket_stride + 1)); + AffineElement* dst_dense = + bucket_partials_dense.data() + bucket_partials_offsets[(tid * windows_in_batch) + w]; + uint8_t* dst_present = + bucket_partials_present.data() + bucket_partials_offsets[(tid * windows_in_batch) + w]; + const size_t* chunk_bucket_lo = chunk_bucket_lo_all.data() + (w * (num_threads + 1)); + const uint32_t my_lo = static_cast(chunk_bucket_lo[tid]); + const size_t my_hi = chunk_bucket_hi_all[(w * num_threads) + tid]; + size_t bucket_cursor = my_lo; + + for (size_t pos = cs_lo; pos < cs_hi;) { + const size_t end = std::min(pos + SUBCHUNK_ENTRIES_CAP, cs_hi); + reduce_chunk(s, + sched_w, + bucket_start, + pos, + end, + bucket_cursor, + my_hi, + points, + std::span(dedup_state.extra_points)); + const size_t len = s.result_len; + for (size_t k = 0; k < len; ++k) { + const uint32_t d = s.curr_buckets[k]; + const size_t slot = d - my_lo; + if (dst_present[slot]) { + s.overflow_slots[s.overflow_len] = static_cast(slot); + s.overflow_pts[s.overflow_len] = s.curr_pts[k]; + ++s.overflow_len; + } else { + dst_dense[slot] = s.curr_pts[k]; + dst_present[slot] = 1; + } + } + pos = end; + } + merge_overflow(s, dst_dense); + } + }; + + // Stage 6b (cross-thread bucket reduction): each rebalanced task `tprime` owns a + // uniform-width slice of the bucket-index space [d_lo'(tprime), d_hi'(tprime)]. + // For each window in the batch, walk the contributing original threads' Stage 6a + // dense outputs (range [orig_thread_lo, orig_thread_hi]), filter to digits in + // this task's slice, scatter into the task's local dense_buckets (with + // projective-add accumulation on the at-most-2 boundary digits per pair of + // contributing originals), then run recursive_affine_bucket_reduce_strided + + // chunk_contribution on a guaranteed-equal buckets_padded across all tasks. + auto bucket_reduce_cross_thread_lambda = [&](size_t tprime) { + BB_BENCH_NAME("MSM_fast::Stage6b_reduce_cross_thread/worker"); + auto& s = thread_scratch[tprime]; + Element* my_partials = window_partial_sums.data() + (tprime * windows_per_batch); + for (size_t w = 0; w < windows_in_batch; ++w) { + my_partials[w] = Curve::Group::point_at_infinity; + } + + const size_t stride = s.affine_bucket_stride; + std::memset(s.is_present.data(), 0, windows_in_batch * stride); + + const size_t lo_d = rebalanced_bucket_lo_partition[tprime]; + const size_t hi_d = + (tprime + 1 == num_threads) ? (B_R - 1) : (rebalanced_bucket_lo_partition[tprime + 1] - 1); + const uint32_t lo_d_u = static_cast(lo_d); + const uint32_t hi_d_u = static_cast(hi_d); + + bool any_nonempty = false; + for (size_t w = 0; w < windows_in_batch; ++w) { + auto& info = s.chunk_infos[w]; + auto& out = chunk_outputs[(w * num_threads) + tprime]; + if (lo_d > hi_d) { + info.empty = 1; + info.lo = 0; + info.hi = 0; + info.buckets_padded = 0; + out.empty = 1; + continue; + } + const size_t orig_lo = orig_thread_lo[(w * num_threads) + tprime]; + const size_t orig_hi = orig_thread_hi[(w * num_threads) + tprime]; + if (orig_lo == num_threads) { + info.empty = 1; + info.lo = 0; + info.hi = 0; + info.buckets_padded = 0; + out.empty = 1; + continue; + } + const size_t base = w * stride; + bool has_data = false; + + // bucket_partials_dense holds per-(orig_t, w, slot) bucket points with + // bucket_partials_present as the populated-slot bitmap. For each + // contributing orig_t, intersect its [chunk_bucket_lo, chunk_bucket_hi] + // range with this task's [lo_d, hi_d] slice and walk the intersection + // only — no sorted scan, O(1) lookup per slot. + const size_t* chunk_bucket_lo_w = chunk_bucket_lo_all.data() + (w * (num_threads + 1)); + const size_t* chunk_bucket_hi_w = chunk_bucket_hi_all.data() + (w * num_threads); + for (size_t t = orig_lo; t <= orig_hi; ++t) { + const size_t cl = chunk_bucket_lo_w[t]; + const size_t ch = chunk_bucket_hi_w[t]; + const size_t d_lo_clip = std::max(lo_d, cl); + const size_t d_hi_clip = std::min(hi_d, ch); + if (d_lo_clip > d_hi_clip) { + continue; + } + const AffineElement* src_dense = + bucket_partials_dense.data() + bucket_partials_offsets[(t * windows_in_batch) + w]; + const uint8_t* src_present = + bucket_partials_present.data() + bucket_partials_offsets[(t * windows_in_batch) + w]; + for (size_t d = d_lo_clip; d <= d_hi_clip; ++d) { + const size_t src_slot = d - cl; + if (src_present[src_slot] == 0) { + continue; + } + const size_t dst_slot = base + (d - lo_d); + if (s.is_present[dst_slot] == 0) { + s.dense_buckets[dst_slot] = src_dense[src_slot]; + s.is_present[dst_slot] = 1; + } else { + // Boundary digit shared between two consecutive originals + // — projective add then re-normalise to affine. Under the + // contiguous-by-schedule-index partition there are at most + // W boundary points per task. + Element acc = Element(s.dense_buckets[dst_slot]); + acc += Element(src_dense[src_slot]); + s.dense_buckets[dst_slot] = AffineElement(acc); + } + has_data = true; + } + } + if (!has_data) { + info.empty = 1; + info.lo = 0; + info.hi = 0; + info.buckets_padded = 0; + out.empty = 1; + continue; + } + any_nonempty = true; + const size_t M = hi_d - lo_d + 1; + const uint32_t buckets_padded = + (M == 1) ? 1 : (uint32_t{ 1 } << (32 - __builtin_clz(static_cast(M - 1)))); + info.empty = 0; + info.lo = lo_d_u; + info.hi = hi_d_u; + info.buckets_padded = buckets_padded; + out.empty = 0; + out.lo = lo_d_u; + out.hi = hi_d_u; + } + + if (!any_nonempty) { + return; + } + + round_parallel_detail::recursive_affine_bucket_reduce_strided( + s, s.chunk_infos.data(), windows_in_batch, chunk_outputs.data() + tprime, num_threads); + + for (size_t w = 0; w < windows_in_batch; ++w) { + auto& out = chunk_outputs[(w * num_threads) + tprime]; + if (out.empty == 0) { + my_partials[w] = round_parallel_detail::chunk_contribution(out); + } + } + }; + + bb::parallel_for(num_threads, bucket_partials_per_thread_lambda); + bb::parallel_for(num_threads, bucket_reduce_cross_thread_lambda); + } + + // Stage 7 (cross-window combine): per-window reduce of `num_threads` per-thread partials. + // (Algebraic identity: `Σ_t (L_t + (lo_t − 1) · R_t) = window's bucket sum`, + // with the per-chunk contributions already accumulated above.) + { + const size_t reduce_threads = std::min(num_threads, windows_in_batch); + bb::parallel_for(reduce_threads, [&](size_t rid) { + BB_BENCH_NAME("MSM_fast::Stage7_combine/worker"); + const size_t lo = rid * windows_in_batch / reduce_threads; + const size_t hi = (rid + 1) * windows_in_batch / reduce_threads; + for (size_t w = lo; w < hi; ++w) { + Element sum = Curve::Group::point_at_infinity; + for (size_t tid = 0; tid < num_threads; ++tid) { + sum += window_partial_sums[(tid * windows_per_batch) + w]; + } + window_sums[batch_start + w] = sum; + } + }); + } + }; + + // Uniform-schedule dispatch over all windows. + { + const size_t B_R = (size_t{ 1 } << (window_bits - 1)) + 1; + for (size_t batch_start = 0; batch_start < sched.num_windows; batch_start += windows_per_batch) { + const size_t windows_in_batch = std::min(windows_per_batch, sched.num_windows - batch_start); + run_batch(batch_start, windows_in_batch, B_R); + } + } + + // Stage 7 horner: walk high-to-low, doubling by `window_bits_per_window[w]` between adjacent windows. + // Init from the top window to skip a wasted doubling on identity. + Element result = (sched.num_windows == 0) ? Curve::Group::point_at_infinity : window_sums[sched.num_windows - 1]; + for (size_t w_rev = sched.num_windows - 1; w_rev > 0; --w_rev) { + const size_t window_bits_w = sched.window_bits_per_window[w_rev - 1]; + for (size_t d = 0; d < window_bits_w; ++d) { + result.self_dbl(); + } + result += window_sums[w_rev - 1]; + } + + // GLV path leaves input_scalars untouched (it reads via from_montgomery_form_reduced into + // a temporary). Non-GLV path mutated in place above and must restore. + if (!use_glv) { + bb::parallel_for(bb::get_num_cpus(), [&](const ThreadChunk& chunk) { + BB_BENCH_NAME("MSM_fast::to_montgomery/worker"); + for (size_t i : chunk.range(n_input)) { + input_scalars[i].self_to_montgomery_form(); + } + }); + } + + return result; +} + +template +typename Curve::Element pippenger_unsafe_fast(PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept +{ + return pippenger_round_parallel(scalars, points, dedup_hint); +} + +template +typename Curve::Element pippenger_fast(PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept +{ + using Element = typename Curve::Element; + using ScalarField = typename Curve::ScalarField; + if (!handle_edge_cases) { + return pippenger_round_parallel(scalars, points, dedup_hint); + } + // Edge-case-handling path: route through the Jacobian fast-path. It uses + // Jacobian additions throughout, so point-at-infinity and equal-x bucket + // collisions don't trigger the affine-add edge-case bug. We need to convert + // PolynomialSpan to a plain ScalarField span: the jacobian fast-path takes + // a contiguous std::span and ignores `start_index`. + const size_t n = scalars.span.size(); + if (n == 0) { + return Curve::Group::point_at_infinity; + } + // Trivially small N: skip Pippenger / Jacobian-fast-path scaffolding entirely. + // Affine operator* + Jacobian sum already handles all edge cases. + if (n < 4) { + return trivial_msm(scalars, points); + } + const auto& start = scalars.start_index; + if (start >= points.size()) { + return Curve::Group::point_at_infinity; + } + const size_t n_used = std::min(n, points.size() - start); + std::span point_slice(points.data() + start, n_used); + std::span scalar_slice(scalars.span.data(), n_used); + // Convert scalars to non-Montgomery form for the jacobian path's bit-extraction loop, + // then restore. Mirrors the round-parallel fast-path's scalar lifecycle. + // Use the `_reduced` variant: the bit-extraction loop reads only bits 0..253 + // (NUM_BITS = 254). Plain `self_from_montgomery_form` leaves the value in [0, 2p), + // so values in [2^254, 2p) would have bit 254 set and silently drop the contribution + // of that bit. `_reduced` brings the value into [0, p) ⊂ [0, 2^254). + auto* mutable_scalars = + const_cast(scalar_slice.data()); // NOLINT(cppcoreguidelines-pro-type-const-cast) + bb::parallel_for(bb::get_num_cpus(), [&](const ThreadChunk& chunk) { + BB_BENCH_NAME("MSM_fast::pu_from_montgomery/worker"); + for (size_t i : chunk.range(n_used)) { + mutable_scalars[i].self_from_montgomery_form_reduced(); + } + }); + const Element result = + round_parallel_detail::pippenger_round_parallel_jacobian_fast(scalar_slice, point_slice, 0); + bb::parallel_for(bb::get_num_cpus(), [&](const ThreadChunk& chunk) { + BB_BENCH_NAME("MSM_fast::pu_to_montgomery/worker"); + for (size_t i : chunk.range(n_used)) { + mutable_scalars[i].self_to_montgomery_form(); + } + }); + return result; +} + +template +typename Curve::AffineElement MSM_fast::msm(std::span points, + PolynomialSpan scalars, + bool handle_edge_cases, + bool dedup_hint) noexcept +{ + return AffineElement(pippenger_fast(scalars, points, handle_edge_cases, dedup_hint)); +} + +#include "./pippenger_batched.hpp" + +// Explicit instantiations. +template curve::BN254::Element pippenger_unsafe_fast( + PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept; +template curve::Grumpkin::Element pippenger_unsafe_fast( + PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept; +template curve::BN254::Element pippenger_fast(PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept; +template curve::Grumpkin::Element pippenger_fast( + PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept; +template class MSM_fast; +template class MSM_fast; + +template curve::BN254::Element pippenger_round_parallel( + PolynomialSpan scalars, + std::span points, + bool dedup_hint, + std::span external_glv_doubled, + std::span external_arena) noexcept; + +template curve::Grumpkin::Element pippenger_round_parallel( + PolynomialSpan scalars, + std::span points, + bool dedup_hint, + std::span external_glv_doubled, + std::span external_arena) noexcept; + +template curve::BN254::Element trivial_msm( + PolynomialSpan scalars_span, + std::span all_points) noexcept; + +template curve::Grumpkin::Element trivial_msm( + PolynomialSpan scalars_span, + std::span all_points) noexcept; + +template curve::BN254::Element trivial_msm_threaded( + PolynomialSpan scalars_span, + std::span all_points) noexcept; + +template curve::Grumpkin::Element trivial_msm_threaded( + PolynomialSpan scalars_span, + std::span all_points) noexcept; + +namespace round_parallel_detail { +template curve::BN254::Element pippenger_round_parallel_jacobian_fast( + std::span scalars, + std::span points, + size_t min_pts_per_thread_override) noexcept; + +template curve::Grumpkin::Element pippenger_round_parallel_jacobian_fast( + std::span scalars, + std::span points, + size_t min_pts_per_thread_override) noexcept; +} // namespace round_parallel_detail + +template size_t compute_arena_bytes_for_msm(size_t, bool, bool) noexcept; + +} // namespace bb::scalar_multiplication diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication_fast.hpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication_fast.hpp new file mode 100644 index 000000000000..029c4ec23bb4 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication_fast.hpp @@ -0,0 +1,250 @@ +#pragma once + +#include "barretenberg/common/thread.hpp" +#include "barretenberg/ecc/curves/bn254/bn254.hpp" +#include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" +#include "barretenberg/polynomials/polynomial.hpp" +#include + +#include +#include +#include +#include +#include + +namespace bb::scalar_multiplication { + +/** + * @brief N-dependent oversubscription factor used ONLY for `choose_window_bits`' + * target_load formula (not for actual thread dispatch). + */ +size_t window_bits_tuning_oversub_factor(size_t n_input); + +/** + * @brief State of the art pippenger_fast multiscalar multiplication algorithm. + * + * @details A traditional pippenger_fast N/logN algorithm (split scalars into windows, use window value to map scalar's + * base point into a bucket. Accumulate buckets. Repeat for all windows (1 window = 1 round)) We add the following + * optimizations on top of the algorithm: 1) Efficient multithreading via round-parallelism. If memory budget allows + * each thread evaluates multiple rounds. Thread efficiency ~90% when measured. 2) Booth in-place recoding of scalars. + * Each slice represents [-(num_buckets/2- 1), ..., (num_buckets/2 - 1)]. halves number of buckets. 3) GLV decompositon + * of input scalars into two half-size scalars. Halves number of bucket accumulation steps. Only used for n<2^{16} due + * to memory throughput tradeoffs. 4) Adaptive bucket range. When mixing large and small scalars (e.g. witness values), + * low bit-ranges use a larger bucket-range. Larger bit-ranges use a bucket-range tuned to the reduced number of large + * scalars. 5) Duplicate stripping. Witness commitments and permutation polynomials contain large numbers of duplicates. + * When dedup flag is active these are detected and base points consolidated prior to main MSM_fast. 6) Batch-affine + * arithmetic. When accumulating points into buckets, independent point additions are gathered and batched, allowing + * for affine point arithmetic w. batch invert. Cost = 6M for addition, 7M for doubling (M=field mult). + * 7) Batch-affine bucket accumulation. Uses novel technique from (TODO REF) to accumulate buckets in runs of + * independent additions. + * 8) If n is small (num points per thread < ~24), fallback to optimised multithreaded Straus MSM_fast + * (windowed double-and-add with GLV endomorphism) + * + * In order to efficiently utilize memory and prevent WASM memory fragmentation, we use a single `arena` memory buffer + * to allocate all temporary data structures. Currently sized at 36MB which defines the upper cap on the memory consumed + * by this algorihtm. + * @param scalars Input scalars + * @param points Input points + * @param dedup_hint Activates duplicate stripping if true. Off by default as dup stripping adds ~5% overhead on random + * inputs + * @pre Inputs are linearly independent: no point-at-infinity, no equal-x within a bucket + * (matches `pippenger_unsafe_fast`). + * @note Scalars are converted out of Montgomery form internally and restored before return. + */ +// `external_glv_doubled`: optional caller-supplied [P, φP, ...] interleaved buffer +// (length 2*n). When non-empty, every n_input is treated as GLV-eligible and the +// doubled points are aliased instead of recomputed — the batched driver uses this +// to share the doubled SRS prefix across MSMs in a batch. +// `external_arena`: optional caller-supplied scratch buffer ≥ this MSM_fast's required +// bytes. When empty, allocated per-MSM_fast and freed at return. The batched driver +// supplies a single arena sized to the largest member. +template +typename Curve::Element pippenger_round_parallel( + PolynomialSpan scalars, + std::span points, + bool dedup_hint = false, + std::span external_glv_doubled = {}, + std::span external_arena = {}) noexcept; + +extern template curve::BN254::Element pippenger_round_parallel( + PolynomialSpan scalars, + std::span points, + bool dedup_hint, + std::span external_glv_doubled, + std::span external_arena) noexcept; + +extern template curve::Grumpkin::Element pippenger_round_parallel( + PolynomialSpan scalars, + std::span points, + bool dedup_hint, + std::span external_glv_doubled, + std::span external_arena) noexcept; + +// =================================================================================== +// Public API (interface-compatible with the legacy `scalar_multiplication::MSM_fast` class). +// =================================================================================== +// +// `pippenger_fast` — handle_edge_cases routed: false → fast affine round-parallel, +// true → Jacobian fast path (handles point-at-infinity / equal-x +// bucket collisions). +// `pippenger_unsafe_fast` — always the fast path; caller asserts linear-independence of points. +// `MSM_fast::msm` — single-MSM_fast convenience wrapper (returns AffineElement). +// `MSM_fast::batch_multi_scalar_mul` — multi-MSM_fast driver: runs each MSM_fast via `pippenger_fast` +// and returns a vector of AffineElement results. + +template +typename Curve::Element pippenger_fast(PolynomialSpan scalars, + std::span points, + bool handle_edge_cases = true, + bool dedup_hint = false) noexcept; + +template +typename Curve::Element pippenger_unsafe_fast(PolynomialSpan scalars, + std::span points, + bool dedup_hint = false) noexcept; + +extern template curve::BN254::Element pippenger_fast( + PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept; + +extern template curve::Grumpkin::Element pippenger_fast( + PolynomialSpan scalars, + std::span points, + bool handle_edge_cases, + bool dedup_hint) noexcept; + +extern template curve::BN254::Element pippenger_unsafe_fast( + PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept; + +extern template curve::Grumpkin::Element pippenger_unsafe_fast( + PolynomialSpan scalars, + std::span points, + bool dedup_hint) noexcept; + +template class MSM_fast { + public: + using Element = typename Curve::Element; + using ScalarField = typename Curve::ScalarField; + using AffineElement = typename Curve::AffineElement; + + /** + * @brief Single MSM_fast convenience wrapper — returns the result as an AffineElement. + * @param handle_edge_cases false (default): fast affine round-parallel path. + * true: Jacobian fast path (handles edge cases). + * @param dedup_hint When true, opts this MSM_fast into the input-scalar dedup pre-pass. + */ + static AffineElement msm(std::span points, + PolynomialSpan scalars, + bool handle_edge_cases = false, + bool dedup_hint = false) noexcept; + + /** + * @brief Batch driver for multiple MSMs. Returns one AffineElement per input MSM_fast. + * + * Every MSM_fast in the batch shares a single contiguous point set (the SRS / GLV table); + * each MSM_fast picks its own range via `scalars[m].start_index` and `scalars[m].size()`. + * MSM_fast `m` computes Σ_i scalars[m][i] * points[scalars[m].start_index + i]. + * + * Independent MSMs run sequentially (each MSM_fast is itself round-parallel internally). + * This matches the legacy interface but the parallelisation strategy is different: + * the legacy implementation work-balanced points across threads spanning multiple + * MSMs; round-parallel parallelises within each MSM_fast, so one MSM_fast at a time uses the + * full thread pool. For the typical chonk workload (commit batches of polys of size + * 2^20), this is faster because per-MSM_fast threading dominates over inter-MSM_fast stealing. + * + * @param points Shared point set (SRS prefix). Every MSM_fast indexes into this span. + * @param scalars Per-MSM_fast scalars carrying the start offset into `points`. + * @param dedup_hints Optional per-MSM_fast dedup opt-ins (parallel to `scalars`): a + * non-zero entry opts that MSM_fast's input scalars into the + * duplicate-cluster pre-pass. Empty span means no dedup anywhere. + */ + static std::vector batch_multi_scalar_mul(std::span points, + std::span> scalars, + bool handle_edge_cases = true, + std::span dedup_hints = {}) noexcept; +}; + +extern template class MSM_fast; +extern template class MSM_fast; + +// `pippenger_round_parallel` falls back to `trivial_msm_threaded` when each worker +// would receive fewer than this many points (after the n_active filter). Exposed so tests +// and bench targets can pin behaviour at the boundary. +inline constexpr size_t MIN_PTS_PER_THREAD_FOR_PIPPENGER = 24; + +// Per-MSM_fast arena sizer. Returns 0 for shapes that fall back to the Jacobian-fast path +// (no affine arena). Mirrors the inline budget calc inside `pippenger_round_parallel`; +// declared here so the test suite can exercise the same sizer. +template +size_t compute_arena_bytes_for_msm(size_t n_input, bool external_glv_provided, bool dedup_active = false) noexcept; + +namespace round_parallel_detail { + +// Above this N, GLV's 2x point-count cost outweighs the windows-halved benefit. +#ifdef __wasm__ +inline constexpr size_t GLV_SMALL_N_THRESHOLD = size_t{ 1 } << 16; +#else +inline constexpr size_t GLV_SMALL_N_THRESHOLD = size_t{ 1 } << 13; +#endif + +/** + * @brief Single-MSM_fast, no-affine-trick Pippenger over window_bits-wide windows. + * + * `min_pts_per_thread_override` lets benchmarks pin behaviour: + * - 0 (default) → use the internal `MIN_PTS_PER_THREAD` heuristic (256 native, single-threaded on WASM). + * - SIZE_MAX → force single-threaded. + * - 1 → maximally multi-threaded (one worker per logical CPU). + */ +template +typename Curve::Element pippenger_round_parallel_jacobian_fast(std::span scalars, + std::span points, + size_t min_pts_per_thread_override = 0) noexcept; + +extern template curve::BN254::Element pippenger_round_parallel_jacobian_fast( + std::span scalars, + std::span points, + size_t min_pts_per_thread_override) noexcept; + +extern template curve::Grumpkin::Element pippenger_round_parallel_jacobian_fast( + std::span scalars, + std::span points, + size_t min_pts_per_thread_override) noexcept; + +} // namespace round_parallel_detail + +/** + * @brief Single-threaded small-MSM_fast driver: `Element::straus_msm` over the input slice. + */ +template +typename Curve::Element trivial_msm(PolynomialSpan scalars_span, + std::span all_points) noexcept; + +extern template curve::BN254::Element trivial_msm( + PolynomialSpan scalars_span, + std::span all_points) noexcept; + +extern template curve::Grumpkin::Element trivial_msm( + PolynomialSpan scalars_span, + std::span all_points) noexcept; + +/** + * @brief Multi-threaded small-MSM_fast driver: parallel `Element::straus_msm` over zero-skipped + * input slices. + */ +template +typename Curve::Element trivial_msm_threaded(PolynomialSpan scalars_span, + std::span all_points) noexcept; + +extern template curve::BN254::Element trivial_msm_threaded( + PolynomialSpan scalars_span, + std::span all_points) noexcept; + +extern template curve::Grumpkin::Element trivial_msm_threaded( + PolynomialSpan scalars_span, + std::span all_points) noexcept; + +} // namespace bb::scalar_multiplication diff --git a/barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp b/barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp index 095c8e4bbcdf..0756de1ae457 100644 --- a/barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp @@ -1057,6 +1057,18 @@ class ECCVMFlavor { Base::lagrange_third = "__LAGRANGE_THIRD"; Base::lagrange_last = "__LAGRANGE_LAST"; }; + + // Used in pippenger_unsafe to activate duplicate stripping. + // Dups need to be > ~14 bits to be worth stripping. + // Empirical tests showed these polys had high duplicate counts under these conditions + static bool wire_has_high_duplicate_density(const std::string& label) noexcept + { + return label == "MSM_X1" || label == "MSM_X2" || label == "MSM_X3" || label == "MSM_X4" || + label == "MSM_Y1" || label == "MSM_Y2" || label == "MSM_Y3" || label == "MSM_Y4" || + label == "MSM_ROUND_MINUS_31_INV" || label == "PRECOMPUTE_DX" || label == "PRECOMPUTE_DY" || + label == "PRECOMPUTE_TX" || label == "PRECOMPUTE_TY" || label == "TRANSCRIPT_ACCUMULATOR_X" || + label == "TRANSCRIPT_ACCUMULATOR_Y" || label == "TRANSCRIPT_PX" || label == "TRANSCRIPT_PY"; + } }; template diff --git a/barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp b/barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp index bb0d5d3a7b3f..d1029685ffd8 100644 --- a/barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp +++ b/barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp @@ -65,7 +65,7 @@ void ECCVMProver::execute_wire_commitments_round() auto batch = key->commitment_key.start_batch(); for (const auto& [wire, label] : zip_view(key->polynomials.get_wires(), commitment_labels.get_wires())) { - batch.add_to_batch(wire, label); + batch.add_to_batch(wire, label, Flavor::CommitmentLabels::wire_has_high_duplicate_density(label)); } batch.commit_and_send_to_verifier(transcript); } @@ -120,7 +120,9 @@ void ECCVMProver::execute_grand_product_computation_round() // Compute permutation grand product (starts after disabled head region via gp_start) compute_grand_products(key->polynomials, relation_parameters); auto& zp = key->polynomials.z_perm; - transcript->send_to_verifier(commitment_labels.z_perm, key->commitment_key.commit(zp)); + // set has_duplicates_hint for Z_PERM (empty row = duplicate Z value) + transcript->send_to_verifier(commitment_labels.z_perm, + key->commitment_key.commit(zp, /*has_duplicates_hint=*/true)); } /** diff --git a/barretenberg/cpp/src/barretenberg/srs/scalar_multiplication.test.cpp b/barretenberg/cpp/src/barretenberg/srs/scalar_multiplication.test.cpp index 6557a4a00ad5..20fec9916395 100644 --- a/barretenberg/cpp/src/barretenberg/srs/scalar_multiplication.test.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/scalar_multiplication.test.cpp @@ -33,43 +33,6 @@ using Curves = ::testing::Types; TYPED_TEST_SUITE(ScalarMultiplicationTests, Curves); -TYPED_TEST(ScalarMultiplicationTests, AddAffinePoints) -{ - using Curve = TypeParam; - using Element = typename Curve::Element; - using AffineElement = typename Curve::AffineElement; - using Fq = typename Curve::BaseField; - - constexpr size_t num_points = 20; - AffineElement* points = (AffineElement*)(aligned_alloc(64, sizeof(AffineElement) * (num_points))); - Fq* scratch_space = (Fq*)(aligned_alloc(64, sizeof(Fq) * (num_points * 2))); - Fq* lambda = (Fq*)(aligned_alloc(64, sizeof(Fq) * (num_points * 2))); - - Element* points_copy = (Element*)(aligned_alloc(64, sizeof(Element) * (num_points))); - for (size_t i = 0; i < num_points; ++i) { - points[i] = AffineElement(Element::random_element()); - points_copy[i].x = points[i].x; - points_copy[i].y = points[i].y; - points_copy[i].z = Fq::one(); - } - - size_t count = num_points - 1; - for (size_t i = num_points - 2; i < num_points; i -= 2) { - points_copy[count--] = points_copy[i] + points_copy[i + 1]; - points_copy[count + 1] = points_copy[count + 1].normalize(); - } - - scalar_multiplication::MSM::add_affine_points(points, num_points, scratch_space); - for (size_t i = num_points - 1; i > num_points - 1 - (num_points / 2); --i) { - EXPECT_EQ((points[i].x == points_copy[i].x), true); - EXPECT_EQ((points[i].y == points_copy[i].y), true); - } - aligned_free(lambda); - aligned_free(points); - aligned_free(points_copy); - aligned_free(scratch_space); -} - TYPED_TEST(ScalarMultiplicationTests, EndomorphismSplit) { using Curve = TypeParam; @@ -99,55 +62,6 @@ TYPED_TEST(ScalarMultiplicationTests, EndomorphismSplit) EXPECT_EQ(result == expected, true); } -TYPED_TEST(ScalarMultiplicationTests, RadixSort) -{ - using Curve = TypeParam; - using Fr = typename Curve::ScalarField; - - // check that our radix sort correctly sorts! - constexpr size_t target_degree = 1 << 8; - const size_t num_rounds = scalar_multiplication::MSM::get_num_rounds(target_degree); - Fr* scalars = (Fr*)(aligned_alloc(64, sizeof(Fr) * target_degree)); - - Fr source_scalar = Fr::random_element(); - for (size_t i = 0; i < target_degree; ++i) { - source_scalar.self_sqr(); - Fr::__copy(source_scalar, scalars[i]); - } - - uint32_t bits_per_slice = scalar_multiplication::MSM::get_optimal_log_num_buckets(target_degree); - - for (uint32_t i = 0; i < num_rounds; ++i) { - - std::vector scalar_slices(target_degree); - std::vector sorted_scalar_slices(target_degree); - - for (size_t j = 0; j < target_degree; ++j) { - scalar_slices[j] = scalar_multiplication::MSM::get_scalar_slice(scalars[j], i, bits_per_slice); - sorted_scalar_slices[j] = scalar_slices[j]; - } - scalar_multiplication::sort_point_schedule_and_count_zero_buckets( - &sorted_scalar_slices[0], target_degree, static_cast(bits_per_slice)); - - const auto find_entry = [scalar_slices, num_entries = target_degree](auto x) { - for (size_t k = 0; k < num_entries; ++k) { - if (scalar_slices[k] == x) { - return true; - } - } - return false; - }; - for (size_t j = 0; j < target_degree; ++j) { - EXPECT_EQ(find_entry(sorted_scalar_slices[j]), true); - if (j > 0) { - EXPECT_EQ((sorted_scalar_slices[j] & 0x7fffffffU) >= (sorted_scalar_slices[j - 1] & 0x7fffffffU), true); - } - } - } - - free(scalars); -} - TYPED_TEST(ScalarMultiplicationTests, OversizedInputs) { using Curve = TypeParam; diff --git a/barretenberg/cpp/src/barretenberg/translator_vm/translator_prover.cpp b/barretenberg/cpp/src/barretenberg/translator_vm/translator_prover.cpp index 69095a231e01..ebd2bad00ff6 100644 --- a/barretenberg/cpp/src/barretenberg/translator_vm/translator_prover.cpp +++ b/barretenberg/cpp/src/barretenberg/translator_vm/translator_prover.cpp @@ -56,9 +56,11 @@ void TranslatorProver::execute_preamble_round() * @param polynomial * @param label */ -void TranslatorProver::commit_to_witness_polynomial(Polynomial& polynomial, const std::string& label) +void TranslatorProver::commit_to_witness_polynomial(Polynomial& polynomial, + const std::string& label, + bool has_duplicates_hint) { - transcript->send_to_verifier(label, key->proving_key->commitment_key.commit(polynomial)); + transcript->send_to_verifier(label, key->proving_key->commitment_key.commit(polynomial, has_duplicates_hint)); } /** @@ -129,7 +131,8 @@ void TranslatorProver::execute_grand_product_computation_round() // Compute constraint permutation grand product compute_grand_products(key->proving_key->polynomials, relation_parameters); - commit_to_witness_polynomial(key->proving_key->polynomials.z_perm, commitment_labels.z_perm); + // set has_duplicates_hint for Z_PERM (empty row = duplicate Z value) + commit_to_witness_polynomial(key->proving_key->polynomials.z_perm, commitment_labels.z_perm, true); } /** diff --git a/barretenberg/cpp/src/barretenberg/translator_vm/translator_prover.hpp b/barretenberg/cpp/src/barretenberg/translator_vm/translator_prover.hpp index 90e9183dc029..0b951e681bc7 100644 --- a/barretenberg/cpp/src/barretenberg/translator_vm/translator_prover.hpp +++ b/barretenberg/cpp/src/barretenberg/translator_vm/translator_prover.hpp @@ -38,7 +38,9 @@ class TranslatorProver { BB_PROFILE void execute_grand_product_computation_round(); BB_PROFILE void execute_relation_check_rounds(); BB_PROFILE void execute_pcs_rounds(); - void commit_to_witness_polynomial(Polynomial& polynomial, const std::string& label); + void commit_to_witness_polynomial(Polynomial& polynomial, + const std::string& label, + bool has_duplicates_hint = false); HonkProof export_proof(); HonkProof construct_proof(); diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp index d2157c11fe73..0a7703b7225b 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp @@ -74,9 +74,9 @@ template void OinkProver::commit_to_wires() // Commit to the first three wire polynomials; w_4 is deferred until after memory records are added // Masking values are already in the polynomials - batch.add_to_batch(prover_instance->polynomials.w_l, commitment_labels.w_l); - batch.add_to_batch(prover_instance->polynomials.w_r, commitment_labels.w_r); - batch.add_to_batch(prover_instance->polynomials.w_o, commitment_labels.w_o); + batch.add_to_batch(prover_instance->polynomials.w_l, commitment_labels.w_l, /*has_duplicates_hint=*/true); + batch.add_to_batch(prover_instance->polynomials.w_r, commitment_labels.w_r, /*has_duplicates_hint=*/true); + batch.add_to_batch(prover_instance->polynomials.w_o, commitment_labels.w_o, /*has_duplicates_hint=*/true); if constexpr (IsMegaFlavor) { for (auto [polynomial, label] : @@ -121,7 +121,7 @@ template void OinkProver::commit_to_lookup_counts_and_ auto batch = commitment_key.start_batch(); batch.add_to_batch(prover_instance->polynomials.lookup_read_counts, commitment_labels.lookup_read_counts); batch.add_to_batch(prover_instance->polynomials.lookup_read_tags, commitment_labels.lookup_read_tags); - batch.add_to_batch(prover_instance->polynomials.w_4, commitment_labels.w_4); + batch.add_to_batch(prover_instance->polynomials.w_4, commitment_labels.w_4, /*has_duplicates_hint=*/true); auto computed_commitments = batch.commit_and_send_to_verifier(transcript); prover_instance->commitments.lookup_read_counts = computed_commitments[0]; @@ -178,7 +178,8 @@ template void OinkProver::commit_to_z_perm() auto& z_perm = prover_instance->polynomials.z_perm; auto batch = commitment_key.start_batch(); - batch.add_to_batch(z_perm, commitment_labels.z_perm); + // set has_duplicates_hint for Z_PERM (empty row = duplicate Z value) + batch.add_to_batch(z_perm, commitment_labels.z_perm, /*has_duplicates_hint=*/true); auto commitments = batch.commit_and_send_to_verifier(transcript); prover_instance->commitments.z_perm = commitments[0]; } diff --git a/barretenberg/cpp/src/barretenberg/vm2/constraining/prover.cpp b/barretenberg/cpp/src/barretenberg/vm2/constraining/prover.cpp index ab7410838ff5..7246aae5599f 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/constraining/prover.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/constraining/prover.cpp @@ -24,10 +24,6 @@ namespace bb::avm2 { -// Maximum number of polynomials to batch commit at once. -const size_t AVM_MAX_MSM_BATCH_SIZE = - getenv("AVM_MAX_MSM_BATCH_SIZE") != nullptr ? std::stoul(getenv("AVM_MAX_MSM_BATCH_SIZE")) : 32; - using Flavor = AvmFlavor; using FF = Flavor::FF; @@ -103,7 +99,7 @@ void AvmProver::execute_wire_commitments_round() for (const auto& [poly, label] : zip_view(prover_polynomials.get_wires(), prover_polynomials.get_wires_labels())) { batch.add_to_batch(poly, label); } - batch.commit_and_send_to_verifier(transcript, AVM_MAX_MSM_BATCH_SIZE); + batch.commit_and_send_to_verifier(transcript); } void AvmProver::execute_log_derivative_inverse_round() @@ -147,7 +143,7 @@ void AvmProver::execute_log_derivative_inverse_commitments_round() batch.add_to_batch(derived_poly, label); } - batch.commit_and_send_to_verifier(transcript, AVM_MAX_MSM_BATCH_SIZE); + batch.commit_and_send_to_verifier(transcript); } /**