From 030938909e53455b2fc3eb28ff38d3c6b60a7cc4 Mon Sep 17 00:00:00 2001 From: Ralph Date: Sat, 18 Jul 2026 00:56:54 -0700 Subject: [PATCH] perf(gc): lazy epoch reset for old-page dirty-slot accounting (#6181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `old_pages_begin_gc_cycle` zeroed every old page's `dirty_slots` on every GC cycle — an O(number of old pages) walk on each minor whose cost grows with old-gen size (issue #6181 sub-cost (c): "lazy epoch-counter resets for page metas"). Replace the per-page reset with a single O(1) bump of a per-cycle epoch: a page whose `dirty_slots_epoch` predates the current epoch reads as zero (`effective_dirty_slots`), and the remembered-set scan re-stamps it on first touch this cycle. Mirrors the generation-counter lazy-invalidation pattern in `native_arena.rs`. `dirty_slots` is a GC-scan-time diagnostic counter (surfaced only via `old_page_summary`, gated behind PERRY_GC_TRACE) and is not coupled to the sweep's mark-clearing, so this is independent of the still-open sub-cost (a) (restricting the minor old-gen sweep walk). Snapshot accessors normalize the copied `dirty_slots` to the effective value so no caller needs the epoch. Regression test asserts a bare epoch bump invalidates the prior cycle's count with no per-page work, and that the next dirty slot re-stamps from 1. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/perry-runtime/src/arena/page_meta.rs | 93 +++++++++++++++++-- .../perry-runtime/src/gc/tests/evacuation.rs | 1 + crates/perry-runtime/src/gc/tests/oldgen.rs | 47 ++++++++++ 3 files changed, 131 insertions(+), 10 deletions(-) diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index 5bb0c4d71a..a71dcbe598 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -149,6 +149,16 @@ pub(crate) struct OldPageMeta { pub(crate) pinned_bytes: usize, pub(crate) pinned_object_count: usize, pub(crate) dirty_slots: usize, + /// Epoch stamp for `dirty_slots` (#6181). `dirty_slots` is a per-cycle + /// count that used to be zeroed for every old page at the start of each + /// GC cycle by `old_pages_begin_gc_cycle` — an O(number of old pages) + /// walk on every minor whose cost grows with old-gen size. Instead of + /// the eager reset, each page records the epoch its `dirty_slots` was + /// last stamped in; the cycle-start "reset" is now a single O(1) bump of + /// the thread-local `OLD_GEN_PAGE_DIRTY_EPOCH`, and a page whose stamp is + /// stale reads as zero (`effective_dirty_slots`). Mirrors the + /// generation-counter lazy-invalidation pattern in `native_arena.rs`. + pub(crate) dirty_slots_epoch: u64, pub(crate) dirty: bool, pub(crate) evacuation_eligible: bool, } @@ -169,11 +179,28 @@ impl OldPageMeta { pinned_bytes: 0, pinned_object_count: 0, dirty_slots: 0, + // Epoch 0 never matches a live cycle epoch (which starts at 1), so + // a freshly materialized page reads zero dirty slots until it is + // stamped by the remembered-set scan (#6181). + dirty_slots_epoch: 0, dirty: false, evacuation_eligible: false, } } + /// `dirty_slots` scoped to `current_epoch`: a page last stamped in an + /// earlier cycle (its `dirty_slots_epoch` is stale) has no dirty slots + /// this cycle. This is what makes the O(1) epoch bump in + /// `old_pages_begin_gc_cycle` equivalent to the old per-page reset (#6181). + #[inline] + pub(crate) fn effective_dirty_slots(&self, current_epoch: u64) -> usize { + if self.dirty_slots_epoch == current_epoch { + self.dirty_slots + } else { + 0 + } + } + #[inline] fn reset_cycle_sweep_accounting(&mut self) { self.live_bytes = 0; @@ -234,6 +261,18 @@ thread_local! { pub(crate) static OLD_GEN_RECLAIM_REUSABLE_BYTES: Cell = const { Cell::new(0) }; pub(crate) static OLD_GEN_RECLAIM_RETURNED_BYTES: Cell = const { Cell::new(0) }; + + /// Monotonic per-cycle epoch for old-page `dirty_slots` (#6181). Bumped + /// once per GC cycle by `old_pages_begin_gc_cycle` instead of walking every + /// old page to zero its `dirty_slots`. Starts at 1 so a freshly allocated + /// page (stamp 0, see `OldPageMeta::zero_for_page`) reads as having no + /// dirty slots. `u64` never wraps in practice (one bump per collection). + static OLD_GEN_PAGE_DIRTY_EPOCH: Cell = const { Cell::new(1) }; +} + +#[inline] +fn old_gen_page_dirty_epoch() -> u64 { + OLD_GEN_PAGE_DIRTY_EPOCH.with(|epoch| epoch.get()) } #[inline] @@ -553,11 +592,12 @@ pub(crate) fn unregister_old_object_pages(header_addr: usize, total_size: usize) } pub(crate) fn old_pages_begin_gc_cycle() { - OLD_GEN_PAGE_META.with(|meta| { - for page_meta in meta.borrow_mut().values_mut() { - page_meta.dirty_slots = 0; - } - }); + // #6181: the per-page `dirty_slots` reset used to iterate every old page + // here (O(old pages) on every minor, growing with old-gen size). It is now + // a single epoch bump — a page whose `dirty_slots_epoch` predates the new + // epoch reads as zero via `effective_dirty_slots`, and the scan re-stamps + // it on first touch this cycle (`old_page_account_dirty_slot`). + OLD_GEN_PAGE_DIRTY_EPOCH.with(|epoch| epoch.set(epoch.get().wrapping_add(1))); OLD_GEN_RECLAIM_REUSABLE_BYTES.with(|bytes| bytes.set(0)); OLD_GEN_RECLAIM_RETURNED_BYTES.with(|bytes| bytes.set(0)); } @@ -639,14 +679,24 @@ pub(crate) fn old_page_account_dirty_slot(slot_addr: usize) { return; } let page = generation_page_for_addr(slot_addr); + let current_epoch = old_gen_page_dirty_epoch(); OLD_GEN_PAGE_META.with(|meta| { if let Some(page_meta) = meta.borrow_mut().get_mut(&page) { - page_meta.dirty_slots = page_meta.dirty_slots.saturating_add(1); + // First dirty slot seen this cycle re-stamps and starts from 1 + // (the lazy equivalent of the old per-cycle reset-to-zero); + // subsequent slots on the same page accumulate (#6181). + if page_meta.dirty_slots_epoch == current_epoch { + page_meta.dirty_slots = page_meta.dirty_slots.saturating_add(1); + } else { + page_meta.dirty_slots_epoch = current_epoch; + page_meta.dirty_slots = 1; + } } }); } pub(crate) fn old_page_summary() -> OldPageSummary { + let current_epoch = old_gen_page_dirty_epoch(); OLD_GEN_PAGE_META.with(|meta| { let meta = meta.borrow(); let mut summary = OldPageSummary { @@ -670,10 +720,11 @@ pub(crate) fn old_page_summary() -> OldPageSummary { summary.pinned_object_count = summary .pinned_object_count .saturating_add(page_meta.pinned_object_count); - if page_meta.dirty || page_meta.dirty_slots > 0 { + let dirty_slots = page_meta.effective_dirty_slots(current_epoch); + if page_meta.dirty || dirty_slots > 0 { summary.dirty_pages = summary.dirty_pages.saturating_add(1); } - summary.dirty_slots = summary.dirty_slots.saturating_add(page_meta.dirty_slots); + summary.dirty_slots = summary.dirty_slots.saturating_add(dirty_slots); if page_meta.live_bytes > 0 && page_meta.dead_bytes > 0 { summary.fragmented_pages = summary.fragmented_pages.saturating_add(1); } @@ -689,13 +740,29 @@ pub(crate) fn old_page_summary() -> OldPageSummary { } pub(crate) fn old_page_meta_snapshot() -> Vec { + let current_epoch = old_gen_page_dirty_epoch(); OLD_GEN_PAGE_META.with(|meta| { - let mut snapshot = meta.borrow().values().copied().collect::>(); + let mut snapshot = meta + .borrow() + .values() + .copied() + .map(|page_meta| normalize_dirty_slots_for_epoch(page_meta, current_epoch)) + .collect::>(); snapshot.sort_unstable_by_key(|page_meta| page_meta.page_base); snapshot }) } +/// Fold a stale `dirty_slots` stamp down to the effective value so a copied +/// `OldPageMeta` handed to a caller always reports this cycle's dirty-slot +/// count directly in `dirty_slots`, without the caller needing the epoch (#6181). +#[inline] +fn normalize_dirty_slots_for_epoch(mut page_meta: OldPageMeta, current_epoch: u64) -> OldPageMeta { + page_meta.dirty_slots = page_meta.effective_dirty_slots(current_epoch); + page_meta.dirty_slots_epoch = current_epoch; + page_meta +} + pub(crate) fn old_arena_source_blocks_for_pages( selected_pages: &crate::fast_hash::PtrHashSet, ) -> OldArenaSourceBlockSelection { @@ -844,5 +911,11 @@ pub(crate) fn old_arena_page_index_clear_for_tests() { #[cfg(test)] pub(crate) fn old_page_meta_for_tests(page: usize) -> Option { - OLD_GEN_PAGE_META.with(|meta| meta.borrow().get(&page).copied()) + let current_epoch = old_gen_page_dirty_epoch(); + OLD_GEN_PAGE_META.with(|meta| { + meta.borrow() + .get(&page) + .copied() + .map(|page_meta| normalize_dirty_slots_for_epoch(page_meta, current_epoch)) + }) } diff --git a/crates/perry-runtime/src/gc/tests/evacuation.rs b/crates/perry-runtime/src/gc/tests/evacuation.rs index 858ae17054..97b6324c16 100644 --- a/crates/perry-runtime/src/gc/tests/evacuation.rs +++ b/crates/perry-runtime/src/gc/tests/evacuation.rs @@ -488,6 +488,7 @@ fn test_old_page_defrag_selection_reports_releasable_granule_bytes() { pinned_bytes, pinned_object_count: usize::from(pinned_bytes > 0), dirty_slots: 0, + dirty_slots_epoch: 0, dirty: false, evacuation_eligible: false, } diff --git a/crates/perry-runtime/src/gc/tests/oldgen.rs b/crates/perry-runtime/src/gc/tests/oldgen.rs index e1dc0ca47a..188a60c44c 100644 --- a/crates/perry-runtime/src/gc/tests/oldgen.rs +++ b/crates/perry-runtime/src/gc/tests/oldgen.rs @@ -331,6 +331,52 @@ fn test_dirty_page_scan_accounts_old_page_dirty_slots() { remembered_set_clear(); } +#[test] +fn test_old_page_dirty_slots_reset_lazily_across_cycles() { + // #6181: `old_pages_begin_gc_cycle` no longer walks every old page to zero + // `dirty_slots` — it bumps a per-cycle epoch, so last cycle's count reads + // as zero without any per-page work, and the next dirty slot re-stamps + // from 1. This asserts that epoch-scoped equivalence directly. + let _isolation = copying_nursery_isolation_lock(); + reset_remembered_set(); + clear_marks(); + clear_mark_seeds(); + + let (_old_obj, fields) = unsafe { alloc_old_test_object(1) }; + let dirty_page = crate::arena::generation_page_for_addr(fields as usize); + + // Cycle A: two dirty-slot accountings land on the page. + crate::arena::old_pages_begin_gc_cycle(); + crate::arena::old_page_account_dirty_slot(fields as usize); + crate::arena::old_page_account_dirty_slot(fields as usize); + let meta_a = crate::arena::old_page_meta_for_tests(dirty_page) + .expect("old page should have metadata after allocation"); + assert_eq!(meta_a.dirty_slots, 2, "both accountings counted in cycle A"); + assert_eq!(crate::arena::old_page_summary().dirty_slots, 2); + + // Cycle B: a bare epoch bump (no per-page reset walk) must invalidate the + // prior count — the page reads zero dirty slots even though nothing + // touched its metadata. + crate::arena::old_pages_begin_gc_cycle(); + let meta_b = crate::arena::old_page_meta_for_tests(dirty_page) + .expect("old page metadata should persist across cycles"); + assert_eq!( + meta_b.dirty_slots, 0, + "epoch bump alone resets last cycle's dirty-slot count" + ); + assert_eq!(crate::arena::old_page_summary().dirty_slots, 0); + + // A fresh dirty slot in cycle B re-stamps the page and counts from 1. + crate::arena::old_page_account_dirty_slot(fields as usize); + let meta_b2 = crate::arena::old_page_meta_for_tests(dirty_page) + .expect("old page metadata should persist"); + assert_eq!(meta_b2.dirty_slots, 1, "re-stamp starts a fresh count"); + assert_eq!(crate::arena::old_page_summary().dirty_slots, 1); + + clear_marks(); + remembered_set_clear(); +} + #[test] fn test_old_page_sweep_accounting_trace_json_includes_summary() { let _isolation = copying_nursery_isolation_lock(); @@ -402,6 +448,7 @@ fn test_old_page_defrag_policy_selection_prefers_fragmented_unpinned_pages() { pinned_bytes, pinned_object_count: usize::from(pinned_bytes > 0), dirty_slots: 0, + dirty_slots_epoch: 0, dirty: false, evacuation_eligible: false, }