diff --git a/crates/perry-codegen/src/expr/index_get.rs b/crates/perry-codegen/src/expr/index_get.rs index b2fe5e3c0f..9a52b86ef8 100644 --- a/crates/perry-codegen/src/expr/index_get.rs +++ b/crates/perry-codegen/src/expr/index_get.rs @@ -1173,6 +1173,10 @@ fn lower_bounded_array_index_get( )) } +// #6132: retired — inline-read a value as a plain ArrayHeader, which is unsafe +// for off-heap typed arrays (garbage reads). Callers now use the typed-feedback +// guarded path. Kept temporarily behind allow(dead_code); slated for deletion. +#[allow(dead_code)] fn lower_legacy_array_index_get( ctx: &mut FnCtx<'_>, arr_box: &str, @@ -1658,11 +1662,15 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { )); } let idx_i32 = lower_expr_as_i32(ctx, index)?; - if !require_numeric_layout - && !matches!(index.as_ref(), Expr::Integer(_) | Expr::Number(_)) - { - return lower_legacy_array_index_get(ctx, &arr_box, &idx_i32); - } + // #6132: a member receiver of unknown type (e.g. `n.buf[i]` where + // `n.buf` is a Uint32Array) must NOT go through the legacy inline + // reader — that path reads the value as a plain `ArrayHeader` + // (gc_type at `handle-8`, raw f64 slot at `handle+8+i*8`), but a + // small typed array is off-heap with no GcHeader, so both reads + // are garbage and it returned nondeterministic junk. Route through + // the typed-feedback-guarded path: its runtime guard rejects + // non-plain arrays and takes the boxed fallback (which dispatches + // typed arrays correctly), while plain arrays keep the fast path. return lower_guarded_array_index_get( ctx, &arr_box, diff --git a/crates/perry-codegen/src/runtime_decls/arrays.rs b/crates/perry-codegen/src/runtime_decls/arrays.rs index 8c0efcfcd5..a3d2801945 100644 --- a/crates/perry-codegen/src/runtime_decls/arrays.rs +++ b/crates/perry-codegen/src/runtime_decls/arrays.rs @@ -87,6 +87,11 @@ pub fn declare_phase_b_arrays(module: &mut LlModule) { module.declare_function("js_shadow_slot_set", VOID, &[I32, I64]); module.declare_function("js_shadow_slot_bind", VOID, &[I32, PTR]); module.declare_function("js_gc_write_barriers_emitted", VOID, &[I32]); + // Phase 2 of the moving-GC project: emitted at loop back-edges (only when + // compiled with the moving-safepoint opt-in) so a deferred nursery + // collection can run at a precise-root safepoint. No-op at runtime unless + // moving mode is on and a collection is pending. + module.declare_function("js_gc_loop_safepoint", VOID, &[]); // Write barrier for the generational GC (Phase C per the // gen-GC plan). Called by codegen-emitted heap-store sites diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index 1504492518..c97ec39a56 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -2654,6 +2654,7 @@ fn lower_for_after_init_with_i32_bound( ctx.block().asm_sideeffect_barrier(); } if !ctx.block().is_terminated() { + emit_gc_loop_safepoint(ctx); ctx.block().br(&update_label); } @@ -2707,6 +2708,45 @@ fn lower_for_after_init_with_i32_bound( Ok(()) } +/// Whether to emit loop back-edge safepoint polls — OPT-IN, default OFF +/// (`PERRY_GC_MOVING_LOOP_POLLS=1`). The moving GC is the default at the +/// event-loop safepoint, but the loop poll emits a `js_gc_loop_safepoint()` +/// CALL at every loop back-edge, which defeats LLVM auto-vectorization and +/// violates the native-region "no runtime calls in hot loop" proofs. Until the +/// poll is emitted only in loops that actually ALLOCATE (so numeric/vectorizable +/// loops stay call-free), it is opt-in and a tight allocating loop defers to the +/// event-loop safepoint instead. +fn moving_safepoint_polls_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_GC_MOVING_LOOP_POLLS").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + +/// Emit a `js_gc_loop_safepoint()` poll at a loop back-edge. Call this AFTER +/// `clear_loop_body_shadow_slots` and only where the block is not terminated: +/// at that point the loop-body expression has completed, so every live heap +/// value is a named local on the shadow stack (no unspilled register temps) — +/// a precise-root safepoint where a deferred copying minor can MOVE survivors. +/// +/// COVERAGE (Phase 2, follow-up): currently wired into the generic `while`, +/// `do..while`, and `for` back-edges. The specialized/versioned `for`-loop +/// lowering paths in this file (i32-bound-optimized, packed-f64/i32/u32, +/// bulk-fill) and `for-of`/`for-in` do NOT yet emit it, so a hot allocating +/// loop that takes one of those paths won't drain a deferred moving minor until +/// the next event-loop safepoint. Adding the poll to every back-edge across +/// those paths is the remaining Phase 2 codegen work. +pub(crate) fn emit_gc_loop_safepoint(ctx: &mut FnCtx<'_>) { + if !moving_safepoint_polls_enabled() || ctx.block().is_terminated() { + return; + } + ctx.block().call_void("js_gc_loop_safepoint", &[]); +} + pub(crate) fn clear_loop_body_shadow_slots(ctx: &mut FnCtx<'_>, body: &[Stmt]) { if ctx.block().is_terminated() || ctx.shadow_slot_map.is_empty() { return; @@ -4454,6 +4494,7 @@ pub(crate) fn lower_while( ctx.block().asm_sideeffect_barrier(); } if !ctx.block().is_terminated() { + emit_gc_loop_safepoint(ctx); ctx.block().br(&cond_label); } ctx.active_region_id = previous_region_id; @@ -4511,6 +4552,7 @@ pub(crate) fn lower_do_while( ctx.block().asm_sideeffect_barrier(); } if !ctx.block().is_terminated() { + emit_gc_loop_safepoint(ctx); ctx.block().br(&cond_label); } diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 78ff60ff92..f6b7096005 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -498,6 +498,29 @@ impl CopyingNurseryCollector { } let total = (*header).size as usize; + // Safety net (partial mitigation, NOT a full fix): a genuine + // young/survivor object is always small — large objects are allocated + // old-gen/malloc, never in the copying nursery — so a "young" object + // whose size is out of range is a corrupt/mis-classified header (e.g. an + // off-heap pointer whose preceding bytes coincidentally pass + // `plausible_gc_header`). Refuse to memmove through such a garbage size: + // that turns the worst outcome (a wild out-of-bounds copy → SIGSEGV) + // into a no-op, and surfaces it under PERRY_GC_DIAG. It does NOT catch a + // plausible-but-wrong *small* size; the root fix is stronger arena + // classification / page unregistration so off-heap addresses never + // reach here. See the copying-minor relocation issue. + const MAX_YOUNG_MOVE_BYTES: usize = 1 << 20; // 1 MiB, >> any real young object + if total < GC_HEADER_SIZE || total > MAX_YOUNG_MOVE_BYTES { + if std::env::var_os("PERRY_GC_DIAG").is_some() { + eprintln!( + "[gc-move-guard] refusing wild young move user={:#x} obj_type={} size={}", + old_user as usize, + (*header).obj_type, + total + ); + } + return old_user as usize; + } let payload = total - GC_HEADER_SIZE; let prior_age = copied_survival_age((*header)._reserved, flags); let next_age = prior_age.saturating_add(1); @@ -903,6 +926,23 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( trace.root_sources.native_stack_fallback.scanned = matches!(decision, ConservativeStackScanDecision::Scan); } + if std::env::var_os("PERRY_GC_DIAG").is_some() { + let reason = match eligibility.fallback_reason { + CopiedMinorFallbackReason::None => "none", + CopiedMinorFallbackReason::NotAttempted => "not_attempted", + CopiedMinorFallbackReason::BarriersInactive => "barriers_inactive", + CopiedMinorFallbackReason::ConservativeStack => "conservative_stack", + CopiedMinorFallbackReason::CopyOnlyRoots => "copy_only_roots", + CopiedMinorFallbackReason::MallocRegistryUnavailable => "malloc_registry_unavailable", + CopiedMinorFallbackReason::PinnedYoungRoot => "pinned_young_root", + CopiedMinorFallbackReason::PinnedYoungDirtySlot => "pinned_young_dirty_slot", + CopiedMinorFallbackReason::PinnedYoungTransitive => "pinned_young_transitive", + }; + eprintln!( + "[gc-copy-minor] eligible={} fallback={}", + eligibility.eligible, reason + ); + } if !eligibility.eligible { return None; } @@ -1111,6 +1151,16 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( trace.capture_layout_scans(); } maybe_schedule_old_reclaim_after_copied_minor(); + if std::env::var_os("PERRY_GC_DIAG").is_some() { + eprintln!( + "[gc-copy-minor] ran copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={}", + collector.stats.copied_objects, + collector.stats.copied_bytes, + collector.stats.promoted_objects, + collector.stats.promoted_bytes, + freed_bytes + ); + } Some(CopiedMinorFastPathOutcome { freed_bytes, malloc_swept: malloc_sweep_due, diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 65dc28fa49..910bc90071 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1073,7 +1073,13 @@ impl GcCycleState { .is_some_and(|minor| minor.evacuation_policy.considered); self.root_scan.get_or_insert_with(RootScanCycleState::new); - let allow_synchronous_scanners = !self.progress_kind.is_budgeted(); + // Phase 4 (incremental old-gen, gated): allow the initial root-scan step + // of a budgeted cycle to run unbudgeted scanners synchronously (a + // bounded initial-mark pause), so the stepper can start on programs that + // register unbudgeted mutable scanners. Marking still proceeds + // incrementally in later steps (they don't consult this flag). + let allow_synchronous_scanners = + !self.progress_kind.is_budgeted() || super::gc_incremental_enabled(); loop { let phase_name = self .root_scan diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index f9043ec648..ae0ef7a22e 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -375,6 +375,65 @@ pub(crate) fn gc_note_external_side_free(bytes: usize) { } #[inline] +/// The moving (copying) minor at the precise-root event-loop safepoint — +/// **Perry's default GC.** At the outermost microtask-pump boundary the JS stack +/// has unwound, so the copying minor runs with precise, rewritable roots and +/// moves survivors (compacting, O(survivors), no sweep). `PERRY_GC_MOVING_SAFEPOINT=0` +/// is a kill switch that reverts to the non-moving path (for bisecting a +/// regression while we harden). Making the moving minor primary INSIDE loops +/// (back-edge polls + alloc-point deferral) is the separate opt-in +/// `PERRY_GC_MOVING_LOOP_POLLS` — off by default because the poll defeats loop +/// vectorization until it is emitted only for allocating loops. +pub(crate) fn gc_moving_safepoint_enabled() -> bool { + static CACHED: OnceLock = OnceLock::new(); + // Default ON; the kill switch is an explicit `=0`/`off`/`false`. + *CACHED.get_or_init(|| { + !matches!( + std::env::var("PERRY_GC_MOVING_SAFEPOINT").as_deref(), + Ok("0") | Ok("off") | Ok("false") + ) + }) +} + +/// Phase 4 of the moving-GC project: gate the INCREMENTAL old-gen collector (the +/// budgeted stepper). **EXPERIMENTAL — default OFF.** Perry has a full budgeted +/// mark/sweep stepper but it never runs, because every compiled program +/// registers unbudgeted mutable root scanners and +/// `registered_root_scanners_block_budgeted_gc()` blocks the cycle from ever +/// starting. When this is on, the stepper is allowed to start and runs those +/// unbudgeted scanners SYNCHRONOUSLY in its initial root-scan step (a bounded +/// initial-mark pause), then marks/sweeps the old gen incrementally across +/// safepoints — the standard "initial-mark + incremental-mark" design. Off ⇒ +/// exactly today's non-incremental GC (the whole path is skipped). Independent +/// of `PERRY_GC_MOVING_SAFEPOINT`; this is the concurrency layer that reduces +/// old-gen pause time. +pub(crate) fn gc_incremental_enabled() -> bool { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_GC_INCREMENTAL").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + +/// Phase 2/3 (opt-in, default OFF): also make the moving minor PRIMARY inside +/// loops — defer the alloc-point nursery collection to a codegen loop back-edge +/// poll (`js_gc_loop_safepoint`) instead of collecting non-moving mid-expression. +/// Off by default because the poll emits a call in every loop, defeating +/// vectorization; when it's emitted only for allocating loops this can flip on. +/// Must match the codegen `moving_safepoint_polls_enabled` (same env) so the +/// deferral and the polls that drain it stay coherent. +pub(crate) fn gc_moving_loop_polls_enabled() -> bool { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_GC_MOVING_LOOP_POLLS").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + pub(super) fn gc_trace_enabled() -> bool { #[cfg(test)] if GC_TRACE_TEST_FORCE.with(Cell::get) { @@ -578,7 +637,23 @@ thread_local! { /// `gc_check_trigger`: the full collection must not recursively trigger /// another reclaim if a hook it runs allocates. pub(super) static GC_OLD_RECLAIM_IN_PROGRESS: Cell = const { Cell::new(false) }; -} + /// Phase 2/3 of the moving-GC project: set when an alloc-point nursery + /// trigger fires while moving mode is on, deferring the collection to the + /// next precise-root safepoint (event-loop boundary or a codegen loop + /// back-edge poll) so the copying minor can MOVE survivors instead of the + /// conservative non-moving minor running mid-expression. + pub(super) static GC_SAFEPOINT_PENDING: Cell = const { Cell::new(false) }; +} + +/// Hard cap on committed arena bytes before which a nursery trigger may be +/// deferred to a safepoint (Phase 2/3). Loop back-edge polls drain the pending +/// flag every iteration, so the arena never grows near this in normal code; the +/// cap bounds RSS for code that reaches no safepoint before the next trigger — +/// a synchronous loop on a specialized lowering path that doesn't yet emit the +/// poll, or a single mega-expression — where the alloc-point non-moving minor +/// runs as the safety valve. Kept modest so those cases don't balloon under the +/// default-on moving GC (raise once poll coverage is complete). +pub(super) const GC_MOVING_DEFER_HARD_CAP_BYTES: usize = 128 * 1024 * 1024; /// RAII guard that marks a #5476 direct old-gen reclaim in progress so a nested /// `gc_check_trigger` can't re-enter it. See `GC_OLD_RECLAIM_IN_PROGRESS`. @@ -1161,6 +1236,19 @@ pub fn gc_check_trigger() { _ => None, }; if let Some(kind) = direct_kind { + // Phase 2/3: with moving mode on, DEFER this alloc-point collection + // to the next precise-root safepoint (event-loop boundary or a + // codegen loop back-edge poll) so the copying minor MOVES survivors + // instead of the conservative non-moving minor running here at a + // register-imprecise point. Safety valve: once committed arena bytes + // pass the hard cap (a mega-expression that reached no poll), fall + // through and collect non-moving here so growth stays bounded. + if gc_moving_loop_polls_enabled() + && crate::arena::arena_total_bytes() < GC_MOVING_DEFER_HARD_CAP_BYTES + { + GC_SAFEPOINT_PENDING.with(|p| p.set(true)); + return; + } let pre_in_use = crate::arena::arena_in_use_bytes(); let pre_malloc_count = malloc_object_count(); let _scan = super::roots::ManualGcScanGuard::force_full_scan(); @@ -1303,6 +1391,70 @@ fn gc_budgeted_due_trigger() -> Option { None } +/// Phase 1 of the moving-GC project: run a copying (moving) minor at a +/// precise-root safepoint — the outermost microtask-pump boundary, where the +/// JS stack has fully unwound so no live heap pointer sits in an unspilled +/// register. Unlike the alloc-point nursery-churn arm, NO `force_full_scan` is +/// taken: `conservative_stack_scan_decision()` stays `SkipDisabled`, so the +/// copying minor is eligible with precise, rewritable roots and actually MOVES +/// (compacting, O(survivors), no sweep) instead of falling back to the +/// non-moving minor. Trigger detection + re-baseline mirror the nursery-churn +/// arm; this is purely additive (the alloc-point fallback is untouched) and +/// gated by `gc_moving_safepoint_enabled` (default off). +pub(crate) fn gc_safepoint_moving_minor() { + // Same start guards the budgeted collector uses, minus the (here + // irrelevant) scanner block: never collect mid-allocation, inside a + // runtime handle scope, in an unsafe FFI zone, or during a budgeted cycle. + if GC_FLAGS.with(|f| f.get()) & (GC_FLAG_IN_ALLOC | GC_FLAG_SUPPRESSED) != 0 + || gc_blocked_by_unsafe_zone() + || GC_ROOT_LOCK_DEPTH.with(|depth| depth.get() != 0) + || gc_budgeted_cycle_active() + { + // Blocked right now — leave GC_SAFEPOINT_PENDING set so the next poll + // retries; do not clear it here. + return; + } + // We are handling this safepoint (collect or find nothing due): clear the + // deferral flag set by the alloc-point arm (Phase 2/3). + GC_SAFEPOINT_PENDING.with(|p| p.set(false)); + // Only nursery-pressure triggers take the moving minor here; OldReclaim + // stays on its existing full mark-sweep path. + let kind = match gc_budgeted_due_trigger() { + Some(BudgetedGcTrigger::ArenaBytes) => GcTriggerKind::ArenaBytes, + Some(BudgetedGcTrigger::MallocCount) => GcTriggerKind::MallocCount, + _ => return, + }; + let pre_in_use = crate::arena::arena_in_use_bytes(); + let pre_malloc_count = malloc_object_count(); + // No `force_full_scan`: roots are precise at this safepoint. + let outcome = super::gc_collect_minor_with_trigger(GcTriggerSnapshot::capture(kind)); + match kind { + GcTriggerKind::MallocCount => { + gc_finish_malloc_trigger_collection(pre_malloc_count, outcome); + } + _ => { + gc_finish_arena_trigger_collection(pre_in_use, outcome); + } + } +} + +/// Phase 2 of the moving-GC project: codegen emits a call to this at loop +/// back-edges — but ONLY when the compiler was invoked with the moving-safepoint +/// opt-in, so default binaries carry zero loop overhead. At a back-edge the +/// loop-body expression has completed, so no heap value lives in an unspilled +/// register (every live value is a named local on the shadow stack): a +/// precise-root safepoint. If moving mode is on and an alloc-point nursery +/// trigger deferred a collection (`GC_SAFEPOINT_PENDING`), drain it here so the +/// copying minor MOVES survivors. Cheap no-op otherwise (one cached-bool load + +/// one thread-local read). +#[no_mangle] +pub extern "C" fn js_gc_loop_safepoint() { + if !gc_moving_loop_polls_enabled() || !GC_SAFEPOINT_PENDING.with(Cell::get) { + return; + } + gc_safepoint_moving_minor(); +} + struct BudgetedGcStepGuard; impl BudgetedGcStepGuard { diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index 35e64bbd11..fae364db35 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -286,6 +286,15 @@ pub(super) fn copy_only_root_scanner_counts() -> (usize, usize) { pub(super) fn registered_root_scanners_block_budgeted_gc() -> bool { let has_copy_only = ROOT_SCANNERS.with(|scanners| !scanners.borrow().is_empty()) || FFI_ROOT_SCANNERS.with(|scanners| !scanners.borrow().is_empty()); + if super::gc_incremental_enabled() { + // Phase 4 (incremental old-gen, gated): unbudgeted MUTABLE / ffi-mutable + // scanners no longer block the cycle — they run synchronously in the + // initial root-scan step (see `allow_synchronous_scanners`), a bounded + // initial-mark pause. Copy-only scanners still block: they can only + // mark, not rewrite, so a moving/incremental cycle can't tolerate them. + // (There are ~none in production — only the FFI copy-only API + tests.) + return has_copy_only; + } let has_unbudgeted_mutable = MUTABLE_ROOT_SCANNERS.with(|scanners| { scanners .borrow() diff --git a/crates/perry-runtime/src/promise/microtasks.rs b/crates/perry-runtime/src/promise/microtasks.rs index 0d69c28b84..18d4259bb3 100644 --- a/crates/perry-runtime/src/promise/microtasks.rs +++ b/crates/perry-runtime/src/promise/microtasks.rs @@ -780,6 +780,19 @@ fn run_microtasks(mode: MicrotaskDrainMode) -> i32 { let _ = crate::gc::gc_runtime_safepoint(); + // Phase 1 of the moving-GC project (see project_gc_one_great_moving_gc): at + // the OUTERMOST microtask-pump boundary the JS stack has fully unwound, so + // there are no live register temporaries and the copying (moving) minor runs + // with precise, rewritable roots — no forced conservative scan. Run it when + // nursery pressure is due so programs that yield to the event loop get + // compacting, O(survivors) young collection instead of the non-moving + // alloc-point fallback. Gated (default off); additive. + if crate::gc::gc_moving_safepoint_enabled() + && MICROTASK_RUN_DEPTH.with(|depth| depth.get()) == 1 + { + crate::gc::gc_safepoint_moving_minor(); + } + MICROTASK_RUN_DEPTH.with(|depth| { depth.set(depth.get().saturating_sub(1)); });