feat(gc): moving (copying) GC is now the default — safepoint-triggered, precise-root (Phases 1-4)

crates/perry-codegen/src/expr/index_get.rs

@@ -1173,6 +1173,10 @@ fn lower_bounded_array_index_get(
     ))
 }
 
+// #6132: retired — inline-read a value as a plain ArrayHeader, which is unsafe
+// for off-heap typed arrays (garbage reads). Callers now use the typed-feedback
+// guarded path. Kept temporarily behind allow(dead_code); slated for deletion.
+#[allow(dead_code)]
 fn lower_legacy_array_index_get(
     ctx: &mut FnCtx<'_>,
     arr_box: &str,
@@ -1658,11 +1662,15 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
                     ));
                 }
                 let idx_i32 = lower_expr_as_i32(ctx, index)?;
-                if !require_numeric_layout
-                    && !matches!(index.as_ref(), Expr::Integer(_) | Expr::Number(_))
-                {
-                    return lower_legacy_array_index_get(ctx, &arr_box, &idx_i32);
-                }
+                // #6132: a member receiver of unknown type (e.g. `n.buf[i]` where
+                // `n.buf` is a Uint32Array) must NOT go through the legacy inline
+                // reader — that path reads the value as a plain `ArrayHeader`
+                // (gc_type at `handle-8`, raw f64 slot at `handle+8+i*8`), but a
+                // small typed array is off-heap with no GcHeader, so both reads
+                // are garbage and it returned nondeterministic junk. Route through
+                // the typed-feedback-guarded path: its runtime guard rejects
+                // non-plain arrays and takes the boxed fallback (which dispatches
+                // typed arrays correctly), while plain arrays keep the fast path.
                 return lower_guarded_array_index_get(
                     ctx,
                     &arr_box,

crates/perry-codegen/src/runtime_decls/arrays.rs

@@ -87,6 +87,11 @@ pub fn declare_phase_b_arrays(module: &mut LlModule) {
     module.declare_function("js_shadow_slot_set", VOID, &[I32, I64]);
     module.declare_function("js_shadow_slot_bind", VOID, &[I32, PTR]);
     module.declare_function("js_gc_write_barriers_emitted", VOID, &[I32]);
+    // Phase 2 of the moving-GC project: emitted at loop back-edges (only when
+    // compiled with the moving-safepoint opt-in) so a deferred nursery
+    // collection can run at a precise-root safepoint. No-op at runtime unless
+    // moving mode is on and a collection is pending.
+    module.declare_function("js_gc_loop_safepoint", VOID, &[]);
 
     // Write barrier for the generational GC (Phase C per the
     // gen-GC plan). Called by codegen-emitted heap-store sites

crates/perry-codegen/src/stmt/loops.rs

@@ -2654,6 +2654,7 @@ fn lower_for_after_init_with_i32_bound(
         ctx.block().asm_sideeffect_barrier();
     }
     if !ctx.block().is_terminated() {
+        emit_gc_loop_safepoint(ctx);
         ctx.block().br(&update_label);
     }
 
@@ -2707,6 +2708,45 @@ fn lower_for_after_init_with_i32_bound(
     Ok(())
 }
 
+/// Whether to emit loop back-edge safepoint polls — OPT-IN, default OFF
+/// (`PERRY_GC_MOVING_LOOP_POLLS=1`). The moving GC is the default at the
+/// event-loop safepoint, but the loop poll emits a `js_gc_loop_safepoint()`
+/// CALL at every loop back-edge, which defeats LLVM auto-vectorization and
+/// violates the native-region "no runtime calls in hot loop" proofs. Until the
+/// poll is emitted only in loops that actually ALLOCATE (so numeric/vectorizable
+/// loops stay call-free), it is opt-in and a tight allocating loop defers to the
+/// event-loop safepoint instead.
+fn moving_safepoint_polls_enabled() -> bool {
+    use std::sync::OnceLock;
+    static CACHED: OnceLock<bool> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        matches!(
+            std::env::var("PERRY_GC_MOVING_LOOP_POLLS").as_deref(),
+            Ok("1") | Ok("on") | Ok("true")
+        )
+    })
+}
+
+/// Emit a `js_gc_loop_safepoint()` poll at a loop back-edge. Call this AFTER
+/// `clear_loop_body_shadow_slots` and only where the block is not terminated:
+/// at that point the loop-body expression has completed, so every live heap
+/// value is a named local on the shadow stack (no unspilled register temps) —
+/// a precise-root safepoint where a deferred copying minor can MOVE survivors.
+///
+/// COVERAGE (Phase 2, follow-up): currently wired into the generic `while`,
+/// `do..while`, and `for` back-edges. The specialized/versioned `for`-loop
+/// lowering paths in this file (i32-bound-optimized, packed-f64/i32/u32,
+/// bulk-fill) and `for-of`/`for-in` do NOT yet emit it, so a hot allocating
+/// loop that takes one of those paths won't drain a deferred moving minor until
+/// the next event-loop safepoint. Adding the poll to every back-edge across
+/// those paths is the remaining Phase 2 codegen work.
+pub(crate) fn emit_gc_loop_safepoint(ctx: &mut FnCtx<'_>) {
+    if !moving_safepoint_polls_enabled() || ctx.block().is_terminated() {
+        return;
+    }
+    ctx.block().call_void("js_gc_loop_safepoint", &[]);
+}
+
 pub(crate) fn clear_loop_body_shadow_slots(ctx: &mut FnCtx<'_>, body: &[Stmt]) {
     if ctx.block().is_terminated() || ctx.shadow_slot_map.is_empty() {
         return;
@@ -4454,6 +4494,7 @@ pub(crate) fn lower_while(
         ctx.block().asm_sideeffect_barrier();
     }
     if !ctx.block().is_terminated() {
+        emit_gc_loop_safepoint(ctx);
         ctx.block().br(&cond_label);
     }
     ctx.active_region_id = previous_region_id;
@@ -4511,6 +4552,7 @@ pub(crate) fn lower_do_while(
         ctx.block().asm_sideeffect_barrier();
     }
     if !ctx.block().is_terminated() {
+        emit_gc_loop_safepoint(ctx);
         ctx.block().br(&cond_label);
     }
 

crates/perry-runtime/src/gc/copying.rs

@@ -498,6 +498,29 @@ impl CopyingNurseryCollector {
         }
 
         let total = (*header).size as usize;
+        // Safety net (partial mitigation, NOT a full fix): a genuine
+        // young/survivor object is always small — large objects are allocated
+        // old-gen/malloc, never in the copying nursery — so a "young" object
+        // whose size is out of range is a corrupt/mis-classified header (e.g. an
+        // off-heap pointer whose preceding bytes coincidentally pass
+        // `plausible_gc_header`). Refuse to memmove through such a garbage size:
+        // that turns the worst outcome (a wild out-of-bounds copy → SIGSEGV)
+        // into a no-op, and surfaces it under PERRY_GC_DIAG. It does NOT catch a
+        // plausible-but-wrong *small* size; the root fix is stronger arena
+        // classification / page unregistration so off-heap addresses never
+        // reach here. See the copying-minor relocation issue.
+        const MAX_YOUNG_MOVE_BYTES: usize = 1 << 20; // 1 MiB, >> any real young object
+        if total < GC_HEADER_SIZE || total > MAX_YOUNG_MOVE_BYTES {
+            if std::env::var_os("PERRY_GC_DIAG").is_some() {
+                eprintln!(
+                    "[gc-move-guard] refusing wild young move user={:#x} obj_type={} size={}",
+                    old_user as usize,
+                    (*header).obj_type,
+                    total
+                );
+            }
+            return old_user as usize;
+        }
         let payload = total - GC_HEADER_SIZE;
         let prior_age = copied_survival_age((*header)._reserved, flags);
         let next_age = prior_age.saturating_add(1);
@@ -903,6 +926,23 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility(
         trace.root_sources.native_stack_fallback.scanned =
             matches!(decision, ConservativeStackScanDecision::Scan);
     }
+    if std::env::var_os("PERRY_GC_DIAG").is_some() {
+        let reason = match eligibility.fallback_reason {
+            CopiedMinorFallbackReason::None => "none",
+            CopiedMinorFallbackReason::NotAttempted => "not_attempted",
+            CopiedMinorFallbackReason::BarriersInactive => "barriers_inactive",
+            CopiedMinorFallbackReason::ConservativeStack => "conservative_stack",
+            CopiedMinorFallbackReason::CopyOnlyRoots => "copy_only_roots",
+            CopiedMinorFallbackReason::MallocRegistryUnavailable => "malloc_registry_unavailable",
+            CopiedMinorFallbackReason::PinnedYoungRoot => "pinned_young_root",
+            CopiedMinorFallbackReason::PinnedYoungDirtySlot => "pinned_young_dirty_slot",
+            CopiedMinorFallbackReason::PinnedYoungTransitive => "pinned_young_transitive",
+        };
+        eprintln!(
+            "[gc-copy-minor] eligible={} fallback={}",
+            eligibility.eligible, reason
+        );
+    }
     if !eligibility.eligible {
         return None;
     }
@@ -1111,6 +1151,16 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility(
         trace.capture_layout_scans();
     }
     maybe_schedule_old_reclaim_after_copied_minor();
+    if std::env::var_os("PERRY_GC_DIAG").is_some() {
+        eprintln!(
+            "[gc-copy-minor] ran copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={}",
+            collector.stats.copied_objects,
+            collector.stats.copied_bytes,
+            collector.stats.promoted_objects,
+            collector.stats.promoted_bytes,
+            freed_bytes
+        );
+    }
     Some(CopiedMinorFastPathOutcome {
         freed_bytes,
         malloc_swept: malloc_sweep_due,

crates/perry-runtime/src/gc/cycle.rs

@@ -1073,7 +1073,13 @@ impl GcCycleState {
             .is_some_and(|minor| minor.evacuation_policy.considered);
 
         self.root_scan.get_or_insert_with(RootScanCycleState::new);
-        let allow_synchronous_scanners = !self.progress_kind.is_budgeted();
+        // Phase 4 (incremental old-gen, gated): allow the initial root-scan step
+        // of a budgeted cycle to run unbudgeted scanners synchronously (a
+        // bounded initial-mark pause), so the stepper can start on programs that
+        // register unbudgeted mutable scanners. Marking still proceeds
+        // incrementally in later steps (they don't consult this flag).
+        let allow_synchronous_scanners =
+            !self.progress_kind.is_budgeted() || super::gc_incremental_enabled();
         loop {
             let phase_name = self
                 .root_scan

crates/perry-runtime/src/gc/policy.rs

@@ -375,6 +375,65 @@ pub(crate) fn gc_note_external_side_free(bytes: usize) {
 }
 
 #[inline]
+/// The moving (copying) minor at the precise-root event-loop safepoint —
+/// **Perry's default GC.** At the outermost microtask-pump boundary the JS stack
+/// has unwound, so the copying minor runs with precise, rewritable roots and
+/// moves survivors (compacting, O(survivors), no sweep). `PERRY_GC_MOVING_SAFEPOINT=0`
+/// is a kill switch that reverts to the non-moving path (for bisecting a
+/// regression while we harden). Making the moving minor primary INSIDE loops
+/// (back-edge polls + alloc-point deferral) is the separate opt-in
+/// `PERRY_GC_MOVING_LOOP_POLLS` — off by default because the poll defeats loop
+/// vectorization until it is emitted only for allocating loops.
+pub(crate) fn gc_moving_safepoint_enabled() -> bool {
+    static CACHED: OnceLock<bool> = OnceLock::new();
+    // Default ON; the kill switch is an explicit `=0`/`off`/`false`.
+    *CACHED.get_or_init(|| {
+        !matches!(
+            std::env::var("PERRY_GC_MOVING_SAFEPOINT").as_deref(),
+            Ok("0") | Ok("off") | Ok("false")
+        )
+    })
+}
+
+/// Phase 4 of the moving-GC project: gate the INCREMENTAL old-gen collector (the
+/// budgeted stepper). **EXPERIMENTAL — default OFF.** Perry has a full budgeted
+/// mark/sweep stepper but it never runs, because every compiled program
+/// registers unbudgeted mutable root scanners and
+/// `registered_root_scanners_block_budgeted_gc()` blocks the cycle from ever
+/// starting. When this is on, the stepper is allowed to start and runs those
+/// unbudgeted scanners SYNCHRONOUSLY in its initial root-scan step (a bounded
+/// initial-mark pause), then marks/sweeps the old gen incrementally across
+/// safepoints — the standard "initial-mark + incremental-mark" design. Off ⇒
+/// exactly today's non-incremental GC (the whole path is skipped). Independent
+/// of `PERRY_GC_MOVING_SAFEPOINT`; this is the concurrency layer that reduces
+/// old-gen pause time.
+pub(crate) fn gc_incremental_enabled() -> bool {
+    static CACHED: OnceLock<bool> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        matches!(
+            std::env::var("PERRY_GC_INCREMENTAL").as_deref(),
+            Ok("1") | Ok("on") | Ok("true")
+        )
+    })
+}
+
+/// Phase 2/3 (opt-in, default OFF): also make the moving minor PRIMARY inside
+/// loops — defer the alloc-point nursery collection to a codegen loop back-edge
+/// poll (`js_gc_loop_safepoint`) instead of collecting non-moving mid-expression.
+/// Off by default because the poll emits a call in every loop, defeating
+/// vectorization; when it's emitted only for allocating loops this can flip on.
+/// Must match the codegen `moving_safepoint_polls_enabled` (same env) so the
+/// deferral and the polls that drain it stay coherent.
+pub(crate) fn gc_moving_loop_polls_enabled() -> bool {
+    static CACHED: OnceLock<bool> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        matches!(
+            std::env::var("PERRY_GC_MOVING_LOOP_POLLS").as_deref(),
+            Ok("1") | Ok("on") | Ok("true")
+        )
+    })
+}
+
 pub(super) fn gc_trace_enabled() -> bool {
     #[cfg(test)]
     if GC_TRACE_TEST_FORCE.with(Cell::get) {
@@ -578,7 +637,23 @@ thread_local! {
     /// `gc_check_trigger`: the full collection must not recursively trigger
     /// another reclaim if a hook it runs allocates.
     pub(super) static GC_OLD_RECLAIM_IN_PROGRESS: Cell<bool> = const { Cell::new(false) };
-}
+    /// Phase 2/3 of the moving-GC project: set when an alloc-point nursery
+    /// trigger fires while moving mode is on, deferring the collection to the
+    /// next precise-root safepoint (event-loop boundary or a codegen loop
+    /// back-edge poll) so the copying minor can MOVE survivors instead of the
+    /// conservative non-moving minor running mid-expression.
+    pub(super) static GC_SAFEPOINT_PENDING: Cell<bool> = const { Cell::new(false) };
+}
+
+/// Hard cap on committed arena bytes before which a nursery trigger may be
+/// deferred to a safepoint (Phase 2/3). Loop back-edge polls drain the pending
+/// flag every iteration, so the arena never grows near this in normal code; the
+/// cap bounds RSS for code that reaches no safepoint before the next trigger —
+/// a synchronous loop on a specialized lowering path that doesn't yet emit the
+/// poll, or a single mega-expression — where the alloc-point non-moving minor
+/// runs as the safety valve. Kept modest so those cases don't balloon under the
+/// default-on moving GC (raise once poll coverage is complete).
+pub(super) const GC_MOVING_DEFER_HARD_CAP_BYTES: usize = 128 * 1024 * 1024;
 
 /// RAII guard that marks a #5476 direct old-gen reclaim in progress so a nested
 /// `gc_check_trigger` can't re-enter it. See `GC_OLD_RECLAIM_IN_PROGRESS`.
@@ -1161,6 +1236,19 @@ pub fn gc_check_trigger() {
             _ => None,
         };
         if let Some(kind) = direct_kind {
+            // Phase 2/3: with moving mode on, DEFER this alloc-point collection
+            // to the next precise-root safepoint (event-loop boundary or a
+            // codegen loop back-edge poll) so the copying minor MOVES survivors
+            // instead of the conservative non-moving minor running here at a
+            // register-imprecise point. Safety valve: once committed arena bytes
+            // pass the hard cap (a mega-expression that reached no poll), fall
+            // through and collect non-moving here so growth stays bounded.
+            if gc_moving_loop_polls_enabled()
+                && crate::arena::arena_total_bytes() < GC_MOVING_DEFER_HARD_CAP_BYTES
+            {
+                GC_SAFEPOINT_PENDING.with(|p| p.set(true));
+                return;
+            }
             let pre_in_use = crate::arena::arena_in_use_bytes();
             let pre_malloc_count = malloc_object_count();
             let _scan = super::roots::ManualGcScanGuard::force_full_scan();
@@ -1303,6 +1391,70 @@ fn gc_budgeted_due_trigger() -> Option<BudgetedGcTrigger> {
     None
 }
 
+/// Phase 1 of the moving-GC project: run a copying (moving) minor at a
+/// precise-root safepoint — the outermost microtask-pump boundary, where the
+/// JS stack has fully unwound so no live heap pointer sits in an unspilled
+/// register. Unlike the alloc-point nursery-churn arm, NO `force_full_scan` is
+/// taken: `conservative_stack_scan_decision()` stays `SkipDisabled`, so the
+/// copying minor is eligible with precise, rewritable roots and actually MOVES
+/// (compacting, O(survivors), no sweep) instead of falling back to the
+/// non-moving minor. Trigger detection + re-baseline mirror the nursery-churn
+/// arm; this is purely additive (the alloc-point fallback is untouched) and
+/// gated by `gc_moving_safepoint_enabled` (default off).
+pub(crate) fn gc_safepoint_moving_minor() {
+    // Same start guards the budgeted collector uses, minus the (here
+    // irrelevant) scanner block: never collect mid-allocation, inside a
+    // runtime handle scope, in an unsafe FFI zone, or during a budgeted cycle.
+    if GC_FLAGS.with(|f| f.get()) & (GC_FLAG_IN_ALLOC | GC_FLAG_SUPPRESSED) != 0
+        || gc_blocked_by_unsafe_zone()
+        || GC_ROOT_LOCK_DEPTH.with(|depth| depth.get() != 0)
+        || gc_budgeted_cycle_active()
+    {
+        // Blocked right now — leave GC_SAFEPOINT_PENDING set so the next poll
+        // retries; do not clear it here.
+        return;
+    }
+    // We are handling this safepoint (collect or find nothing due): clear the
+    // deferral flag set by the alloc-point arm (Phase 2/3).
+    GC_SAFEPOINT_PENDING.with(|p| p.set(false));
+    // Only nursery-pressure triggers take the moving minor here; OldReclaim
+    // stays on its existing full mark-sweep path.
+    let kind = match gc_budgeted_due_trigger() {
+        Some(BudgetedGcTrigger::ArenaBytes) => GcTriggerKind::ArenaBytes,
+        Some(BudgetedGcTrigger::MallocCount) => GcTriggerKind::MallocCount,
+        _ => return,
+    };
+    let pre_in_use = crate::arena::arena_in_use_bytes();
+    let pre_malloc_count = malloc_object_count();
+    // No `force_full_scan`: roots are precise at this safepoint.
+    let outcome = super::gc_collect_minor_with_trigger(GcTriggerSnapshot::capture(kind));
+    match kind {
+        GcTriggerKind::MallocCount => {
+            gc_finish_malloc_trigger_collection(pre_malloc_count, outcome);
+        }
+        _ => {
+            gc_finish_arena_trigger_collection(pre_in_use, outcome);
+        }
+    }
+}
+
+/// Phase 2 of the moving-GC project: codegen emits a call to this at loop
+/// back-edges — but ONLY when the compiler was invoked with the moving-safepoint
+/// opt-in, so default binaries carry zero loop overhead. At a back-edge the
+/// loop-body expression has completed, so no heap value lives in an unspilled
+/// register (every live value is a named local on the shadow stack): a
+/// precise-root safepoint. If moving mode is on and an alloc-point nursery
+/// trigger deferred a collection (`GC_SAFEPOINT_PENDING`), drain it here so the
+/// copying minor MOVES survivors. Cheap no-op otherwise (one cached-bool load +
+/// one thread-local read).
+#[no_mangle]
+pub extern "C" fn js_gc_loop_safepoint() {
+    if !gc_moving_loop_polls_enabled() || !GC_SAFEPOINT_PENDING.with(Cell::get) {
+        return;
+    }
+    gc_safepoint_moving_minor();
+}
+
 struct BudgetedGcStepGuard;
 
 impl BudgetedGcStepGuard {