From fb03b961c27070534810733a0fb6e9183f30745c Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:13:44 -0700 Subject: [PATCH 01/22] Refactor: retire the PTO2 prefix on the resource-shape type `PTO2ResourceShape` becomes `ResourceShape` and `PTO2_NUM_RESOURCE_SHAPES` becomes `NUM_RESOURCE_SHAPES`. Both names are free: nothing else in the tree, in the CANN headers, or in pto-isa spells either of them, so this is a plain prefix strip. 474 occurrences across 46 files. The constant is that enum's cardinality, so it moves with the type rather than in a commit of its own. Co-Authored-By: Claude Opus 5 (1M context) --- .../orchestrator_core/orchestrator.cpp | 4 +- .../runtime/scheduler/scheduler.cpp | 2 +- .../runtime/scheduler/scheduler.h | 20 +++---- .../scheduler/scheduler_completion.cpp | 18 +++--- .../runtime/scheduler/scheduler_context.h | 26 ++++----- .../runtime/scheduler/scheduler_dispatch.cpp | 58 +++++++++---------- .../runtime/scheduler/scheduler_types.h | 30 +++++----- .../runtime/shared/runtime_init.cpp | 30 +++++----- .../host_build_graph/runtime/submit_types.h | 14 ++--- .../docs/RUNTIME_LOGIC.md | 2 +- .../runtime/orchestrator.cpp | 4 +- .../runtime/scheduler/scheduler.cpp | 2 +- .../runtime/scheduler/scheduler.h | 32 +++++----- .../scheduler/scheduler_completion.cpp | 18 +++--- .../runtime/scheduler/scheduler_context.h | 26 ++++----- .../runtime/scheduler/scheduler_dispatch.cpp | 58 +++++++++---------- .../runtime/scheduler/scheduler_types.h | 30 +++++----- .../runtime/shared/runtime_init.cpp | 30 +++++----- .../runtime/submit_types.h | 14 ++--- .../orchestrator_core/orchestrator.cpp | 4 +- .../runtime/scheduler/scheduler.cpp | 2 +- .../runtime/scheduler/scheduler.h | 20 +++---- .../scheduler/scheduler_completion.cpp | 18 +++--- .../runtime/scheduler/scheduler_context.h | 26 ++++----- .../runtime/scheduler/scheduler_dispatch.cpp | 58 +++++++++---------- .../runtime/scheduler/scheduler_types.h | 30 +++++----- .../runtime/shared/runtime_init.cpp | 30 +++++----- .../host_build_graph/runtime/submit_types.h | 14 ++--- .../docs/RUNTIME_LOGIC.md | 2 +- .../runtime/orchestrator.cpp | 4 +- .../runtime/scheduler/scheduler.cpp | 2 +- .../runtime/scheduler/scheduler.h | 32 +++++----- .../scheduler/scheduler_completion.cpp | 16 ++--- .../runtime/scheduler/scheduler_context.h | 26 ++++----- .../runtime/scheduler/scheduler_dispatch.cpp | 58 +++++++++---------- .../runtime/scheduler/scheduler_types.h | 30 +++++----- .../runtime/shared/runtime_init.cpp | 30 +++++----- .../runtime/submit_types.h | 14 ++--- .../include/common/chip_swimlane_profiling.h | 2 +- tests/ut/cpp/a2a3/test_graph_activation.cpp | 6 +- tests/ut/cpp/a2a3/test_scheduler_state.cpp | 32 +++++----- tests/ut/cpp/a2a3/test_wiring.cpp | 14 ++--- tests/ut/cpp/a5/test_graph_activation.cpp | 6 +- tests/ut/cpp/a5/test_scheduler_state.cpp | 36 ++++++------ tests/ut/cpp/a5/test_wiring.cpp | 12 ++-- .../cpp/common/test_hbg_scheduler_drain.cpp | 2 +- 46 files changed, 468 insertions(+), 476 deletions(-) diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 246215e379..9e72bb384b 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -2575,8 +2575,8 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k // Deadlock check: block_num >= total available slots of the required type. // For MIX/AIC: limit is total_cluster_count (one AIC per cluster). // For AIV: limit is total_aiv_count. - PTO2ResourceShape shape = active_mask.to_shape(); - int32_t limit = (shape == PTO2ResourceShape::AIV) ? orch->total_aiv_count : orch->total_cluster_count; + ResourceShape shape = active_mask.to_shape(); + int32_t limit = (shape == ResourceShape::AIV) ? orch->total_aiv_count : orch->total_cluster_count; if (limit > 0 && block_num > limit) { report_fatal( SIMPLER_ERROR_REQUIRE_SYNC_START_INVALID, __FUNCTION__, diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.cpp index 1a23c66fe3..6458fcb747 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.cpp @@ -77,7 +77,7 @@ void PTO2SchedulerState::print_stats() { void PTO2SchedulerState::print_queues() { PTO2SchedulerState *sched = this; const char *shape_names[] = {"AIC", "AIV", "MIX"}; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { LOG_TIMING( "QPROBE rq[%s] pushes=%llu maxocc=%llu cap=%llu", shape_names[i], (unsigned long long)sched->ready_queues[i].enqueue_pos.load(std::memory_order_relaxed), diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h index f3fb6cc7ba..720c0c63bf 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -453,12 +453,12 @@ struct CompletionStats { * capacities used at layout time (init_from_layout reuses them). */ struct PTO2SchedulerLayout { - size_t off_ready_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; - size_t off_ready_sync_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; + size_t off_ready_queue_slots[NUM_RESOURCE_SHAPES]; + size_t off_ready_sync_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_dummy_ready_queue_slots; size_t off_graph_ready_queue_slots; size_t off_graph_prepare_queue_slots; - size_t off_early_dispatch_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; + size_t off_early_dispatch_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_early_sync_start_queue_slots; uint64_t ready_queue_capacity; }; @@ -489,14 +489,14 @@ struct PTO2SchedulerState { } ring_sched_state; // Ready queues remain global (scheduling is ring-agnostic) - PTO2ReadyQueue ready_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; // Ready sync_start queues, one per shape. A ready sync_start cohort parks here // instead of ready_queues[] so the dispatch loop can drain it as a strict Tier-0 // (sync_start > MIX > C/V) before any regular ready task takes a core, while // reusing the same per-shape dispatch_shape machinery (fits-local inline vs // stop-the-world drain, per-core MIX placement, head-start spacing). - PTO2ReadyQueue ready_sync_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; // Dependency-only tasks (active_mask is empty, shape == DUMMY). Drained by // the dispatch loop and completed inline -- never goes to AICore. @@ -548,8 +548,8 @@ struct PTO2SchedulerState { if (slot_state->task_kind == TaskKind::GRAPH) { pushed = graph_ready_queue.push(slot_state); } else { - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); - if (shape == PTO2ResourceShape::DUMMY || + ResourceShape shape = slot_state->active_mask.to_shape(); + if (shape == ResourceShape::DUMMY || (slot_state->task_attrs.has_predicate() && !slot_state->payload->predicate.pass())) { pushed = dummy_ready_queue.push(slot_state); } else if (slot_state->task_attrs.requires_sync_start()) { @@ -707,7 +707,7 @@ struct PTO2SchedulerState { // the ready queue (scheduler_dispatch: pop -> claim -> re-push). A stale/released // entry fails the STAGING check on pop and is dropped; a push that overflows is // logged and the consumer's blocks fall back to normal dispatch. - PTO2ReadyQueue early_dispatch_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; // sync_start early-dispatch candidates park here instead of early_dispatch_queues[]: // they need an atomic all-or-nothing stage, not early_dispatch_shape's @@ -879,13 +879,13 @@ struct PTO2SchedulerState { // ready_sync_queues (unaffected). void propagate_dispatch_fanin(ChipTaskSlotState & /*p*/) {} - int get_ready_tasks_batch(PTO2ReadyQueue *queues, PTO2ResourceShape shape, ChipTaskSlotState **out, int max_count) { + int get_ready_tasks_batch(PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { return queues[static_cast(shape)].pop_batch(out, max_count); } #if SIMPLER_SCHED_PROFILING int get_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, + PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, uint64_t &wait_cycle ) { return queues[static_cast(shape)].pop_batch(out, max_count, atomic_count, wait_cycle); diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index 3f8d333b83..44e1579ce9 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -464,7 +464,7 @@ bool SchedulerContext::enter_drain_mode(ChipTaskSlotState *slot_state, int32_t b // include_pending adds each thread's pending-capable cores/clusters — used by the // gated (early) sync_start drain, which pre-stages onto idle running slots AND onto // busy cores' pending slots. The ready drain (include_pending=false) counts idle only. -int32_t SchedulerContext::count_global_available(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending) { +int32_t SchedulerContext::count_global_available(ResourceShape shape, uint8_t core_mask, bool include_pending) { int32_t total = 0; for (int32_t t = 0; t < active_sched_threads_; t++) { total += core_trackers_[t].count_available_blocks(shape, core_mask, include_pending); @@ -492,9 +492,9 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( [[maybe_unused]] bool record_drain_phases ) { CoreTracker &tracker = core_trackers_[thread_idx]; - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); + ResourceShape shape = slot_state->active_mask.to_shape(); uint8_t core_mask = slot_state->active_mask.core_mask(); - bool mix_split = gated && shape == PTO2ResourceShape::MIX; + bool mix_split = gated && shape == ResourceShape::MIX; SyncStartStageResult result; // Stage from this thread's `valid` cores/clusters: CAS-claim a block-index range sized to @@ -518,12 +518,12 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( int32_t claimed[CoreTracker::MAX_CLUSTERS * 3]; for (int32_t b = 0; b < claim; b++) claimed[b] = valid.pop_first(); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); if (claim > 0) prefetch_block_dst(thread_idx, claimed[0], is_mix); for (int32_t b = 0; b < claim; b++) { if (b + 1 < claim) prefetch_block_dst(thread_idx, claimed[b + 1], is_mix); auto core_offset = claimed[b]; - if (shape == PTO2ResourceShape::MIX) { + if (shape == ResourceShape::MIX) { result.running_cores += tracker.mix_cluster_idle_core_count(core_offset, core_mask); } handle_count += prepare_block_for_dispatch( @@ -581,7 +581,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( sched_->record_published_blocks(*slot_state, claim); // AIC/AIV running placement (whole block on idle cores); MIX running cores are // counted per-cluster above (mix_cluster_idle_core_count). - if (gated && shape != PTO2ResourceShape::MIX && !to_pending) result.running_cores += handle_count; + if (gated && shape != ResourceShape::MIX && !to_pending) result.running_cores += handle_count; } }; @@ -590,8 +590,8 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( // used cores take running slots (prepare_block_for_dispatch: to_pending && !is_core_idle). stage(tracker.get_mix_split_cluster_offset_states(core_mask), /*to_pending=*/true); } else { - auto idle = (shape == PTO2ResourceShape::MIX) ? tracker.get_mix_running_cluster_offset_states(core_mask) : - tracker.get_idle_core_offset_states(shape); + auto idle = (shape == ResourceShape::MIX) ? tracker.get_mix_running_cluster_offset_states(core_mask) : + tracker.get_idle_core_offset_states(shape); stage(idle, /*to_pending=*/false); // idle -> running (ready launch + gated pre-stage) if (gated) { stage(tracker.get_pending_core_offset_states(shape), /*to_pending=*/true); @@ -680,7 +680,7 @@ void SchedulerContext::handle_drain_mode(int32_t thread_idx, [[maybe_unused]] ui bool gated = slot_state->payload != nullptr && PTO2SchedulerState::owns_early_sync_drain(*slot_state->payload); if (coordinator) { - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); + ResourceShape shape = slot_state->active_mask.to_shape(); // A gated drain may pre-stage onto pending slots too (idle+pending); the ready drain // needs block_num idle cores/clusters. int32_t available = diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index 693e85729c..249c3223cd 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -281,7 +281,7 @@ class SchedulerContext { // Dispatch (scheduler_dispatch.cpp) // ========================================================================= - static const char *shape_name(PTO2ResourceShape shape); + static const char *shape_name(ResourceShape shape); // Lower-case rendering of PTO2SubtaskSlot, used by dispatch and stall logs. // Kept lower-case to match the `kernels=[aic:N aiv0:N aiv1:N]` field @@ -299,7 +299,7 @@ class SchedulerContext { } int pop_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ); void build_payload( @@ -380,12 +380,12 @@ class SchedulerContext { // Fan out one block's subtasks (1 for AIC/AIV, 1-3 for MIX) into the // caller-supplied handles buffer. Returns the number of handles written. int prepare_block_for_dispatch( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2ResourceShape shape, - bool to_pending, int32_t block_idx, PublishHandle *out_handles, bool force_gate = false + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, ResourceShape shape, bool to_pending, + int32_t block_idx, PublishHandle *out_handles, bool force_gate = false ); void dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ); @@ -407,7 +407,7 @@ class SchedulerContext { // concurrently with peers (mirrors the normal SPMD dispatch path). Returns the // number of blocks staged. int32_t stage_consumer_blocks( - int32_t thread_idx, ChipTaskSlotState *c, PTO2ResourceShape shape, int32_t start, int32_t count, + int32_t thread_idx, ChipTaskSlotState *c, ResourceShape shape, int32_t start, int32_t count, CoreTracker::BitStates &idle, CoreTracker::BitStates &pend ); @@ -416,7 +416,7 @@ class SchedulerContext { // given phase (IDLE -> onto idle cores in the RUNNING slot; PENDING -> onto a // running core's gated pending slot). Pop is sized to the shape's capacity exactly // as dispatch_shape sizes normal dispatch. Returns the number of blocks staged. - int32_t early_dispatch_shape(int32_t thread_idx, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase); + int32_t early_dispatch_shape(int32_t thread_idx, ResourceShape shape, CoreTracker::DispatchPhase phase); // One pass of "Phase 4" in the resolve_and_dispatch loop: IDLE-stage dispatch // for MIX then (if no mix residual) AIC/AIV; then PENDING-stage dispatch with @@ -448,22 +448,20 @@ class SchedulerContext { // matching `shape`. Used as a scheduling hint on the PENDING dispatch path // — see the implementation in scheduler_dispatch.cpp for the hint-semantics // rationale and the safety argument against the drain worker. - bool has_idle_in_other_threads(int32_t self_thread_idx, PTO2ResourceShape shape) const; + bool has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const; // True if mix tasks remain in the global MIX ready queue. Approximate — // PTO2ReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. - bool has_residual_mix() const { - return sched_->ready_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; - } + bool has_residual_mix() const { return sched_->ready_queues[static_cast(ResourceShape::MIX)].size() > 0; } // Tier-0 analog of has_residual_mix for the ready sync_start lane: true if MIX // sync_start cohorts remain queued, so the Tier-0 pass keeps MIX strict priority // over its own AIC/AIV sync work. Same relaxed-size snapshot caveat. bool has_residual_sync_mix() const { - return sched_->ready_sync_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; + return sched_->ready_sync_queues[static_cast(ResourceShape::MIX)].size() > 0; } // Early-dispatch analog of has_residual_mix: true if MIX early-dispatch candidates @@ -472,7 +470,7 @@ class SchedulerContext { // empty), so early-dispatch MIX priority needs its own residual check against // early_dispatch_queues[MIX]. Same relaxed-size snapshot caveat as has_residual_mix. bool has_residual_early_mix() const { - return sched_->early_dispatch_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; + return sched_->early_dispatch_queues[static_cast(ResourceShape::MIX)].size() > 0; } // ========================================================================= @@ -501,7 +499,7 @@ class SchedulerContext { ); bool enter_drain_mode(ChipTaskSlotState *slot_state, int32_t block_num); - int32_t count_global_available(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending = false); + int32_t count_global_available(ResourceShape shape, uint8_t core_mask, bool include_pending = false); struct SyncStartStageResult { int32_t staged_blocks{0}; int32_t running_cores{0}; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 3903830a97..632f4b88e4 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -59,21 +59,21 @@ static_assert( "staged_core_mask too small for RUNTIME_MAX_WORKER cores" ); -const char *SchedulerContext::shape_name(PTO2ResourceShape shape) { +const char *SchedulerContext::shape_name(ResourceShape shape) { switch (shape) { - case PTO2ResourceShape::AIC: + case ResourceShape::AIC: return "AIC"; - case PTO2ResourceShape::AIV: + case ResourceShape::AIV: return "AIV"; - case PTO2ResourceShape::MIX: + case ResourceShape::MIX: return "MIX"; - case PTO2ResourceShape::DUMMY: + case ResourceShape::DUMMY: return "DUMMY"; } return "UNKNOWN"; } -bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, PTO2ResourceShape shape) const { +bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const { // Cross-thread read of peer trackers without explicit synchronization. The // backing `core_states_` is a naturally aligned uint64_t; aarch64 guarantees // single-copy atomicity for an 8-byte aligned load, so no torn read. The @@ -92,7 +92,7 @@ bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, PTO2Re } int SchedulerContext::pop_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ) { #if SIMPLER_DFX auto &chip_swimlane = sched_chip_swimlane_[thread_idx]; @@ -242,7 +242,7 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( } int SchedulerContext::prepare_block_for_dispatch( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2ResourceShape shape, bool to_pending, + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, ResourceShape shape, bool to_pending, int32_t block_idx, PublishHandle *out_handles, bool force_gate ) { #if SIMPLER_DFX @@ -260,7 +260,7 @@ int SchedulerContext::prepare_block_for_dispatch( } #endif CoreTracker &tracker = core_trackers_[thread_idx]; - if (shape == PTO2ResourceShape::MIX) { + if (shape == ResourceShape::MIX) { uint8_t cmask = slot_state.active_mask.core_mask(); int n = 0; // Per-core slot placement (#1308): an idle used core takes its running slot @@ -293,7 +293,7 @@ int SchedulerContext::prepare_block_for_dispatch( sched_chip_swimlane_[thread_idx].phase_dispatch_count += __builtin_popcount(cmask); #endif return n; - } else if (shape == PTO2ResourceShape::AIC) { + } else if (shape == ResourceShape::AIC) { out_handles[0] = prepare_subtask_to_core( thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIC, to_pending, block_idx, force_gate ); @@ -313,7 +313,7 @@ int SchedulerContext::prepare_block_for_dispatch( } void SchedulerContext::dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ) { #if SIMPLER_SCHED_PROFILING @@ -322,7 +322,7 @@ void SchedulerContext::dispatch_shape( if (entered_drain) return; bool is_pending = (phase == CoreTracker::DispatchPhase::PENDING); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); auto cores = is_mix ? tracker.get_cluster_offset_states() : tracker.get_dispatchable_cores(shape, phase); if (!cores.has_value()) return; @@ -508,14 +508,14 @@ void SchedulerContext::run_staging_order( // MIX is handled explicitly at the top of each stage; only AIC/AIV cycle // through this 2-elem array, with order toggled by thread parity for // shape-level load balancing across threads. - static constexpr PTO2ResourceShape kAicAivOrder[2][2] = { - {PTO2ResourceShape::AIC, PTO2ResourceShape::AIV}, - {PTO2ResourceShape::AIV, PTO2ResourceShape::AIC}, + static constexpr ResourceShape kAicAivOrder[2][2] = { + {ResourceShape::AIC, ResourceShape::AIV}, + {ResourceShape::AIV, ResourceShape::AIC}, }; - const PTO2ResourceShape *aic_aiv = kAicAivOrder[thread_idx & 1]; + const ResourceShape *aic_aiv = kAicAivOrder[thread_idx & 1]; // ===== IDLE stage ===== - if (stage(PTO2ResourceShape::MIX, Phase::IDLE)) return; + if (stage(ResourceShape::MIX, Phase::IDLE)) return; // MIX-IDLE residual: AIC/AIV (both IDLE and PENDING) yield for this pass. // MIX-PENDING below still runs — that is the core of "mix strict priority": @@ -539,8 +539,8 @@ void SchedulerContext::run_staging_order( // // The gate is NOT subject to skip_aic_aiv — residual mix continues to drain // via pending slots on this thread when no peer is idle. - if (!has_idle_in_other_threads(thread_idx, PTO2ResourceShape::MIX)) { - if (stage(PTO2ResourceShape::MIX, Phase::PENDING)) return; + if (!has_idle_in_other_threads(thread_idx, ResourceShape::MIX)) { + if (stage(ResourceShape::MIX, Phase::PENDING)) return; } // Re-check after MIX-PENDING. If MIX-IDLE already set skip_aic_aiv, leave @@ -554,7 +554,7 @@ void SchedulerContext::run_staging_order( // AIC/AIV-PENDING gate: a peer-idle skip is a delay, not a loss — the peer // will pull from the global queue on its next IDLE pass. for (int i = 0; i < 2; i++) { - PTO2ResourceShape s = aic_aiv[i]; + ResourceShape s = aic_aiv[i]; if (has_idle_in_other_threads(thread_idx, s)) continue; if (stage(s, Phase::PENDING)) return; } @@ -573,7 +573,7 @@ void SchedulerContext::dispatch_ready_tasks( // (entered_drain), which also short-circuits the regular tier below. run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { dispatch_shape( thread_idx, sched_->ready_sync_queues, shape, phase, tracker, entered_drain, made_progress, try_pushed ); @@ -588,7 +588,7 @@ void SchedulerContext::dispatch_ready_tasks( // Tier 1: regular ready work. run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { dispatch_shape( thread_idx, sched_->ready_queues, shape, phase, tracker, entered_drain, made_progress, try_pushed ); @@ -620,7 +620,7 @@ void SchedulerContext::dispatch_ready_tasks( // those release did not take and rings them from its immutable local handles. // The seq_cst order guarantees every gated core has exactly one writer. int32_t SchedulerContext::stage_consumer_blocks( - int32_t thread_idx, ChipTaskSlotState *c, PTO2ResourceShape shape, int32_t start, int32_t count, + int32_t thread_idx, ChipTaskSlotState *c, ResourceShape shape, int32_t start, int32_t count, CoreTracker::BitStates &idle, CoreTracker::BitStates &pend ) { CoreTracker &tracker = core_trackers_[thread_idx]; @@ -706,10 +706,10 @@ int32_t SchedulerContext::stage_consumer_blocks( // propagate_dispatch_fanin, so the shape is the queue index (no per-consumer // to_shape()). Returns the number of blocks staged. int32_t -SchedulerContext::early_dispatch_shape(int32_t thread_idx, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { +SchedulerContext::early_dispatch_shape(int32_t thread_idx, ResourceShape shape, CoreTracker::DispatchPhase phase) { CoreTracker &tracker = core_trackers_[thread_idx]; int32_t s = static_cast(shape); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); bool is_idle = (phase == CoreTracker::DispatchPhase::IDLE); // Size the pop exactly as dispatch_shape does: MIX to the cluster count, else the @@ -808,7 +808,7 @@ int32_t SchedulerContext::try_early_dispatch( // regular ready_queues) strictly precedes early — there is no real ready task // to delay only when every normal queue is drained. if (pmu_active || !tracker.has_any_free_slot()) return 0; - for (int s = 0; s < PTO2_NUM_RESOURCE_SHAPES; s++) { + for (int s = 0; s < NUM_RESOURCE_SHAPES; s++) { if (sched_->ready_sync_queues[s].size() > 0 || sched_->ready_queues[s].size() > 0) return 0; } @@ -860,7 +860,7 @@ int32_t SchedulerContext::try_early_dispatch( // enters drain, so the stage callback always reports "no stop". run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { total_staged += early_dispatch_shape(thread_idx, shape, phase); return false; }, @@ -1108,7 +1108,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // snapshot so the next phase's "at_start" equals the previous phase's // "at_end". // - // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches PTO2_NUM_RESOURCE_SHAPES: AIC/AIV/MIX. + // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches NUM_RESOURCE_SHAPES: AIC/AIV/MIX. // // **Hot-path cost discipline.** Shared depth (PTO2ReadyQueue::size) is two // atomic relaxed loads against cache lines that all peer sched threads also @@ -1119,7 +1119,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // complete-emit and dispatch-emit in the same iter both reuse the same // shared sample. static_assert( - CHIP_SWIMLANE_NUM_QUEUE_SHAPES == PTO2_NUM_RESOURCE_SHAPES, + CHIP_SWIMLANE_NUM_QUEUE_SHAPES == NUM_RESOURCE_SHAPES, "queue snapshot width must match runtime resource shape count" ); int16_t phase_start_shared[CHIP_SWIMLANE_NUM_QUEUE_SHAPES] = {0}; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h index 012aa37868..3ba97ee8a7 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h @@ -288,15 +288,15 @@ class alignas(64) CoreTracker { // --- Cluster matching --- - BitStates get_valid_cluster_offset_states(PTO2ResourceShape shape) const { + BitStates get_valid_cluster_offset_states(ResourceShape shape) const { switch (shape) { - case PTO2ResourceShape::AIC: + case ResourceShape::AIC: return core_states_ & aic_mask_; - case PTO2ResourceShape::AIV: + case ResourceShape::AIV: return ((core_states_ >> 1) | (core_states_ >> 2)) & aic_mask_; - case PTO2ResourceShape::MIX: + case ResourceShape::MIX: return (core_states_ >> 1) & (core_states_ >> 2) & core_states_ & aic_mask_; - case PTO2ResourceShape::DUMMY: + case ResourceShape::DUMMY: // DUMMY tasks never reach the core-tracker dispatch path; they are // completed inline by resolve_and_dispatch via dummy_ready_queue. return {}; @@ -345,11 +345,11 @@ class alignas(64) CoreTracker { // always have pending_occupied=0, so AIV/MIX need no extra filtering. // Skipping the AIC-centric filter also fixes a latent bug where a running+pending AIC core // would incorrectly block AIV idle dispatch on the same cluster. - BitStates get_idle_core_offset_states(PTO2ResourceShape shape) const { - if (shape == PTO2ResourceShape::AIC) { + BitStates get_idle_core_offset_states(ResourceShape shape) const { + if (shape == ResourceShape::AIC) { return get_valid_cluster_offset_states(shape) & ~(pending_occupied_ & aic_mask_); } - if (shape == PTO2ResourceShape::AIV) { + if (shape == ResourceShape::AIV) { return core_states_ & aiv_mask_; } return get_valid_cluster_offset_states(shape); // MIX: cluster-level @@ -461,11 +461,11 @@ class alignas(64) CoreTracker { // staging. AIC/AIV count cores; MIX counts clusters because one logical MIX // block may occupy multiple cores in the same cluster. Gated early staging // includes pending slots, while ready staging is restricted to idle slots. - int32_t count_available_blocks(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending) const { - if (shape == PTO2ResourceShape::MIX) { + int32_t count_available_blocks(ResourceShape shape, uint8_t core_mask, bool include_pending) const { + if (shape == ResourceShape::MIX) { return include_pending ? count_mix_split_clusters(core_mask) : count_mix_running_clusters(core_mask); } - if (shape == PTO2ResourceShape::DUMMY) return 0; + if (shape == ResourceShape::DUMMY) return 0; int32_t available = get_idle_core_offset_states(shape).count(); if (include_pending) { @@ -474,8 +474,8 @@ class alignas(64) CoreTracker { return available; } - BitStates get_pending_core_offset_states(PTO2ResourceShape shape) const { - if (shape == PTO2ResourceShape::MIX) { + BitStates get_pending_core_offset_states(ResourceShape shape) const { + if (shape == ResourceShape::MIX) { // Shape-level query kept conservative for legacy callers/tests. // The real MIX dispatch path applies active_mask in classify_mix_cluster(). // Any core without a pending payload can accept a dispatch (idle or running). @@ -489,7 +489,7 @@ class alignas(64) CoreTracker { (running & aic_mask_) & ((running >> 1) & aic_mask_) & ((running >> 2) & aic_mask_); return mix_available & cluster_all_running; } - if (shape == PTO2ResourceShape::AIC) { + if (shape == ResourceShape::AIC) { return (~core_states_) & aic_mask_ & ~(pending_occupied_ & aic_mask_); } // AIV @@ -500,7 +500,7 @@ class alignas(64) CoreTracker { enum class DispatchPhase : uint8_t { IDLE, PENDING }; - BitStates get_dispatchable_cores(PTO2ResourceShape shape, DispatchPhase phase) const { + BitStates get_dispatchable_cores(ResourceShape shape, DispatchPhase phase) const { return (phase == DispatchPhase::IDLE) ? get_idle_core_offset_states(shape) : get_pending_core_offset_states(shape); } diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 833f5bf5b5..71d205a338 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -85,14 +85,14 @@ PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena) { // Fixed-capacity early-dispatch queues first, then the PTO2_READY_QUEUE_SIZE // ones. The big nine are the arena's last reservations so that the bytes bind // uploads stay one contiguous range no matter how much of them is in use. - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); } layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); } layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); @@ -117,16 +117,16 @@ bool PTO2SchedulerState::init_data_from_layout( return false; } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_init_data_from_layout(&sched->ready_queues[i], layout.ready_queue_capacity); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_init_data_from_layout(&sched->ready_sync_queues[i], layout.ready_queue_capacity); } ready_queue_init_data_from_layout(&sched->dummy_ready_queue, layout.ready_queue_capacity); ready_queue_init_data_from_layout(&sched->graph_ready_queue, layout.ready_queue_capacity); ready_queue_init_data_from_layout(&sched->graph_prepare_queue, layout.ready_queue_capacity); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_init_data_from_layout(&sched->early_dispatch_queues[i], PTO2_EARLY_DISPATCH_QUEUE_SIZE); } ready_queue_init_data_from_layout(&sched->early_sync_start_queue, PTO2_EARLY_DISPATCH_QUEUE_SIZE); @@ -143,16 +143,16 @@ bool PTO2SchedulerState::init_data_from_layout( // sequence already equals the position being claimed. void PTO2SchedulerState::seed_queue_slots() { PTO2SchedulerState *sched = this; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_queues[i].seed_slots(); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_sync_queues[i].seed_slots(); } sched->dummy_ready_queue.seed_slots(); sched->graph_ready_queue.seed_slots(); sched->graph_prepare_queue.seed_slots(); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->early_dispatch_queues[i].seed_slots(); } sched->early_sync_start_queue.seed_slots(); @@ -160,16 +160,16 @@ void PTO2SchedulerState::seed_queue_slots() { void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena) { PTO2SchedulerState *sched = this; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_queues[i], arena, layout.off_ready_queue_slots[i]); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_sync_queues[i], arena, layout.off_ready_sync_queue_slots[i]); } ready_queue_wire_arena_pointers(&sched->dummy_ready_queue, arena, layout.off_dummy_ready_queue_slots); ready_queue_wire_arena_pointers(&sched->graph_ready_queue, arena, layout.off_graph_ready_queue_slots); ready_queue_wire_arena_pointers(&sched->graph_prepare_queue, arena, layout.off_graph_prepare_queue_slots); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers( &sched->early_dispatch_queues[i], arena, layout.off_early_dispatch_queue_slots[i] ); @@ -180,16 +180,16 @@ void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, void PTO2SchedulerState::destroy() { PTO2SchedulerState *sched = this; sched->ring_sched_state.destroy(); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_queues[i]); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_sync_queues[i]); } ready_queue_destroy(&sched->dummy_ready_queue); ready_queue_destroy(&sched->graph_ready_queue); ready_queue_destroy(&sched->graph_prepare_queue); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->early_dispatch_queues[i]); } ready_queue_destroy(&sched->early_sync_start_queue); diff --git a/src/a2a3/runtime/host_build_graph/runtime/submit_types.h b/src/a2a3/runtime/host_build_graph/runtime/submit_types.h index aa58f02aad..abcebb86f1 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/submit_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/submit_types.h @@ -105,7 +105,7 @@ struct DispatchPredicate { * with an empty core_mask route to a dedicated DUMMY ready queue and are * completed inline by the scheduler dispatch loop, bypassing core allocation. */ -enum class PTO2ResourceShape : uint8_t { +enum class ResourceShape : uint8_t { AIC = 0, // Single AIC AIV = 1, // Single AIV MIX = 2, // Full cluster (dispatch uses active_mask) @@ -115,7 +115,7 @@ enum class PTO2ResourceShape : uint8_t { // Number of *dispatchable* resource shapes (AIC, AIV, MIX). DUMMY does not // allocate a per-shape ready_queue entry / local buffer — it lives in a // dedicated queue inside PTO2SchedulerState. -inline constexpr int32_t PTO2_NUM_RESOURCE_SHAPES = 3; +inline constexpr int32_t NUM_RESOURCE_SHAPES = 3; /** * Bitmask of active subtask slots (AIC/AIV0/AIV1), sizeof == 1. @@ -136,13 +136,13 @@ class ActiveMask { uint8_t core_mask() const { return raw_ & 0x07u; } - PTO2ResourceShape to_shape() const { + ResourceShape to_shape() const { uint8_t cmask = core_mask(); - if (cmask == 0) return PTO2ResourceShape::DUMMY; + if (cmask == 0) return ResourceShape::DUMMY; int bit_count = __builtin_popcount(cmask); - if (bit_count >= 2) return PTO2ResourceShape::MIX; - if (cmask & PTO2_SUBTASK_MASK_AIC) return PTO2ResourceShape::AIC; - return PTO2ResourceShape::AIV; + if (bit_count >= 2) return ResourceShape::MIX; + if (cmask & PTO2_SUBTASK_MASK_AIC) return ResourceShape::AIC; + return ResourceShape::AIV; } bool operator==(ActiveMask other) const { return raw_ == other.raw_; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index c258687d09..1ed0687425 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -566,7 +566,7 @@ After these phases, the scheduler updates profiling headers and checks for termi Ready queues use a lock-free bounded MPMC (Vyukov) design: -- One `PTO2ReadyQueue` per resource shape — 3 shapes (`PTO2_NUM_RESOURCE_SHAPES`): `MIX` +- One `PTO2ReadyQueue` per resource shape — 3 shapes (`NUM_RESOURCE_SHAPES`): `MIX` (AIC+AIV cluster), `AIC`, `AIV`. Alongside `ready_queues[]` there is a per-shape `ready_sync_queues[]` (sync_start Tier-0) and the speculative `early_dispatch_queues[]` / `early_sync_start_queue` — see §8.6 for the full source × tier model. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 7729787d51..f07b98dbf3 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -1196,8 +1196,8 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k // Deadlock check: block_num >= total available slots of the required type. // For MIX/AIC: limit is total_cluster_count (one AIC per cluster). // For AIV: limit is total_aiv_count. - PTO2ResourceShape shape = active_mask.to_shape(); - int32_t limit = (shape == PTO2ResourceShape::AIV) ? orch->total_aiv_count : orch->total_cluster_count; + ResourceShape shape = active_mask.to_shape(); + int32_t limit = (shape == ResourceShape::AIV) ? orch->total_aiv_count : orch->total_cluster_count; if (limit > 0 && block_num > limit) { report_fatal( SIMPLER_ERROR_REQUIRE_SYNC_START_INVALID, __FUNCTION__, diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp index f49b418abe..23feaf064a 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp @@ -100,7 +100,7 @@ void PTO2SchedulerState::print_queues() { const char *shape_names[] = {"AIC", "AIV", "MIX"}; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { LOG_DEBUG(" %s: count=%" PRIu64, shape_names[i], sched->ready_queues[i].size()); } LOG_DEBUG(" DUMMY: count=%" PRIu64, sched->dummy_ready_queue.size()); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index e3dfd5e96c..e6a508b66f 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -424,10 +424,10 @@ struct CompletionStats { * capacities used at layout time (init_data_from_layout reuses them). */ struct PTO2SchedulerLayout { - size_t off_ready_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; - size_t off_ready_sync_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; + size_t off_ready_queue_slots[NUM_RESOURCE_SHAPES]; + size_t off_ready_sync_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_dummy_ready_queue_slots; - size_t off_early_dispatch_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; + size_t off_early_dispatch_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_early_sync_start_queue_slots; size_t off_dep_pool_entries[CHIP_MAX_RING_DEPTH]; uint64_t ready_queue_capacity; @@ -512,14 +512,14 @@ struct PTO2SchedulerState { alignas(64) std::atomic advance_pending_mask; // Ready queues remain global (scheduling is ring-agnostic) - PTO2ReadyQueue ready_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; // Ready sync_start queues, one per shape. A ready sync_start cohort parks here // instead of ready_queues[] so the dispatch loop can drain it as a strict Tier-0 // (sync_start > MIX > C/V) before any regular ready task takes a core, while // reusing the same per-shape dispatch_shape machinery (fits-local inline vs // stop-the-world drain, per-core MIX placement, head-start spacing). - PTO2ReadyQueue ready_sync_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; // Dependency-only tasks (active_mask is empty, shape == DUMMY). Drained by // the dispatch loop and completed inline -- never goes to AICore. @@ -542,8 +542,8 @@ struct PTO2SchedulerState { // the per-shape ready_sync_queues[] (drained as Tier-0); everything else to // ready_queues[]. void push_ready_routed(ChipTaskSlotState *slot_state) { - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); - if (shape == PTO2ResourceShape::DUMMY || + ResourceShape shape = slot_state->active_mask.to_shape(); + if (shape == ResourceShape::DUMMY || (slot_state->task_attrs.has_predicate() && !slot_state->payload->predicate.pass())) { dummy_ready_queue.push(slot_state); } else if (slot_state->task_attrs.requires_sync_start()) { @@ -756,7 +756,7 @@ struct PTO2SchedulerState { // staged and sends the rest to ready_queues. An initial push that overflows also // rolls STAGING back to NONE, since nothing is staged yet and a task with no // claim should not read as speculatively claimed. - PTO2ReadyQueue early_dispatch_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; // sync_start early-dispatch candidates park here instead of early_dispatch_queues[]: // they need an atomic all-or-nothing owner stage, not early_dispatch_shape's @@ -924,8 +924,8 @@ struct PTO2SchedulerState { // Predicated tasks resolve at the ready point, never early-dispatch. The // wiring caller has no separate predicate filter, so the gate lives here. if (consumer.task_attrs.has_predicate()) return; - PTO2ResourceShape shape = consumer.active_mask.to_shape(); - if (shape != PTO2ResourceShape::AIC && shape != PTO2ResourceShape::AIV && shape != PTO2ResourceShape::MIX) { + ResourceShape shape = consumer.active_mask.to_shape(); + if (shape != ResourceShape::AIC && shape != ResourceShape::AIV && shape != ResourceShape::MIX) { return; } @@ -1098,8 +1098,8 @@ struct PTO2SchedulerState { return true; } - PTO2ResourceShape shape = slot_state.active_mask.to_shape(); - if (shape == PTO2ResourceShape::DUMMY || + ResourceShape shape = slot_state.active_mask.to_shape(); + if (shape == ResourceShape::DUMMY || (slot_state.task_attrs.has_predicate() && !slot_state.payload->predicate.pass())) { dummy_ready_queue.push(&slot_state); } else if (slot_state.task_attrs.requires_sync_start()) { @@ -1139,8 +1139,8 @@ struct PTO2SchedulerState { return true; } - PTO2ResourceShape shape = slot_state.active_mask.to_shape(); - if (shape == PTO2ResourceShape::DUMMY || + ResourceShape shape = slot_state.active_mask.to_shape(); + if (shape == ResourceShape::DUMMY || (slot_state.task_attrs.has_predicate() && !slot_state.payload->predicate.pass())) { dummy_ready_queue.push(&slot_state, atomic_count, push_wait); } else if (slot_state.task_attrs.requires_sync_start()) { @@ -1179,13 +1179,13 @@ struct PTO2SchedulerState { } #endif - int get_ready_tasks_batch(PTO2ReadyQueue *queues, PTO2ResourceShape shape, ChipTaskSlotState **out, int max_count) { + int get_ready_tasks_batch(PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { return queues[static_cast(shape)].pop_batch(out, max_count); } #if SIMPLER_SCHED_PROFILING int get_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, + PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, uint64_t &wait_cycle ) { return queues[static_cast(shape)].pop_batch(out, max_count, atomic_count, wait_cycle); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index e84686cba1..cd6856bc29 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -525,7 +525,7 @@ bool SchedulerContext::enter_drain_mode(ChipTaskSlotState *slot_state, int32_t b // include_pending adds each thread's pending-capable cores/clusters — used by the // gated (early) sync_start drain, which pre-stages onto idle running slots AND onto // busy cores' pending slots. The ready drain (include_pending=false) counts idle only. -int32_t SchedulerContext::count_global_available(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending) { +int32_t SchedulerContext::count_global_available(ResourceShape shape, uint8_t core_mask, bool include_pending) { int32_t total = 0; for (int32_t t = 0; t < active_sched_threads_; t++) { total += core_trackers_[t].count_available_blocks(shape, core_mask, include_pending); @@ -553,9 +553,9 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( [[maybe_unused]] bool record_drain_phases ) { CoreTracker &tracker = core_trackers_[thread_idx]; - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); + ResourceShape shape = slot_state->active_mask.to_shape(); uint8_t core_mask = slot_state->active_mask.core_mask(); - bool mix_split = gated && shape == PTO2ResourceShape::MIX; + bool mix_split = gated && shape == ResourceShape::MIX; SyncStartStageResult result; // Stage from this thread's `valid` cores/clusters: CAS-claim a block-index range sized to @@ -579,12 +579,12 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( int32_t claimed[CoreTracker::MAX_CLUSTERS * 3]; for (int32_t b = 0; b < claim; b++) claimed[b] = valid.pop_first(); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); if (claim > 0) prefetch_block_dst(thread_idx, claimed[0], is_mix); for (int32_t b = 0; b < claim; b++) { if (b + 1 < claim) prefetch_block_dst(thread_idx, claimed[b + 1], is_mix); auto core_offset = claimed[b]; - if (shape == PTO2ResourceShape::MIX) { + if (shape == ResourceShape::MIX) { result.running_cores += tracker.mix_cluster_idle_core_count(core_offset, core_mask); } handle_count += prepare_block_for_dispatch( @@ -642,7 +642,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( sched_->record_published_blocks(*slot_state, claim); // AIC/AIV running placement (whole block on idle cores); MIX running cores are // counted per-cluster above (mix_cluster_idle_core_count). - if (gated && shape != PTO2ResourceShape::MIX && !to_pending) result.running_cores += handle_count; + if (gated && shape != ResourceShape::MIX && !to_pending) result.running_cores += handle_count; } }; @@ -651,8 +651,8 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( // used cores take running slots (prepare_block_for_dispatch: to_pending && !is_core_idle). stage(tracker.get_mix_split_cluster_offset_states(core_mask), /*to_pending=*/true); } else { - auto idle = (shape == PTO2ResourceShape::MIX) ? tracker.get_mix_running_cluster_offset_states(core_mask) : - tracker.get_idle_core_offset_states(shape); + auto idle = (shape == ResourceShape::MIX) ? tracker.get_mix_running_cluster_offset_states(core_mask) : + tracker.get_idle_core_offset_states(shape); stage(idle, /*to_pending=*/false); // idle -> running (ready launch + gated pre-stage) if (gated) { stage(tracker.get_pending_core_offset_states(shape), /*to_pending=*/true); @@ -734,7 +734,7 @@ void SchedulerContext::handle_drain_mode( bool gated = slot_state->payload != nullptr && PTO2SchedulerState::owns_early_sync_drain(*slot_state->payload); if (coordinator) { - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); + ResourceShape shape = slot_state->active_mask.to_shape(); // A gated drain may pre-stage onto pending slots too (idle+pending); the ready drain // needs block_num idle cores/clusters. int32_t available = diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 4e4168787a..9d4f0ab8f3 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -231,7 +231,7 @@ class SchedulerContext { // Dispatch (scheduler_dispatch.cpp) // ========================================================================= - static const char *shape_name(PTO2ResourceShape shape); + static const char *shape_name(ResourceShape shape); // Lower-case rendering of PTO2SubtaskSlot, used by dispatch and stall logs. // Kept lower-case to match the `kernels=[aic:N aiv0:N aiv1:N]` field @@ -249,7 +249,7 @@ class SchedulerContext { } int pop_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ); void build_payload( @@ -330,12 +330,12 @@ class SchedulerContext { // Fan out one block's subtasks (1 for AIC/AIV, 1-3 for MIX) into the // caller-supplied handles buffer. Returns the number of handles written. int prepare_block_for_dispatch( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2ResourceShape shape, - bool to_pending, int32_t block_idx, PublishHandle *out_handles, bool force_gate = false + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, ResourceShape shape, bool to_pending, + int32_t block_idx, PublishHandle *out_handles, bool force_gate = false ); void dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ); @@ -357,7 +357,7 @@ class SchedulerContext { // concurrently with peers (mirrors the normal SPMD dispatch path). Returns the // number of blocks staged. int32_t stage_consumer_blocks( - int32_t thread_idx, ChipTaskSlotState *c, PTO2ResourceShape shape, int32_t start, int32_t count, + int32_t thread_idx, ChipTaskSlotState *c, ResourceShape shape, int32_t start, int32_t count, CoreTracker::BitStates &idle, CoreTracker::BitStates &pend ); @@ -366,7 +366,7 @@ class SchedulerContext { // given phase (IDLE -> onto idle cores in the RUNNING slot; PENDING -> onto a // running core's gated pending slot). Pop is sized to the shape's capacity exactly // as dispatch_shape sizes normal dispatch. Returns the number of blocks staged. - int32_t early_dispatch_shape(int32_t thread_idx, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase); + int32_t early_dispatch_shape(int32_t thread_idx, ResourceShape shape, CoreTracker::DispatchPhase phase); // One pass of "Phase 4" in the resolve_and_dispatch loop: IDLE-stage dispatch // for MIX then (if no mix residual) AIC/AIV; then PENDING-stage dispatch with @@ -398,22 +398,20 @@ class SchedulerContext { // matching `shape`. Used as a scheduling hint on the PENDING dispatch path // — see the implementation in scheduler_dispatch.cpp for the hint-semantics // rationale and the safety argument against the drain worker. - bool has_idle_in_other_threads(int32_t self_thread_idx, PTO2ResourceShape shape) const; + bool has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const; // True if mix tasks remain in the global MIX ready queue. Approximate — // PTO2ReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. - bool has_residual_mix() const { - return sched_->ready_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; - } + bool has_residual_mix() const { return sched_->ready_queues[static_cast(ResourceShape::MIX)].size() > 0; } // Tier-0 analog of has_residual_mix for the ready sync_start lane: true if MIX // sync_start cohorts remain queued, so the Tier-0 pass keeps MIX strict priority // over its own AIC/AIV sync work. Same relaxed-size snapshot caveat. bool has_residual_sync_mix() const { - return sched_->ready_sync_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; + return sched_->ready_sync_queues[static_cast(ResourceShape::MIX)].size() > 0; } // Early-dispatch analog of has_residual_mix: true if MIX early-dispatch candidates @@ -422,7 +420,7 @@ class SchedulerContext { // empty), so early-dispatch MIX priority needs its own residual check against // early_dispatch_queues[MIX]. Same relaxed-size snapshot caveat as has_residual_mix. bool has_residual_early_mix() const { - return sched_->early_dispatch_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; + return sched_->early_dispatch_queues[static_cast(ResourceShape::MIX)].size() > 0; } // ========================================================================= @@ -452,7 +450,7 @@ class SchedulerContext { ); bool enter_drain_mode(ChipTaskSlotState *slot_state, int32_t block_num); - int32_t count_global_available(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending = false); + int32_t count_global_available(ResourceShape shape, uint8_t core_mask, bool include_pending = false); struct SyncStartStageResult { int32_t staged_blocks{0}; int32_t running_cores{0}; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 574dc4399c..3ab4d3ad57 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -61,21 +61,21 @@ static_assert( "staged_core_mask too small for RUNTIME_MAX_WORKER cores" ); -const char *SchedulerContext::shape_name(PTO2ResourceShape shape) { +const char *SchedulerContext::shape_name(ResourceShape shape) { switch (shape) { - case PTO2ResourceShape::AIC: + case ResourceShape::AIC: return "AIC"; - case PTO2ResourceShape::AIV: + case ResourceShape::AIV: return "AIV"; - case PTO2ResourceShape::MIX: + case ResourceShape::MIX: return "MIX"; - case PTO2ResourceShape::DUMMY: + case ResourceShape::DUMMY: return "DUMMY"; } return "UNKNOWN"; } -bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, PTO2ResourceShape shape) const { +bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const { // Cross-thread read of peer trackers without explicit synchronization. The // backing `core_states_` is a naturally aligned uint64_t; aarch64 guarantees // single-copy atomicity for an 8-byte aligned load, so no torn read. The @@ -94,7 +94,7 @@ bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, PTO2Re } int SchedulerContext::pop_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ) { #if SIMPLER_DFX auto &chip_swimlane = sched_chip_swimlane_[thread_idx]; @@ -238,7 +238,7 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( } int SchedulerContext::prepare_block_for_dispatch( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2ResourceShape shape, bool to_pending, + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, ResourceShape shape, bool to_pending, int32_t block_idx, PublishHandle *out_handles, bool force_gate ) { #if SIMPLER_DFX @@ -255,7 +255,7 @@ int SchedulerContext::prepare_block_for_dispatch( } #endif CoreTracker &tracker = core_trackers_[thread_idx]; - if (shape == PTO2ResourceShape::MIX) { + if (shape == ResourceShape::MIX) { uint8_t cmask = slot_state.active_mask.core_mask(); int n = 0; // Per-core slot placement (#1308): an idle used core takes its running slot @@ -288,7 +288,7 @@ int SchedulerContext::prepare_block_for_dispatch( sched_chip_swimlane_[thread_idx].phase_dispatch_count += __builtin_popcount(cmask); #endif return n; - } else if (shape == PTO2ResourceShape::AIC) { + } else if (shape == ResourceShape::AIC) { out_handles[0] = prepare_subtask_to_core( thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIC, to_pending, block_idx, force_gate ); @@ -308,7 +308,7 @@ int SchedulerContext::prepare_block_for_dispatch( } void SchedulerContext::dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ) { #if SIMPLER_SCHED_PROFILING @@ -317,7 +317,7 @@ void SchedulerContext::dispatch_shape( if (entered_drain) return; bool is_pending = (phase == CoreTracker::DispatchPhase::PENDING); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); auto cores = is_mix ? tracker.get_cluster_offset_states() : tracker.get_dispatchable_cores(shape, phase); if (!cores.has_value()) return; @@ -509,14 +509,14 @@ void SchedulerContext::run_staging_order( // MIX is handled explicitly at the top of each stage; only AIC/AIV cycle // through this 2-elem array, with order toggled by thread parity for // shape-level load balancing across threads. - static constexpr PTO2ResourceShape kAicAivOrder[2][2] = { - {PTO2ResourceShape::AIC, PTO2ResourceShape::AIV}, - {PTO2ResourceShape::AIV, PTO2ResourceShape::AIC}, + static constexpr ResourceShape kAicAivOrder[2][2] = { + {ResourceShape::AIC, ResourceShape::AIV}, + {ResourceShape::AIV, ResourceShape::AIC}, }; - const PTO2ResourceShape *aic_aiv = kAicAivOrder[thread_idx & 1]; + const ResourceShape *aic_aiv = kAicAivOrder[thread_idx & 1]; // ===== IDLE stage ===== - if (stage(PTO2ResourceShape::MIX, Phase::IDLE)) return; + if (stage(ResourceShape::MIX, Phase::IDLE)) return; // MIX-IDLE residual: AIC/AIV (both IDLE and PENDING) yield for this pass. // MIX-PENDING below still runs — that is the core of "mix strict priority": @@ -540,8 +540,8 @@ void SchedulerContext::run_staging_order( // // The gate is NOT subject to skip_aic_aiv — residual mix continues to drain // via pending slots on this thread when no peer is idle. - if (!has_idle_in_other_threads(thread_idx, PTO2ResourceShape::MIX)) { - if (stage(PTO2ResourceShape::MIX, Phase::PENDING)) return; + if (!has_idle_in_other_threads(thread_idx, ResourceShape::MIX)) { + if (stage(ResourceShape::MIX, Phase::PENDING)) return; } // Re-check after MIX-PENDING. If MIX-IDLE already set skip_aic_aiv, leave @@ -555,7 +555,7 @@ void SchedulerContext::run_staging_order( // AIC/AIV-PENDING gate: a peer-idle skip is a delay, not a loss — the peer // will pull from the global queue on its next IDLE pass. for (int i = 0; i < 2; i++) { - PTO2ResourceShape s = aic_aiv[i]; + ResourceShape s = aic_aiv[i]; if (has_idle_in_other_threads(thread_idx, s)) continue; if (stage(s, Phase::PENDING)) return; } @@ -574,7 +574,7 @@ void SchedulerContext::dispatch_ready_tasks( // (entered_drain), which also short-circuits the regular tier below. run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { dispatch_shape( thread_idx, sched_->ready_sync_queues, shape, phase, tracker, entered_drain, made_progress, try_pushed ); @@ -589,7 +589,7 @@ void SchedulerContext::dispatch_ready_tasks( // Tier 1: regular ready work. run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { dispatch_shape( thread_idx, sched_->ready_queues, shape, phase, tracker, entered_drain, made_progress, try_pushed ); @@ -621,7 +621,7 @@ void SchedulerContext::dispatch_ready_tasks( // those release did not take and rings them from its immutable local handles. // The seq_cst order guarantees every gated core has exactly one writer. int32_t SchedulerContext::stage_consumer_blocks( - int32_t thread_idx, ChipTaskSlotState *c, PTO2ResourceShape shape, int32_t start, int32_t count, + int32_t thread_idx, ChipTaskSlotState *c, ResourceShape shape, int32_t start, int32_t count, CoreTracker::BitStates &idle, CoreTracker::BitStates &pend ) { CoreTracker &tracker = core_trackers_[thread_idx]; @@ -707,10 +707,10 @@ int32_t SchedulerContext::stage_consumer_blocks( // shape's queue when dispatch_fanin becomes complete, so the shape is the queue // index (no per-consumer to_shape()). Returns the number of blocks staged. int32_t -SchedulerContext::early_dispatch_shape(int32_t thread_idx, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { +SchedulerContext::early_dispatch_shape(int32_t thread_idx, ResourceShape shape, CoreTracker::DispatchPhase phase) { CoreTracker &tracker = core_trackers_[thread_idx]; int32_t s = static_cast(shape); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); bool is_idle = (phase == CoreTracker::DispatchPhase::IDLE); // Size the pop exactly as dispatch_shape does: MIX to the cluster count, else the @@ -815,7 +815,7 @@ int32_t SchedulerContext::try_early_dispatch( // regular ready_queues) strictly precedes early — there is no real ready task // to delay only when every normal queue is drained. if (pmu_active || !tracker.has_any_free_slot()) return 0; - for (int s = 0; s < PTO2_NUM_RESOURCE_SHAPES; s++) { + for (int s = 0; s < NUM_RESOURCE_SHAPES; s++) { if (sched_->ready_sync_queues[s].size() > 0 || sched_->ready_queues[s].size() > 0) return 0; } @@ -867,7 +867,7 @@ int32_t SchedulerContext::try_early_dispatch( // enters drain, so the stage callback always reports "no stop". run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { total_staged += early_dispatch_shape(thread_idx, shape, phase); return false; }, @@ -940,7 +940,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // snapshot so the next phase's "at_start" equals the previous phase's // "at_end". // - // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches PTO2_NUM_RESOURCE_SHAPES: AIC/AIV/MIX. + // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches NUM_RESOURCE_SHAPES: AIC/AIV/MIX. // // **Hot-path cost discipline.** Shared depth (PTO2ReadyQueue::size) is two // atomic relaxed loads against cache lines that all peer sched threads also @@ -951,7 +951,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // complete-emit and dispatch-emit in the same iter both reuse the same // shared sample. static_assert( - CHIP_SWIMLANE_NUM_QUEUE_SHAPES == PTO2_NUM_RESOURCE_SHAPES, + CHIP_SWIMLANE_NUM_QUEUE_SHAPES == NUM_RESOURCE_SHAPES, "queue snapshot width must match runtime resource shape count" ); int16_t phase_start_shared[CHIP_SWIMLANE_NUM_QUEUE_SHAPES] = {0}; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h index d1b8e59f29..e83f92cbe7 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h @@ -264,15 +264,15 @@ class alignas(64) CoreTracker { // --- Cluster matching --- - BitStates get_valid_cluster_offset_states(PTO2ResourceShape shape) const { + BitStates get_valid_cluster_offset_states(ResourceShape shape) const { switch (shape) { - case PTO2ResourceShape::AIC: + case ResourceShape::AIC: return core_states_ & aic_mask_; - case PTO2ResourceShape::AIV: + case ResourceShape::AIV: return ((core_states_ >> 1) | (core_states_ >> 2)) & aic_mask_; - case PTO2ResourceShape::MIX: + case ResourceShape::MIX: return (core_states_ >> 1) & (core_states_ >> 2) & core_states_ & aic_mask_; - case PTO2ResourceShape::DUMMY: + case ResourceShape::DUMMY: // DUMMY tasks never reach the core-tracker dispatch path; they are // completed inline by resolve_and_dispatch via dummy_ready_queue. return {}; @@ -321,11 +321,11 @@ class alignas(64) CoreTracker { // always have pending_occupied=0, so AIV/MIX need no extra filtering. // Skipping the AIC-centric filter also fixes a latent bug where a running+pending AIC core // would incorrectly block AIV idle dispatch on the same cluster. - BitStates get_idle_core_offset_states(PTO2ResourceShape shape) const { - if (shape == PTO2ResourceShape::AIC) { + BitStates get_idle_core_offset_states(ResourceShape shape) const { + if (shape == ResourceShape::AIC) { return get_valid_cluster_offset_states(shape) & ~(pending_occupied_ & aic_mask_); } - if (shape == PTO2ResourceShape::AIV) { + if (shape == ResourceShape::AIV) { return core_states_ & aiv_mask_; } return get_valid_cluster_offset_states(shape); // MIX: cluster-level @@ -429,8 +429,8 @@ class alignas(64) CoreTracker { return get_mix_split_cluster_offset_states(core_mask).count(); } - BitStates get_pending_core_offset_states(PTO2ResourceShape shape) const { - if (shape == PTO2ResourceShape::MIX) { + BitStates get_pending_core_offset_states(ResourceShape shape) const { + if (shape == ResourceShape::MIX) { // Shape-level query kept conservative for legacy callers/tests. // The real MIX dispatch path applies active_mask in classify_mix_cluster(). // Any core without a pending payload can accept a dispatch (idle or running). @@ -444,7 +444,7 @@ class alignas(64) CoreTracker { (running & aic_mask_) & ((running >> 1) & aic_mask_) & ((running >> 2) & aic_mask_); return mix_available & cluster_all_running; } - if (shape == PTO2ResourceShape::AIC) { + if (shape == ResourceShape::AIC) { return (~core_states_) & aic_mask_ & ~(pending_occupied_ & aic_mask_); } // AIV @@ -455,11 +455,11 @@ class alignas(64) CoreTracker { // one core per block; MIX uses one active-mask-compatible cluster per block. // Gated early staging may use both idle running slots and free pending slots, // while a ready cohort is restricted to idle running slots. - int32_t count_available_blocks(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending) const { - if (shape == PTO2ResourceShape::MIX) { + int32_t count_available_blocks(ResourceShape shape, uint8_t core_mask, bool include_pending) const { + if (shape == ResourceShape::MIX) { return include_pending ? count_mix_split_clusters(core_mask) : count_mix_running_clusters(core_mask); } - if (shape != PTO2ResourceShape::AIC && shape != PTO2ResourceShape::AIV) return 0; + if (shape != ResourceShape::AIC && shape != ResourceShape::AIV) return 0; int32_t available = get_idle_core_offset_states(shape).count(); if (include_pending) available += get_pending_core_offset_states(shape).count(); @@ -470,7 +470,7 @@ class alignas(64) CoreTracker { enum class DispatchPhase : uint8_t { IDLE, PENDING }; - BitStates get_dispatchable_cores(PTO2ResourceShape shape, DispatchPhase phase) const { + BitStates get_dispatchable_cores(ResourceShape shape, DispatchPhase phase) const { return (phase == DispatchPhase::IDLE) ? get_idle_core_offset_states(shape) : get_pending_core_offset_states(shape); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index c0b2b0c7fd..5bf019b2f5 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -135,14 +135,14 @@ PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_ca layout.dep_pool_capacities[r] = dep_pool_capacities[r]; } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); } layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); } layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); @@ -172,14 +172,14 @@ bool PTO2SchedulerState::init_data_from_layout( } } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { if (!ready_queue_init_data_from_layout( &sched->ready_queues[i], arena, layout.off_ready_queue_slots[i], layout.ready_queue_capacity )) { return false; } } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { if (!ready_queue_init_data_from_layout( &sched->ready_sync_queues[i], arena, layout.off_ready_sync_queue_slots[i], layout.ready_queue_capacity )) { @@ -191,7 +191,7 @@ bool PTO2SchedulerState::init_data_from_layout( )) { return false; } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { if (!ready_queue_init_data_from_layout( &sched->early_dispatch_queues[i], arena, layout.off_early_dispatch_queue_slots[i], PTO2_EARLY_DISPATCH_QUEUE_SIZE @@ -230,14 +230,14 @@ void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void sched->ring_sched_states[r].reset_for_reuse(sm_dev_base, r, orch_err); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_queues[i].reset_for_reuse(); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_sync_queues[i].reset_for_reuse(); } sched->dummy_ready_queue.reset_for_reuse(); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->early_dispatch_queues[i].reset_for_reuse(); } sched->early_sync_start_queue.reset_for_reuse(); @@ -248,14 +248,14 @@ void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena) { PTO2SchedulerState *sched = this; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_queues[i], arena, layout.off_ready_queue_slots[i]); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_sync_queues[i], arena, layout.off_ready_sync_queue_slots[i]); } ready_queue_wire_arena_pointers(&sched->dummy_ready_queue, arena, layout.off_dummy_ready_queue_slots); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers( &sched->early_dispatch_queues[i], arena, layout.off_early_dispatch_queue_slots[i] ); @@ -273,14 +273,14 @@ void PTO2SchedulerState::destroy() { sched->ring_sched_states[r].destroy(); sched->ring_sched_states[r].dep_pool.base = nullptr; } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_queues[i]); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_sync_queues[i]); } ready_queue_destroy(&sched->dummy_ready_queue); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->early_dispatch_queues[i]); } ready_queue_destroy(&sched->early_sync_start_queue); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h index adc88770ca..fa2645c45e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h @@ -105,7 +105,7 @@ struct DispatchPredicate { * with an empty core_mask route to a dedicated DUMMY ready queue and are * completed inline by the scheduler dispatch loop, bypassing core allocation. */ -enum class PTO2ResourceShape : uint8_t { +enum class ResourceShape : uint8_t { AIC = 0, // Single AIC AIV = 1, // Single AIV MIX = 2, // Full cluster (dispatch uses active_mask) @@ -115,7 +115,7 @@ enum class PTO2ResourceShape : uint8_t { // Number of *dispatchable* resource shapes (AIC, AIV, MIX). DUMMY does not // allocate a per-shape ready_queue entry / local buffer — it lives in a // dedicated queue inside PTO2SchedulerState. -inline constexpr int32_t PTO2_NUM_RESOURCE_SHAPES = 3; +inline constexpr int32_t NUM_RESOURCE_SHAPES = 3; /** * Bitmask of active subtask slots (AIC/AIV0/AIV1), sizeof == 1. @@ -136,13 +136,13 @@ class ActiveMask { uint8_t core_mask() const { return raw_ & 0x07u; } - PTO2ResourceShape to_shape() const { + ResourceShape to_shape() const { uint8_t cmask = core_mask(); - if (cmask == 0) return PTO2ResourceShape::DUMMY; + if (cmask == 0) return ResourceShape::DUMMY; int bit_count = __builtin_popcount(cmask); - if (bit_count >= 2) return PTO2ResourceShape::MIX; - if (cmask & PTO2_SUBTASK_MASK_AIC) return PTO2ResourceShape::AIC; - return PTO2ResourceShape::AIV; + if (bit_count >= 2) return ResourceShape::MIX; + if (cmask & PTO2_SUBTASK_MASK_AIC) return ResourceShape::AIC; + return ResourceShape::AIV; } bool operator==(ActiveMask other) const { return raw_ == other.raw_; } diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 246215e379..9e72bb384b 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -2575,8 +2575,8 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k // Deadlock check: block_num >= total available slots of the required type. // For MIX/AIC: limit is total_cluster_count (one AIC per cluster). // For AIV: limit is total_aiv_count. - PTO2ResourceShape shape = active_mask.to_shape(); - int32_t limit = (shape == PTO2ResourceShape::AIV) ? orch->total_aiv_count : orch->total_cluster_count; + ResourceShape shape = active_mask.to_shape(); + int32_t limit = (shape == ResourceShape::AIV) ? orch->total_aiv_count : orch->total_cluster_count; if (limit > 0 && block_num > limit) { report_fatal( SIMPLER_ERROR_REQUIRE_SYNC_START_INVALID, __FUNCTION__, diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.cpp index 1a23c66fe3..6458fcb747 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.cpp @@ -77,7 +77,7 @@ void PTO2SchedulerState::print_stats() { void PTO2SchedulerState::print_queues() { PTO2SchedulerState *sched = this; const char *shape_names[] = {"AIC", "AIV", "MIX"}; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { LOG_TIMING( "QPROBE rq[%s] pushes=%llu maxocc=%llu cap=%llu", shape_names[i], (unsigned long long)sched->ready_queues[i].enqueue_pos.load(std::memory_order_relaxed), diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h index f3fb6cc7ba..720c0c63bf 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -453,12 +453,12 @@ struct CompletionStats { * capacities used at layout time (init_from_layout reuses them). */ struct PTO2SchedulerLayout { - size_t off_ready_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; - size_t off_ready_sync_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; + size_t off_ready_queue_slots[NUM_RESOURCE_SHAPES]; + size_t off_ready_sync_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_dummy_ready_queue_slots; size_t off_graph_ready_queue_slots; size_t off_graph_prepare_queue_slots; - size_t off_early_dispatch_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; + size_t off_early_dispatch_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_early_sync_start_queue_slots; uint64_t ready_queue_capacity; }; @@ -489,14 +489,14 @@ struct PTO2SchedulerState { } ring_sched_state; // Ready queues remain global (scheduling is ring-agnostic) - PTO2ReadyQueue ready_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; // Ready sync_start queues, one per shape. A ready sync_start cohort parks here // instead of ready_queues[] so the dispatch loop can drain it as a strict Tier-0 // (sync_start > MIX > C/V) before any regular ready task takes a core, while // reusing the same per-shape dispatch_shape machinery (fits-local inline vs // stop-the-world drain, per-core MIX placement, head-start spacing). - PTO2ReadyQueue ready_sync_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; // Dependency-only tasks (active_mask is empty, shape == DUMMY). Drained by // the dispatch loop and completed inline -- never goes to AICore. @@ -548,8 +548,8 @@ struct PTO2SchedulerState { if (slot_state->task_kind == TaskKind::GRAPH) { pushed = graph_ready_queue.push(slot_state); } else { - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); - if (shape == PTO2ResourceShape::DUMMY || + ResourceShape shape = slot_state->active_mask.to_shape(); + if (shape == ResourceShape::DUMMY || (slot_state->task_attrs.has_predicate() && !slot_state->payload->predicate.pass())) { pushed = dummy_ready_queue.push(slot_state); } else if (slot_state->task_attrs.requires_sync_start()) { @@ -707,7 +707,7 @@ struct PTO2SchedulerState { // the ready queue (scheduler_dispatch: pop -> claim -> re-push). A stale/released // entry fails the STAGING check on pop and is dropped; a push that overflows is // logged and the consumer's blocks fall back to normal dispatch. - PTO2ReadyQueue early_dispatch_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; // sync_start early-dispatch candidates park here instead of early_dispatch_queues[]: // they need an atomic all-or-nothing stage, not early_dispatch_shape's @@ -879,13 +879,13 @@ struct PTO2SchedulerState { // ready_sync_queues (unaffected). void propagate_dispatch_fanin(ChipTaskSlotState & /*p*/) {} - int get_ready_tasks_batch(PTO2ReadyQueue *queues, PTO2ResourceShape shape, ChipTaskSlotState **out, int max_count) { + int get_ready_tasks_batch(PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { return queues[static_cast(shape)].pop_batch(out, max_count); } #if SIMPLER_SCHED_PROFILING int get_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, + PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, uint64_t &wait_cycle ) { return queues[static_cast(shape)].pop_batch(out, max_count, atomic_count, wait_cycle); diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index 88ecc641df..e94b50d8af 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -469,7 +469,7 @@ bool SchedulerContext::enter_drain_mode(ChipTaskSlotState *slot_state, int32_t b // include_pending adds each thread's pending-capable cores/clusters — used by the // gated (early) sync_start drain, which pre-stages onto idle running slots AND onto // busy cores' pending slots. The ready drain (include_pending=false) counts idle only. -int32_t SchedulerContext::count_global_available(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending) { +int32_t SchedulerContext::count_global_available(ResourceShape shape, uint8_t core_mask, bool include_pending) { int32_t total = 0; for (int32_t t = 0; t < active_sched_threads_; t++) { total += core_trackers_[t].count_available_blocks(shape, core_mask, include_pending); @@ -497,9 +497,9 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( [[maybe_unused]] bool record_drain_phases ) { CoreTracker &tracker = core_trackers_[thread_idx]; - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); + ResourceShape shape = slot_state->active_mask.to_shape(); uint8_t core_mask = slot_state->active_mask.core_mask(); - bool mix_split = gated && shape == PTO2ResourceShape::MIX; + bool mix_split = gated && shape == ResourceShape::MIX; SyncStartStageResult result; // Stage from this thread's `valid` cores/clusters: CAS-claim a block-index range sized to @@ -523,12 +523,12 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( int32_t claimed[CoreTracker::MAX_CLUSTERS * 3]; for (int32_t b = 0; b < claim; b++) claimed[b] = valid.pop_first(); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); if (claim > 0) prefetch_block_dst(thread_idx, claimed[0], is_mix); for (int32_t b = 0; b < claim; b++) { if (b + 1 < claim) prefetch_block_dst(thread_idx, claimed[b + 1], is_mix); auto core_offset = claimed[b]; - if (shape == PTO2ResourceShape::MIX) { + if (shape == ResourceShape::MIX) { result.running_cores += tracker.mix_cluster_idle_core_count(core_offset, core_mask); } handle_count += prepare_block_for_dispatch( @@ -586,7 +586,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( sched_->record_published_blocks(*slot_state, claim); // AIC/AIV running placement (whole block on idle cores); MIX running cores are // counted per-cluster above (mix_cluster_idle_core_count). - if (gated && shape != PTO2ResourceShape::MIX && !to_pending) result.running_cores += handle_count; + if (gated && shape != ResourceShape::MIX && !to_pending) result.running_cores += handle_count; } }; @@ -595,8 +595,8 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( // used cores take running slots (prepare_block_for_dispatch: to_pending && !is_core_idle). stage(tracker.get_mix_split_cluster_offset_states(core_mask), /*to_pending=*/true); } else { - auto idle = (shape == PTO2ResourceShape::MIX) ? tracker.get_mix_running_cluster_offset_states(core_mask) : - tracker.get_idle_core_offset_states(shape); + auto idle = (shape == ResourceShape::MIX) ? tracker.get_mix_running_cluster_offset_states(core_mask) : + tracker.get_idle_core_offset_states(shape); stage(idle, /*to_pending=*/false); // idle -> running (ready launch + gated pre-stage) if (gated) { stage(tracker.get_pending_core_offset_states(shape), /*to_pending=*/true); @@ -685,7 +685,7 @@ void SchedulerContext::handle_drain_mode(int32_t thread_idx, [[maybe_unused]] ui bool gated = slot_state->payload != nullptr && PTO2SchedulerState::owns_early_sync_drain(*slot_state->payload); if (coordinator) { - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); + ResourceShape shape = slot_state->active_mask.to_shape(); // A gated drain may pre-stage onto pending slots too (idle+pending); the ready drain // needs block_num idle cores/clusters. int32_t available = diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index a1d0bd96d6..5cfd7c1c83 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -281,7 +281,7 @@ class SchedulerContext { // Dispatch (scheduler_dispatch.cpp) // ========================================================================= - static const char *shape_name(PTO2ResourceShape shape); + static const char *shape_name(ResourceShape shape); // Lower-case rendering of PTO2SubtaskSlot, used by dispatch and stall logs. // Kept lower-case to match the `kernels=[aic:N aiv0:N aiv1:N]` field @@ -299,7 +299,7 @@ class SchedulerContext { } int pop_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ); void build_payload( @@ -380,12 +380,12 @@ class SchedulerContext { // Fan out one block's subtasks (1 for AIC/AIV, 1-3 for MIX) into the // caller-supplied handles buffer. Returns the number of handles written. int prepare_block_for_dispatch( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2ResourceShape shape, - bool to_pending, int32_t block_idx, PublishHandle *out_handles, bool force_gate = false + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, ResourceShape shape, bool to_pending, + int32_t block_idx, PublishHandle *out_handles, bool force_gate = false ); void dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ); @@ -407,7 +407,7 @@ class SchedulerContext { // concurrently with peers (mirrors the normal SPMD dispatch path). Returns the // number of blocks staged. int32_t stage_consumer_blocks( - int32_t thread_idx, ChipTaskSlotState *c, PTO2ResourceShape shape, int32_t start, int32_t count, + int32_t thread_idx, ChipTaskSlotState *c, ResourceShape shape, int32_t start, int32_t count, CoreTracker::BitStates &idle, CoreTracker::BitStates &pend ); @@ -416,7 +416,7 @@ class SchedulerContext { // given phase (IDLE -> onto idle cores in the RUNNING slot; PENDING -> onto a // running core's gated pending slot). Pop is sized to the shape's capacity exactly // as dispatch_shape sizes normal dispatch. Returns the number of blocks staged. - int32_t early_dispatch_shape(int32_t thread_idx, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase); + int32_t early_dispatch_shape(int32_t thread_idx, ResourceShape shape, CoreTracker::DispatchPhase phase); // One pass of "Phase 4" in the resolve_and_dispatch loop: IDLE-stage dispatch // for MIX then (if no mix residual) AIC/AIV; then PENDING-stage dispatch with @@ -448,22 +448,20 @@ class SchedulerContext { // matching `shape`. Used as a scheduling hint on the PENDING dispatch path // — see the implementation in scheduler_dispatch.cpp for the hint-semantics // rationale and the safety argument against the drain worker. - bool has_idle_in_other_threads(int32_t self_thread_idx, PTO2ResourceShape shape) const; + bool has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const; // True if mix tasks remain in the global MIX ready queue. Approximate — // PTO2ReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. - bool has_residual_mix() const { - return sched_->ready_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; - } + bool has_residual_mix() const { return sched_->ready_queues[static_cast(ResourceShape::MIX)].size() > 0; } // Tier-0 analog of has_residual_mix for the ready sync_start lane: true if MIX // sync_start cohorts remain queued, so the Tier-0 pass keeps MIX strict priority // over its own AIC/AIV sync work. Same relaxed-size snapshot caveat. bool has_residual_sync_mix() const { - return sched_->ready_sync_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; + return sched_->ready_sync_queues[static_cast(ResourceShape::MIX)].size() > 0; } // Early-dispatch analog of has_residual_mix: true if MIX early-dispatch candidates @@ -472,7 +470,7 @@ class SchedulerContext { // empty), so early-dispatch MIX priority needs its own residual check against // early_dispatch_queues[MIX]. Same relaxed-size snapshot caveat as has_residual_mix. bool has_residual_early_mix() const { - return sched_->early_dispatch_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; + return sched_->early_dispatch_queues[static_cast(ResourceShape::MIX)].size() > 0; } // ========================================================================= @@ -501,7 +499,7 @@ class SchedulerContext { ); bool enter_drain_mode(ChipTaskSlotState *slot_state, int32_t block_num); - int32_t count_global_available(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending = false); + int32_t count_global_available(ResourceShape shape, uint8_t core_mask, bool include_pending = false); struct SyncStartStageResult { int32_t staged_blocks{0}; int32_t running_cores{0}; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 3903830a97..632f4b88e4 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -59,21 +59,21 @@ static_assert( "staged_core_mask too small for RUNTIME_MAX_WORKER cores" ); -const char *SchedulerContext::shape_name(PTO2ResourceShape shape) { +const char *SchedulerContext::shape_name(ResourceShape shape) { switch (shape) { - case PTO2ResourceShape::AIC: + case ResourceShape::AIC: return "AIC"; - case PTO2ResourceShape::AIV: + case ResourceShape::AIV: return "AIV"; - case PTO2ResourceShape::MIX: + case ResourceShape::MIX: return "MIX"; - case PTO2ResourceShape::DUMMY: + case ResourceShape::DUMMY: return "DUMMY"; } return "UNKNOWN"; } -bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, PTO2ResourceShape shape) const { +bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const { // Cross-thread read of peer trackers without explicit synchronization. The // backing `core_states_` is a naturally aligned uint64_t; aarch64 guarantees // single-copy atomicity for an 8-byte aligned load, so no torn read. The @@ -92,7 +92,7 @@ bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, PTO2Re } int SchedulerContext::pop_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ) { #if SIMPLER_DFX auto &chip_swimlane = sched_chip_swimlane_[thread_idx]; @@ -242,7 +242,7 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( } int SchedulerContext::prepare_block_for_dispatch( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2ResourceShape shape, bool to_pending, + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, ResourceShape shape, bool to_pending, int32_t block_idx, PublishHandle *out_handles, bool force_gate ) { #if SIMPLER_DFX @@ -260,7 +260,7 @@ int SchedulerContext::prepare_block_for_dispatch( } #endif CoreTracker &tracker = core_trackers_[thread_idx]; - if (shape == PTO2ResourceShape::MIX) { + if (shape == ResourceShape::MIX) { uint8_t cmask = slot_state.active_mask.core_mask(); int n = 0; // Per-core slot placement (#1308): an idle used core takes its running slot @@ -293,7 +293,7 @@ int SchedulerContext::prepare_block_for_dispatch( sched_chip_swimlane_[thread_idx].phase_dispatch_count += __builtin_popcount(cmask); #endif return n; - } else if (shape == PTO2ResourceShape::AIC) { + } else if (shape == ResourceShape::AIC) { out_handles[0] = prepare_subtask_to_core( thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIC, to_pending, block_idx, force_gate ); @@ -313,7 +313,7 @@ int SchedulerContext::prepare_block_for_dispatch( } void SchedulerContext::dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ) { #if SIMPLER_SCHED_PROFILING @@ -322,7 +322,7 @@ void SchedulerContext::dispatch_shape( if (entered_drain) return; bool is_pending = (phase == CoreTracker::DispatchPhase::PENDING); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); auto cores = is_mix ? tracker.get_cluster_offset_states() : tracker.get_dispatchable_cores(shape, phase); if (!cores.has_value()) return; @@ -508,14 +508,14 @@ void SchedulerContext::run_staging_order( // MIX is handled explicitly at the top of each stage; only AIC/AIV cycle // through this 2-elem array, with order toggled by thread parity for // shape-level load balancing across threads. - static constexpr PTO2ResourceShape kAicAivOrder[2][2] = { - {PTO2ResourceShape::AIC, PTO2ResourceShape::AIV}, - {PTO2ResourceShape::AIV, PTO2ResourceShape::AIC}, + static constexpr ResourceShape kAicAivOrder[2][2] = { + {ResourceShape::AIC, ResourceShape::AIV}, + {ResourceShape::AIV, ResourceShape::AIC}, }; - const PTO2ResourceShape *aic_aiv = kAicAivOrder[thread_idx & 1]; + const ResourceShape *aic_aiv = kAicAivOrder[thread_idx & 1]; // ===== IDLE stage ===== - if (stage(PTO2ResourceShape::MIX, Phase::IDLE)) return; + if (stage(ResourceShape::MIX, Phase::IDLE)) return; // MIX-IDLE residual: AIC/AIV (both IDLE and PENDING) yield for this pass. // MIX-PENDING below still runs — that is the core of "mix strict priority": @@ -539,8 +539,8 @@ void SchedulerContext::run_staging_order( // // The gate is NOT subject to skip_aic_aiv — residual mix continues to drain // via pending slots on this thread when no peer is idle. - if (!has_idle_in_other_threads(thread_idx, PTO2ResourceShape::MIX)) { - if (stage(PTO2ResourceShape::MIX, Phase::PENDING)) return; + if (!has_idle_in_other_threads(thread_idx, ResourceShape::MIX)) { + if (stage(ResourceShape::MIX, Phase::PENDING)) return; } // Re-check after MIX-PENDING. If MIX-IDLE already set skip_aic_aiv, leave @@ -554,7 +554,7 @@ void SchedulerContext::run_staging_order( // AIC/AIV-PENDING gate: a peer-idle skip is a delay, not a loss — the peer // will pull from the global queue on its next IDLE pass. for (int i = 0; i < 2; i++) { - PTO2ResourceShape s = aic_aiv[i]; + ResourceShape s = aic_aiv[i]; if (has_idle_in_other_threads(thread_idx, s)) continue; if (stage(s, Phase::PENDING)) return; } @@ -573,7 +573,7 @@ void SchedulerContext::dispatch_ready_tasks( // (entered_drain), which also short-circuits the regular tier below. run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { dispatch_shape( thread_idx, sched_->ready_sync_queues, shape, phase, tracker, entered_drain, made_progress, try_pushed ); @@ -588,7 +588,7 @@ void SchedulerContext::dispatch_ready_tasks( // Tier 1: regular ready work. run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { dispatch_shape( thread_idx, sched_->ready_queues, shape, phase, tracker, entered_drain, made_progress, try_pushed ); @@ -620,7 +620,7 @@ void SchedulerContext::dispatch_ready_tasks( // those release did not take and rings them from its immutable local handles. // The seq_cst order guarantees every gated core has exactly one writer. int32_t SchedulerContext::stage_consumer_blocks( - int32_t thread_idx, ChipTaskSlotState *c, PTO2ResourceShape shape, int32_t start, int32_t count, + int32_t thread_idx, ChipTaskSlotState *c, ResourceShape shape, int32_t start, int32_t count, CoreTracker::BitStates &idle, CoreTracker::BitStates &pend ) { CoreTracker &tracker = core_trackers_[thread_idx]; @@ -706,10 +706,10 @@ int32_t SchedulerContext::stage_consumer_blocks( // propagate_dispatch_fanin, so the shape is the queue index (no per-consumer // to_shape()). Returns the number of blocks staged. int32_t -SchedulerContext::early_dispatch_shape(int32_t thread_idx, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { +SchedulerContext::early_dispatch_shape(int32_t thread_idx, ResourceShape shape, CoreTracker::DispatchPhase phase) { CoreTracker &tracker = core_trackers_[thread_idx]; int32_t s = static_cast(shape); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); bool is_idle = (phase == CoreTracker::DispatchPhase::IDLE); // Size the pop exactly as dispatch_shape does: MIX to the cluster count, else the @@ -808,7 +808,7 @@ int32_t SchedulerContext::try_early_dispatch( // regular ready_queues) strictly precedes early — there is no real ready task // to delay only when every normal queue is drained. if (pmu_active || !tracker.has_any_free_slot()) return 0; - for (int s = 0; s < PTO2_NUM_RESOURCE_SHAPES; s++) { + for (int s = 0; s < NUM_RESOURCE_SHAPES; s++) { if (sched_->ready_sync_queues[s].size() > 0 || sched_->ready_queues[s].size() > 0) return 0; } @@ -860,7 +860,7 @@ int32_t SchedulerContext::try_early_dispatch( // enters drain, so the stage callback always reports "no stop". run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { total_staged += early_dispatch_shape(thread_idx, shape, phase); return false; }, @@ -1108,7 +1108,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // snapshot so the next phase's "at_start" equals the previous phase's // "at_end". // - // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches PTO2_NUM_RESOURCE_SHAPES: AIC/AIV/MIX. + // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches NUM_RESOURCE_SHAPES: AIC/AIV/MIX. // // **Hot-path cost discipline.** Shared depth (PTO2ReadyQueue::size) is two // atomic relaxed loads against cache lines that all peer sched threads also @@ -1119,7 +1119,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // complete-emit and dispatch-emit in the same iter both reuse the same // shared sample. static_assert( - CHIP_SWIMLANE_NUM_QUEUE_SHAPES == PTO2_NUM_RESOURCE_SHAPES, + CHIP_SWIMLANE_NUM_QUEUE_SHAPES == NUM_RESOURCE_SHAPES, "queue snapshot width must match runtime resource shape count" ); int16_t phase_start_shared[CHIP_SWIMLANE_NUM_QUEUE_SHAPES] = {0}; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h index 012aa37868..3ba97ee8a7 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h @@ -288,15 +288,15 @@ class alignas(64) CoreTracker { // --- Cluster matching --- - BitStates get_valid_cluster_offset_states(PTO2ResourceShape shape) const { + BitStates get_valid_cluster_offset_states(ResourceShape shape) const { switch (shape) { - case PTO2ResourceShape::AIC: + case ResourceShape::AIC: return core_states_ & aic_mask_; - case PTO2ResourceShape::AIV: + case ResourceShape::AIV: return ((core_states_ >> 1) | (core_states_ >> 2)) & aic_mask_; - case PTO2ResourceShape::MIX: + case ResourceShape::MIX: return (core_states_ >> 1) & (core_states_ >> 2) & core_states_ & aic_mask_; - case PTO2ResourceShape::DUMMY: + case ResourceShape::DUMMY: // DUMMY tasks never reach the core-tracker dispatch path; they are // completed inline by resolve_and_dispatch via dummy_ready_queue. return {}; @@ -345,11 +345,11 @@ class alignas(64) CoreTracker { // always have pending_occupied=0, so AIV/MIX need no extra filtering. // Skipping the AIC-centric filter also fixes a latent bug where a running+pending AIC core // would incorrectly block AIV idle dispatch on the same cluster. - BitStates get_idle_core_offset_states(PTO2ResourceShape shape) const { - if (shape == PTO2ResourceShape::AIC) { + BitStates get_idle_core_offset_states(ResourceShape shape) const { + if (shape == ResourceShape::AIC) { return get_valid_cluster_offset_states(shape) & ~(pending_occupied_ & aic_mask_); } - if (shape == PTO2ResourceShape::AIV) { + if (shape == ResourceShape::AIV) { return core_states_ & aiv_mask_; } return get_valid_cluster_offset_states(shape); // MIX: cluster-level @@ -461,11 +461,11 @@ class alignas(64) CoreTracker { // staging. AIC/AIV count cores; MIX counts clusters because one logical MIX // block may occupy multiple cores in the same cluster. Gated early staging // includes pending slots, while ready staging is restricted to idle slots. - int32_t count_available_blocks(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending) const { - if (shape == PTO2ResourceShape::MIX) { + int32_t count_available_blocks(ResourceShape shape, uint8_t core_mask, bool include_pending) const { + if (shape == ResourceShape::MIX) { return include_pending ? count_mix_split_clusters(core_mask) : count_mix_running_clusters(core_mask); } - if (shape == PTO2ResourceShape::DUMMY) return 0; + if (shape == ResourceShape::DUMMY) return 0; int32_t available = get_idle_core_offset_states(shape).count(); if (include_pending) { @@ -474,8 +474,8 @@ class alignas(64) CoreTracker { return available; } - BitStates get_pending_core_offset_states(PTO2ResourceShape shape) const { - if (shape == PTO2ResourceShape::MIX) { + BitStates get_pending_core_offset_states(ResourceShape shape) const { + if (shape == ResourceShape::MIX) { // Shape-level query kept conservative for legacy callers/tests. // The real MIX dispatch path applies active_mask in classify_mix_cluster(). // Any core without a pending payload can accept a dispatch (idle or running). @@ -489,7 +489,7 @@ class alignas(64) CoreTracker { (running & aic_mask_) & ((running >> 1) & aic_mask_) & ((running >> 2) & aic_mask_); return mix_available & cluster_all_running; } - if (shape == PTO2ResourceShape::AIC) { + if (shape == ResourceShape::AIC) { return (~core_states_) & aic_mask_ & ~(pending_occupied_ & aic_mask_); } // AIV @@ -500,7 +500,7 @@ class alignas(64) CoreTracker { enum class DispatchPhase : uint8_t { IDLE, PENDING }; - BitStates get_dispatchable_cores(PTO2ResourceShape shape, DispatchPhase phase) const { + BitStates get_dispatchable_cores(ResourceShape shape, DispatchPhase phase) const { return (phase == DispatchPhase::IDLE) ? get_idle_core_offset_states(shape) : get_pending_core_offset_states(shape); } diff --git a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 833f5bf5b5..71d205a338 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -85,14 +85,14 @@ PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena) { // Fixed-capacity early-dispatch queues first, then the PTO2_READY_QUEUE_SIZE // ones. The big nine are the arena's last reservations so that the bytes bind // uploads stay one contiguous range no matter how much of them is in use. - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); } layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); } layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); @@ -117,16 +117,16 @@ bool PTO2SchedulerState::init_data_from_layout( return false; } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_init_data_from_layout(&sched->ready_queues[i], layout.ready_queue_capacity); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_init_data_from_layout(&sched->ready_sync_queues[i], layout.ready_queue_capacity); } ready_queue_init_data_from_layout(&sched->dummy_ready_queue, layout.ready_queue_capacity); ready_queue_init_data_from_layout(&sched->graph_ready_queue, layout.ready_queue_capacity); ready_queue_init_data_from_layout(&sched->graph_prepare_queue, layout.ready_queue_capacity); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_init_data_from_layout(&sched->early_dispatch_queues[i], PTO2_EARLY_DISPATCH_QUEUE_SIZE); } ready_queue_init_data_from_layout(&sched->early_sync_start_queue, PTO2_EARLY_DISPATCH_QUEUE_SIZE); @@ -143,16 +143,16 @@ bool PTO2SchedulerState::init_data_from_layout( // sequence already equals the position being claimed. void PTO2SchedulerState::seed_queue_slots() { PTO2SchedulerState *sched = this; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_queues[i].seed_slots(); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_sync_queues[i].seed_slots(); } sched->dummy_ready_queue.seed_slots(); sched->graph_ready_queue.seed_slots(); sched->graph_prepare_queue.seed_slots(); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->early_dispatch_queues[i].seed_slots(); } sched->early_sync_start_queue.seed_slots(); @@ -160,16 +160,16 @@ void PTO2SchedulerState::seed_queue_slots() { void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena) { PTO2SchedulerState *sched = this; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_queues[i], arena, layout.off_ready_queue_slots[i]); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_sync_queues[i], arena, layout.off_ready_sync_queue_slots[i]); } ready_queue_wire_arena_pointers(&sched->dummy_ready_queue, arena, layout.off_dummy_ready_queue_slots); ready_queue_wire_arena_pointers(&sched->graph_ready_queue, arena, layout.off_graph_ready_queue_slots); ready_queue_wire_arena_pointers(&sched->graph_prepare_queue, arena, layout.off_graph_prepare_queue_slots); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers( &sched->early_dispatch_queues[i], arena, layout.off_early_dispatch_queue_slots[i] ); @@ -180,16 +180,16 @@ void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, void PTO2SchedulerState::destroy() { PTO2SchedulerState *sched = this; sched->ring_sched_state.destroy(); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_queues[i]); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_sync_queues[i]); } ready_queue_destroy(&sched->dummy_ready_queue); ready_queue_destroy(&sched->graph_ready_queue); ready_queue_destroy(&sched->graph_prepare_queue); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->early_dispatch_queues[i]); } ready_queue_destroy(&sched->early_sync_start_queue); diff --git a/src/a5/runtime/host_build_graph/runtime/submit_types.h b/src/a5/runtime/host_build_graph/runtime/submit_types.h index aa58f02aad..abcebb86f1 100644 --- a/src/a5/runtime/host_build_graph/runtime/submit_types.h +++ b/src/a5/runtime/host_build_graph/runtime/submit_types.h @@ -105,7 +105,7 @@ struct DispatchPredicate { * with an empty core_mask route to a dedicated DUMMY ready queue and are * completed inline by the scheduler dispatch loop, bypassing core allocation. */ -enum class PTO2ResourceShape : uint8_t { +enum class ResourceShape : uint8_t { AIC = 0, // Single AIC AIV = 1, // Single AIV MIX = 2, // Full cluster (dispatch uses active_mask) @@ -115,7 +115,7 @@ enum class PTO2ResourceShape : uint8_t { // Number of *dispatchable* resource shapes (AIC, AIV, MIX). DUMMY does not // allocate a per-shape ready_queue entry / local buffer — it lives in a // dedicated queue inside PTO2SchedulerState. -inline constexpr int32_t PTO2_NUM_RESOURCE_SHAPES = 3; +inline constexpr int32_t NUM_RESOURCE_SHAPES = 3; /** * Bitmask of active subtask slots (AIC/AIV0/AIV1), sizeof == 1. @@ -136,13 +136,13 @@ class ActiveMask { uint8_t core_mask() const { return raw_ & 0x07u; } - PTO2ResourceShape to_shape() const { + ResourceShape to_shape() const { uint8_t cmask = core_mask(); - if (cmask == 0) return PTO2ResourceShape::DUMMY; + if (cmask == 0) return ResourceShape::DUMMY; int bit_count = __builtin_popcount(cmask); - if (bit_count >= 2) return PTO2ResourceShape::MIX; - if (cmask & PTO2_SUBTASK_MASK_AIC) return PTO2ResourceShape::AIC; - return PTO2ResourceShape::AIV; + if (bit_count >= 2) return ResourceShape::MIX; + if (cmask & PTO2_SUBTASK_MASK_AIC) return ResourceShape::AIC; + return ResourceShape::AIV; } bool operator==(ActiveMask other) const { return raw_ == other.raw_; } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 0f09749384..b2158cfe71 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -581,7 +581,7 @@ After these phases, the scheduler updates profiling headers and checks for termi Ready queues use a lock-free bounded MPMC (Vyukov) design: -- One `PTO2ReadyQueue` per resource shape — 3 shapes (`PTO2_NUM_RESOURCE_SHAPES`): `MIX` +- One `PTO2ReadyQueue` per resource shape — 3 shapes (`NUM_RESOURCE_SHAPES`): `MIX` (AIC+AIV cluster), `AIC`, `AIV`. Alongside `ready_queues[]` there is a per-shape `ready_sync_queues[]` (sync_start Tier-0) and the speculative `early_dispatch_queues[]` / `early_sync_start_queue` — see §8.6 for the full source × tier model. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index c30b04bcb1..c2dd39548d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -1198,8 +1198,8 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k // Deadlock check: block_num >= total available slots of the required type. // For MIX/AIC: limit is total_cluster_count (one AIC per cluster). // For AIV: limit is total_aiv_count. - PTO2ResourceShape shape = active_mask.to_shape(); - int32_t limit = (shape == PTO2ResourceShape::AIV) ? orch->total_aiv_count : orch->total_cluster_count; + ResourceShape shape = active_mask.to_shape(); + int32_t limit = (shape == ResourceShape::AIV) ? orch->total_aiv_count : orch->total_cluster_count; if (limit > 0 && block_num > limit) { report_fatal( SIMPLER_ERROR_REQUIRE_SYNC_START_INVALID, __FUNCTION__, diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp index f49b418abe..23feaf064a 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp @@ -100,7 +100,7 @@ void PTO2SchedulerState::print_queues() { const char *shape_names[] = {"AIC", "AIV", "MIX"}; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { LOG_DEBUG(" %s: count=%" PRIu64, shape_names[i], sched->ready_queues[i].size()); } LOG_DEBUG(" DUMMY: count=%" PRIu64, sched->dummy_ready_queue.size()); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 2937bc25c4..37f74e50c3 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -425,10 +425,10 @@ struct CompletionStats { * capacities used at layout time (init_data_from_layout reuses them). */ struct PTO2SchedulerLayout { - size_t off_ready_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; - size_t off_ready_sync_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; + size_t off_ready_queue_slots[NUM_RESOURCE_SHAPES]; + size_t off_ready_sync_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_dummy_ready_queue_slots; - size_t off_early_dispatch_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; + size_t off_early_dispatch_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_early_sync_start_queue_slots; size_t off_dep_pool_entries[CHIP_MAX_RING_DEPTH]; uint64_t ready_queue_capacity; @@ -528,14 +528,14 @@ struct PTO2SchedulerState { std::atomic publication_ack_mask; // Ready queues remain global (scheduling is ring-agnostic) - PTO2ReadyQueue ready_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; // Ready sync_start queues, one per shape. A ready sync_start cohort parks here // instead of ready_queues[] so the dispatch loop can drain it as a strict Tier-0 // (sync_start > MIX > C/V) before any regular ready task takes a core, while // reusing the same per-shape dispatch_shape machinery (fits-local inline vs // stop-the-world drain, per-core MIX placement, head-start spacing). - PTO2ReadyQueue ready_sync_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; // Dependency-only tasks (active_mask is empty, shape == DUMMY). Drained by // the dispatch loop and completed inline -- never goes to AICore. @@ -558,8 +558,8 @@ struct PTO2SchedulerState { // the per-shape ready_sync_queues[] (drained as Tier-0); everything else to // ready_queues[]. void push_ready_routed(ChipTaskSlotState *slot_state) { - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); - if (shape == PTO2ResourceShape::DUMMY || + ResourceShape shape = slot_state->active_mask.to_shape(); + if (shape == ResourceShape::DUMMY || (slot_state->task_attrs.has_predicate() && !slot_state->payload->predicate.pass())) { dummy_ready_queue.push(slot_state); } else if (slot_state->task_attrs.requires_sync_start()) { @@ -766,7 +766,7 @@ struct PTO2SchedulerState { uint32_t token{0}; }; EarlyDispatchDoorbell early_dispatch_doorbell_table[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; - PTO2ReadyQueue early_dispatch_queues[PTO2_NUM_RESOURCE_SHAPES]; + PTO2ReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; // Shape-agnostic sync_start candidates require an all-or-nothing owner // decision. The owner stages locally when one tracker can hold the complete // cohort and falls back to the generation-tagged global drain otherwise. @@ -909,8 +909,8 @@ struct PTO2SchedulerState { inline void try_enqueue_early_dispatch_candidate(ChipTaskSlotState &consumer) { if (consumer.task_attrs.has_predicate()) return; - PTO2ResourceShape shape = consumer.active_mask.to_shape(); - if (shape != PTO2ResourceShape::AIC && shape != PTO2ResourceShape::AIV && shape != PTO2ResourceShape::MIX) { + ResourceShape shape = consumer.active_mask.to_shape(); + if (shape != ResourceShape::AIC && shape != ResourceShape::AIV && shape != ResourceShape::MIX) { return; } @@ -1030,8 +1030,8 @@ struct PTO2SchedulerState { return true; } - PTO2ResourceShape shape = slot_state.active_mask.to_shape(); - if (shape == PTO2ResourceShape::DUMMY || + ResourceShape shape = slot_state.active_mask.to_shape(); + if (shape == ResourceShape::DUMMY || (slot_state.task_attrs.has_predicate() && !slot_state.payload->predicate.pass())) { dummy_ready_queue.push(&slot_state); } else if (slot_state.task_attrs.requires_sync_start()) { @@ -1067,8 +1067,8 @@ struct PTO2SchedulerState { } else if (early_handled) { return true; } - PTO2ResourceShape shape = slot_state.active_mask.to_shape(); - if (shape == PTO2ResourceShape::DUMMY || + ResourceShape shape = slot_state.active_mask.to_shape(); + if (shape == ResourceShape::DUMMY || (slot_state.task_attrs.has_predicate() && !slot_state.payload->predicate.pass())) { dummy_ready_queue.push(&slot_state, atomic_count, push_wait); } else if (slot_state.task_attrs.requires_sync_start()) { @@ -1107,13 +1107,13 @@ struct PTO2SchedulerState { } #endif - int get_ready_tasks_batch(PTO2ReadyQueue *queues, PTO2ResourceShape shape, ChipTaskSlotState **out, int max_count) { + int get_ready_tasks_batch(PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { return queues[static_cast(shape)].pop_batch(out, max_count); } #if SIMPLER_SCHED_PROFILING int get_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, + PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, uint64_t &wait_cycle ) { return queues[static_cast(shape)].pop_batch(out, max_count, atomic_count, wait_cycle); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index 495d56154c..7176b1cb9f 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -470,7 +470,7 @@ bool SchedulerContext::enter_drain_mode(ChipTaskSlotState *slot_state, int32_t b } // Count total available resources across all scheduler threads for a given shape. -int32_t SchedulerContext::count_global_available(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending) { +int32_t SchedulerContext::count_global_available(ResourceShape shape, uint8_t core_mask, bool include_pending) { int32_t total = 0; for (int32_t t = 0; t < active_sched_threads_; t++) { total += core_trackers_[t].count_available_blocks(shape, core_mask, include_pending); @@ -488,9 +488,9 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( [[maybe_unused]] bool record_drain_phases ) { CoreTracker &tracker = core_trackers_[thread_idx]; - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); + ResourceShape shape = slot_state->active_mask.to_shape(); uint8_t core_mask = slot_state->active_mask.core_mask(); - bool mix_split = gated && shape == PTO2ResourceShape::MIX; + bool mix_split = gated && shape == ResourceShape::MIX; SyncStartStageResult result; auto stage = [&](CoreTracker::BitStates valid, bool to_pending) { @@ -511,7 +511,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( claimed[b] = valid.pop_first(); for (int32_t b = 0; b < claim; b++) { auto core_offset = claimed[b]; - if (shape == PTO2ResourceShape::MIX) { + if (shape == ResourceShape::MIX) { result.running_cores += tracker.mix_cluster_idle_core_count(core_offset, core_mask); } handle_count += prepare_block_for_dispatch( @@ -559,15 +559,15 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( } #endif sched_->record_published_blocks(*slot_state, claim); - if (gated && shape != PTO2ResourceShape::MIX && !to_pending) result.running_cores += handle_count; + if (gated && shape != ResourceShape::MIX && !to_pending) result.running_cores += handle_count; } }; if (mix_split) { stage(tracker.get_mix_split_cluster_offset_states(core_mask), /*to_pending=*/true); } else { - auto idle = (shape == PTO2ResourceShape::MIX) ? tracker.get_mix_running_cluster_offset_states(core_mask) : - tracker.get_idle_core_offset_states(shape); + auto idle = (shape == ResourceShape::MIX) ? tracker.get_mix_running_cluster_offset_states(core_mask) : + tracker.get_idle_core_offset_states(shape); stage(idle, /*to_pending=*/false); if (gated) { stage(tracker.get_pending_core_offset_states(shape), /*to_pending=*/true); @@ -620,7 +620,7 @@ void SchedulerContext::handle_drain_mode( bool gated = slot_state->payload != nullptr && PTO2SchedulerState::owns_early_sync_drain(*slot_state->payload); if (coordinator) { - PTO2ResourceShape shape = slot_state->active_mask.to_shape(); + ResourceShape shape = slot_state->active_mask.to_shape(); int32_t available = count_global_available(shape, slot_state->active_mask.core_mask(), /*include_pending=*/gated); if (available < block_num) { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 8121386ede..fec3a2ef64 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -229,7 +229,7 @@ class SchedulerContext { // Dispatch (scheduler_dispatch.cpp) // ========================================================================= - static const char *shape_name(PTO2ResourceShape shape); + static const char *shape_name(ResourceShape shape); // Lower-case rendering of PTO2SubtaskSlot, used by dispatch and stall logs. // Kept lower-case to match the `kernels=[aic:N aiv0:N aiv1:N]` field @@ -247,7 +247,7 @@ class SchedulerContext { } int pop_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ); void build_payload( @@ -295,12 +295,12 @@ class SchedulerContext { // Fan out one block's subtasks (1 for AIC/AIV, 1-3 for MIX) into the // caller-supplied handles buffer. Returns the number of handles written. int prepare_block_for_dispatch( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2ResourceShape shape, - bool to_pending, int32_t block_idx, PublishHandle *out_handles, bool force_gate = false + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, ResourceShape shape, bool to_pending, + int32_t block_idx, PublishHandle *out_handles, bool force_gate = false ); void dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ); @@ -335,35 +335,33 @@ class SchedulerContext { int32_t thread_idx, CoreTracker &tracker, bool pmu_active, bool &made_progress, bool &try_pushed ); int32_t stage_consumer_blocks( - int32_t thread_idx, ChipTaskSlotState *c, PTO2ResourceShape shape, int32_t start, int32_t count, + int32_t thread_idx, ChipTaskSlotState *c, ResourceShape shape, int32_t start, int32_t count, CoreTracker::BitStates &idle, CoreTracker::BitStates &pend ); - int32_t early_dispatch_shape(int32_t thread_idx, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase); + int32_t early_dispatch_shape(int32_t thread_idx, ResourceShape shape, CoreTracker::DispatchPhase phase); // Returns true if any *other* scheduler thread currently has an idle core // matching `shape`. Used as a scheduling hint on the PENDING dispatch path // — see the implementation in scheduler_dispatch.cpp for the hint-semantics // rationale and the safety argument against the drain worker. - bool has_idle_in_other_threads(int32_t self_thread_idx, PTO2ResourceShape shape) const; + bool has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const; // True if mix tasks remain in the global MIX ready queue. Approximate — // PTO2ReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. - bool has_residual_mix() const { - return sched_->ready_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; - } + bool has_residual_mix() const { return sched_->ready_queues[static_cast(ResourceShape::MIX)].size() > 0; } // Tier-0 analog of has_residual_mix for the ready sync_start lane: true if MIX // sync_start cohorts remain queued, so the Tier-0 pass keeps MIX strict priority // over its own AIC/AIV sync work. Same relaxed-size snapshot caveat. bool has_residual_sync_mix() const { - return sched_->ready_sync_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; + return sched_->ready_sync_queues[static_cast(ResourceShape::MIX)].size() > 0; } bool has_residual_early_mix() const { - return sched_->early_dispatch_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; + return sched_->early_dispatch_queues[static_cast(ResourceShape::MIX)].size() > 0; } // ========================================================================= @@ -393,7 +391,7 @@ class SchedulerContext { ); bool enter_drain_mode(ChipTaskSlotState *slot_state, int32_t block_num); - int32_t count_global_available(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending = false); + int32_t count_global_available(ResourceShape shape, uint8_t core_mask, bool include_pending = false); struct SyncStartStageResult { int32_t staged_blocks{0}; int32_t running_cores{0}; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index f1f6b21343..1f09e12464 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -50,21 +50,21 @@ static_assert(offsetof(PTO2TaskPayload, scalars) == PTO2_TASKPAYLOAD_SCALARS_OFF static_assert(sizeof(ChipTensor) == PTO2_TASKPAYLOAD_TENSOR_STRIDE); static_assert(RUNTIME_MAX_WORKER <= PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64); -const char *SchedulerContext::shape_name(PTO2ResourceShape shape) { +const char *SchedulerContext::shape_name(ResourceShape shape) { switch (shape) { - case PTO2ResourceShape::AIC: + case ResourceShape::AIC: return "AIC"; - case PTO2ResourceShape::AIV: + case ResourceShape::AIV: return "AIV"; - case PTO2ResourceShape::MIX: + case ResourceShape::MIX: return "MIX"; - case PTO2ResourceShape::DUMMY: + case ResourceShape::DUMMY: return "DUMMY"; } return "UNKNOWN"; } -bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, PTO2ResourceShape shape) const { +bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const { // Cross-thread read of peer trackers without explicit synchronization. The // backing `core_states_` is a naturally aligned uint64_t; aarch64 guarantees // single-copy atomicity for an 8-byte aligned load, so no torn read. The @@ -83,7 +83,7 @@ bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, PTO2Re } int SchedulerContext::pop_ready_tasks_batch( - PTO2ReadyQueue *queues, PTO2ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ) { #if SIMPLER_DFX auto &chip_swimlane = sched_chip_swimlane_[thread_idx]; @@ -206,7 +206,7 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( } int SchedulerContext::prepare_block_for_dispatch( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2ResourceShape shape, bool to_pending, + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, ResourceShape shape, bool to_pending, int32_t block_idx, PublishHandle *out_handles, bool force_gate ) { #if SIMPLER_DFX @@ -223,7 +223,7 @@ int SchedulerContext::prepare_block_for_dispatch( } #endif CoreTracker &tracker = core_trackers_[thread_idx]; - if (shape == PTO2ResourceShape::MIX) { + if (shape == ResourceShape::MIX) { uint8_t cmask = slot_state.active_mask.core_mask(); int n = 0; // Preserve a5 MIX per-core slot placement: idle used cores take the @@ -255,7 +255,7 @@ int SchedulerContext::prepare_block_for_dispatch( sched_chip_swimlane_[thread_idx].phase_dispatch_count += __builtin_popcount(cmask); #endif return n; - } else if (shape == PTO2ResourceShape::AIC) { + } else if (shape == ResourceShape::AIC) { out_handles[0] = prepare_subtask_to_core( thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIC, to_pending, block_idx, force_gate ); @@ -275,7 +275,7 @@ int SchedulerContext::prepare_block_for_dispatch( } void SchedulerContext::dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ) { #if SIMPLER_SCHED_PROFILING @@ -284,7 +284,7 @@ void SchedulerContext::dispatch_shape( if (entered_drain) return; bool is_pending = (phase == CoreTracker::DispatchPhase::PENDING); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); auto cores = is_mix ? tracker.get_cluster_offset_states() : tracker.get_dispatchable_cores(shape, phase); if (!cores.has_value()) return; @@ -465,14 +465,14 @@ void SchedulerContext::run_staging_order( // MIX is handled explicitly at the top of each stage; only AIC/AIV cycle // through this 2-elem array, with order toggled by thread parity for // shape-level load balancing across threads. - static constexpr PTO2ResourceShape kAicAivOrder[2][2] = { - {PTO2ResourceShape::AIC, PTO2ResourceShape::AIV}, - {PTO2ResourceShape::AIV, PTO2ResourceShape::AIC}, + static constexpr ResourceShape kAicAivOrder[2][2] = { + {ResourceShape::AIC, ResourceShape::AIV}, + {ResourceShape::AIV, ResourceShape::AIC}, }; - const PTO2ResourceShape *aic_aiv = kAicAivOrder[thread_idx & 1]; + const ResourceShape *aic_aiv = kAicAivOrder[thread_idx & 1]; // ===== IDLE stage ===== - if (stage(PTO2ResourceShape::MIX, Phase::IDLE)) return; + if (stage(ResourceShape::MIX, Phase::IDLE)) return; // MIX-IDLE residual: AIC/AIV (both IDLE and PENDING) yield for this pass. // MIX-PENDING below still runs — that is the core of "mix strict priority": @@ -496,8 +496,8 @@ void SchedulerContext::run_staging_order( // // The gate is NOT subject to skip_aic_aiv — residual mix continues to drain // via pending slots on this thread when no peer is idle. - if (!has_idle_in_other_threads(thread_idx, PTO2ResourceShape::MIX)) { - if (stage(PTO2ResourceShape::MIX, Phase::PENDING)) return; + if (!has_idle_in_other_threads(thread_idx, ResourceShape::MIX)) { + if (stage(ResourceShape::MIX, Phase::PENDING)) return; } // Re-check after MIX-PENDING. If MIX-IDLE already set skip_aic_aiv, leave @@ -511,7 +511,7 @@ void SchedulerContext::run_staging_order( // AIC/AIV-PENDING gate: a peer-idle skip is a delay, not a loss — the peer // will pull from the global queue on its next IDLE pass. for (int i = 0; i < 2; i++) { - PTO2ResourceShape s = aic_aiv[i]; + ResourceShape s = aic_aiv[i]; if (has_idle_in_other_threads(thread_idx, s)) continue; if (stage(s, Phase::PENDING)) return; } @@ -530,7 +530,7 @@ void SchedulerContext::dispatch_ready_tasks( // (entered_drain), which also short-circuits the regular tier below. run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { dispatch_shape( thread_idx, sched_->ready_sync_queues, shape, phase, tracker, entered_drain, made_progress, try_pushed ); @@ -545,7 +545,7 @@ void SchedulerContext::dispatch_ready_tasks( // Tier 1: regular ready work. run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { dispatch_shape( thread_idx, sched_->ready_queues, shape, phase, tracker, entered_drain, made_progress, try_pushed ); @@ -577,7 +577,7 @@ void SchedulerContext::dispatch_ready_tasks( // those release did not take and rings them from its immutable local handles. // The seq_cst order guarantees every gated core has exactly one writer. int32_t SchedulerContext::stage_consumer_blocks( - int32_t thread_idx, ChipTaskSlotState *c, PTO2ResourceShape shape, int32_t start, int32_t count, + int32_t thread_idx, ChipTaskSlotState *c, ResourceShape shape, int32_t start, int32_t count, CoreTracker::BitStates &idle, CoreTracker::BitStates &pend ) { CoreTracker &tracker = core_trackers_[thread_idx]; @@ -663,10 +663,10 @@ int32_t SchedulerContext::stage_consumer_blocks( // shape's queue when dispatch_fanin becomes complete, so the shape is the queue // index (no per-consumer to_shape()). Returns the number of blocks staged. int32_t -SchedulerContext::early_dispatch_shape(int32_t thread_idx, PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { +SchedulerContext::early_dispatch_shape(int32_t thread_idx, ResourceShape shape, CoreTracker::DispatchPhase phase) { CoreTracker &tracker = core_trackers_[thread_idx]; int32_t s = static_cast(shape); - bool is_mix = (shape == PTO2ResourceShape::MIX); + bool is_mix = (shape == ResourceShape::MIX); bool is_idle = (phase == CoreTracker::DispatchPhase::IDLE); // Size the pop exactly as dispatch_shape does: MIX to the cluster count, else the @@ -770,7 +770,7 @@ int32_t SchedulerContext::try_early_dispatch( // regular ready_queues) strictly precedes early — there is no real ready task // to delay only when every normal queue is drained. if (pmu_active || !tracker.has_any_free_slot()) return 0; - for (int s = 0; s < PTO2_NUM_RESOURCE_SHAPES; s++) { + for (int s = 0; s < NUM_RESOURCE_SHAPES; s++) { if (sched_->ready_sync_queues[s].size() > 0 || sched_->ready_queues[s].size() > 0) return 0; } @@ -819,7 +819,7 @@ int32_t SchedulerContext::try_early_dispatch( run_staging_order( thread_idx, pmu_active, - [&](PTO2ResourceShape shape, CoreTracker::DispatchPhase phase) { + [&](ResourceShape shape, CoreTracker::DispatchPhase phase) { total_staged += early_dispatch_shape(thread_idx, shape, phase); return false; }, @@ -891,7 +891,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // snapshot so the next phase's "at_start" equals the previous phase's // "at_end". // - // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches PTO2_NUM_RESOURCE_SHAPES: AIC/AIV/MIX. + // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches NUM_RESOURCE_SHAPES: AIC/AIV/MIX. // // **Hot-path cost discipline.** Shared depth (PTO2ReadyQueue::size) is two // atomic relaxed loads against cache lines that all peer sched threads also @@ -902,7 +902,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // complete-emit and dispatch-emit in the same iter both reuse the same // shared sample. static_assert( - CHIP_SWIMLANE_NUM_QUEUE_SHAPES == PTO2_NUM_RESOURCE_SHAPES, + CHIP_SWIMLANE_NUM_QUEUE_SHAPES == NUM_RESOURCE_SHAPES, "queue snapshot width must match runtime resource shape count" ); int16_t phase_start_shared[CHIP_SWIMLANE_NUM_QUEUE_SHAPES] = {0}; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h index d1b8e59f29..e83f92cbe7 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h @@ -264,15 +264,15 @@ class alignas(64) CoreTracker { // --- Cluster matching --- - BitStates get_valid_cluster_offset_states(PTO2ResourceShape shape) const { + BitStates get_valid_cluster_offset_states(ResourceShape shape) const { switch (shape) { - case PTO2ResourceShape::AIC: + case ResourceShape::AIC: return core_states_ & aic_mask_; - case PTO2ResourceShape::AIV: + case ResourceShape::AIV: return ((core_states_ >> 1) | (core_states_ >> 2)) & aic_mask_; - case PTO2ResourceShape::MIX: + case ResourceShape::MIX: return (core_states_ >> 1) & (core_states_ >> 2) & core_states_ & aic_mask_; - case PTO2ResourceShape::DUMMY: + case ResourceShape::DUMMY: // DUMMY tasks never reach the core-tracker dispatch path; they are // completed inline by resolve_and_dispatch via dummy_ready_queue. return {}; @@ -321,11 +321,11 @@ class alignas(64) CoreTracker { // always have pending_occupied=0, so AIV/MIX need no extra filtering. // Skipping the AIC-centric filter also fixes a latent bug where a running+pending AIC core // would incorrectly block AIV idle dispatch on the same cluster. - BitStates get_idle_core_offset_states(PTO2ResourceShape shape) const { - if (shape == PTO2ResourceShape::AIC) { + BitStates get_idle_core_offset_states(ResourceShape shape) const { + if (shape == ResourceShape::AIC) { return get_valid_cluster_offset_states(shape) & ~(pending_occupied_ & aic_mask_); } - if (shape == PTO2ResourceShape::AIV) { + if (shape == ResourceShape::AIV) { return core_states_ & aiv_mask_; } return get_valid_cluster_offset_states(shape); // MIX: cluster-level @@ -429,8 +429,8 @@ class alignas(64) CoreTracker { return get_mix_split_cluster_offset_states(core_mask).count(); } - BitStates get_pending_core_offset_states(PTO2ResourceShape shape) const { - if (shape == PTO2ResourceShape::MIX) { + BitStates get_pending_core_offset_states(ResourceShape shape) const { + if (shape == ResourceShape::MIX) { // Shape-level query kept conservative for legacy callers/tests. // The real MIX dispatch path applies active_mask in classify_mix_cluster(). // Any core without a pending payload can accept a dispatch (idle or running). @@ -444,7 +444,7 @@ class alignas(64) CoreTracker { (running & aic_mask_) & ((running >> 1) & aic_mask_) & ((running >> 2) & aic_mask_); return mix_available & cluster_all_running; } - if (shape == PTO2ResourceShape::AIC) { + if (shape == ResourceShape::AIC) { return (~core_states_) & aic_mask_ & ~(pending_occupied_ & aic_mask_); } // AIV @@ -455,11 +455,11 @@ class alignas(64) CoreTracker { // one core per block; MIX uses one active-mask-compatible cluster per block. // Gated early staging may use both idle running slots and free pending slots, // while a ready cohort is restricted to idle running slots. - int32_t count_available_blocks(PTO2ResourceShape shape, uint8_t core_mask, bool include_pending) const { - if (shape == PTO2ResourceShape::MIX) { + int32_t count_available_blocks(ResourceShape shape, uint8_t core_mask, bool include_pending) const { + if (shape == ResourceShape::MIX) { return include_pending ? count_mix_split_clusters(core_mask) : count_mix_running_clusters(core_mask); } - if (shape != PTO2ResourceShape::AIC && shape != PTO2ResourceShape::AIV) return 0; + if (shape != ResourceShape::AIC && shape != ResourceShape::AIV) return 0; int32_t available = get_idle_core_offset_states(shape).count(); if (include_pending) available += get_pending_core_offset_states(shape).count(); @@ -470,7 +470,7 @@ class alignas(64) CoreTracker { enum class DispatchPhase : uint8_t { IDLE, PENDING }; - BitStates get_dispatchable_cores(PTO2ResourceShape shape, DispatchPhase phase) const { + BitStates get_dispatchable_cores(ResourceShape shape, DispatchPhase phase) const { return (phase == DispatchPhase::IDLE) ? get_idle_core_offset_states(shape) : get_pending_core_offset_states(shape); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 29cefa6afd..11c7fa190d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -142,14 +142,14 @@ PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_ca layout.dep_pool_capacities[r] = dep_pool_capacities[r]; } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); } layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); } layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); @@ -181,14 +181,14 @@ bool PTO2SchedulerState::init_data_from_layout( } } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { if (!ready_queue_init_data_from_layout( &sched->ready_queues[i], arena, layout.off_ready_queue_slots[i], layout.ready_queue_capacity )) { return false; } } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { if (!ready_queue_init_data_from_layout( &sched->ready_sync_queues[i], arena, layout.off_ready_sync_queue_slots[i], layout.ready_queue_capacity )) { @@ -200,7 +200,7 @@ bool PTO2SchedulerState::init_data_from_layout( )) { return false; } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { if (!ready_queue_init_data_from_layout( &sched->early_dispatch_queues[i], arena, layout.off_early_dispatch_queue_slots[i], PTO2_EARLY_DISPATCH_QUEUE_SIZE @@ -244,14 +244,14 @@ void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void sched->ring_sched_states[r].reset_for_reuse(sm_dev_base, r, orch_err); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_queues[i].reset_for_reuse(); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_sync_queues[i].reset_for_reuse(); } sched->dummy_ready_queue.reset_for_reuse(); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->early_dispatch_queues[i].reset_for_reuse(); } sched->early_sync_start_queue.reset_for_reuse(); @@ -262,14 +262,14 @@ void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena) { PTO2SchedulerState *sched = this; - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_queues[i], arena, layout.off_ready_queue_slots[i]); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_sync_queues[i], arena, layout.off_ready_sync_queue_slots[i]); } ready_queue_wire_arena_pointers(&sched->dummy_ready_queue, arena, layout.off_dummy_ready_queue_slots); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers( &sched->early_dispatch_queues[i], arena, layout.off_early_dispatch_queue_slots[i] ); @@ -290,14 +290,14 @@ void PTO2SchedulerState::destroy() { sched->ring_sched_states[r].destroy(); sched->ring_sched_states[r].dep_pool.base = nullptr; } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_queues[i]); } - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_sync_queues[i]); } ready_queue_destroy(&sched->dummy_ready_queue); - for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { + for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->early_dispatch_queues[i]); } ready_queue_destroy(&sched->early_sync_start_queue); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h index 9a5fbacded..40c4c80907 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h @@ -105,7 +105,7 @@ struct DispatchPredicate { * with an empty core_mask route to a dedicated DUMMY ready queue and are * completed inline by the scheduler dispatch loop, bypassing core allocation. */ -enum class PTO2ResourceShape : uint8_t { +enum class ResourceShape : uint8_t { AIC = 0, // Single AIC AIV = 1, // Single AIV MIX = 2, // Full cluster (dispatch uses active_mask) @@ -115,7 +115,7 @@ enum class PTO2ResourceShape : uint8_t { // Number of *dispatchable* resource shapes (AIC, AIV, MIX). DUMMY does not // allocate a per-shape ready_queue entry / local buffer — it lives in a // dedicated queue inside PTO2SchedulerState. -inline constexpr int32_t PTO2_NUM_RESOURCE_SHAPES = 3; +inline constexpr int32_t NUM_RESOURCE_SHAPES = 3; /** * Bitmask of active subtask slots (AIC/AIV0/AIV1), sizeof == 1. @@ -136,13 +136,13 @@ class ActiveMask { uint8_t core_mask() const { return raw_ & 0x07u; } - PTO2ResourceShape to_shape() const { + ResourceShape to_shape() const { uint8_t cmask = core_mask(); - if (cmask == 0) return PTO2ResourceShape::DUMMY; + if (cmask == 0) return ResourceShape::DUMMY; int bit_count = __builtin_popcount(cmask); - if (bit_count >= 2) return PTO2ResourceShape::MIX; - if (cmask & PTO2_SUBTASK_MASK_AIC) return PTO2ResourceShape::AIC; - return PTO2ResourceShape::AIV; + if (bit_count >= 2) return ResourceShape::MIX; + if (cmask & PTO2_SUBTASK_MASK_AIC) return ResourceShape::AIC; + return ResourceShape::AIV; } bool operator==(ActiveMask other) const { return raw_ == other.raw_; } diff --git a/src/common/platform/include/common/chip_swimlane_profiling.h b/src/common/platform/include/common/chip_swimlane_profiling.h index ff770bf443..0773d2920c 100644 --- a/src/common/platform/include/common/chip_swimlane_profiling.h +++ b/src/common/platform/include/common/chip_swimlane_profiling.h @@ -547,7 +547,7 @@ enum class ChipSwimlaneSchedPhaseKind : uint32_t { }; /** Index layout of the queue-depth snapshot arrays below: AIC=0, AIV=1, MIX=2. - * Must match PTO2ResourceShape's first three values (see submit_types.h). + * Must match ResourceShape's first three values (see submit_types.h). * Hardcoded here rather than included to keep this header runtime-independent. */ constexpr int CHIP_SWIMLANE_NUM_QUEUE_SHAPES = 3; diff --git a/tests/ut/cpp/a2a3/test_graph_activation.cpp b/tests/ut/cpp/a2a3/test_graph_activation.cpp index 2555fbbfef..c55180a193 100644 --- a/tests/ut/cpp/a2a3/test_graph_activation.cpp +++ b/tests/ut/cpp/a2a3/test_graph_activation.cpp @@ -89,7 +89,7 @@ TEST_F(GraphActivationTest, WakeRoutesConsumerWhenProducerCompletedBeforeRegiste sched.register_graph_wake(exec, &nodes[0].slot, &nodes[1].slot); ChipTaskSlotState *out[2]; - ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, PTO2ResourceShape::AIC, out, 2), 1) + ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 2), 1) << "consumer must route to ready, not hang on the SENTINEL wake list"; EXPECT_EQ(out[0], &nodes[1].slot); } @@ -115,13 +115,13 @@ TEST_F(GraphActivationTest, IncrementalPublishRoutesCompletedDepsAndWakeChainsPe EXPECT_EQ(exec.published_nodes.load(), 4); ChipTaskSlotState *out[4]; - ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, PTO2ResourceShape::AIC, out, 4), 1) + ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 4), 1) << "only the consumer whose producers are all COMPLETED routes at publish time"; EXPECT_EQ(out[0], &nodes[2].slot); nodes[1].slot.task_state.store(PTO2_TASK_COMPLETED); sched.drain_graph_wake_list(exec, nodes[1].slot); - ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, PTO2ResourceShape::AIC, out, 4), 1) + ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 4), 1) << "the wake-chained consumer must route once its pending producer completes"; EXPECT_EQ(out[0], &nodes[3].slot); } diff --git a/tests/ut/cpp/a2a3/test_scheduler_state.cpp b/tests/ut/cpp/a2a3/test_scheduler_state.cpp index f8237dda32..447490f1b0 100644 --- a/tests/ut/cpp/a2a3/test_scheduler_state.cpp +++ b/tests/ut/cpp/a2a3/test_scheduler_state.cpp @@ -364,7 +364,7 @@ TEST_F(SchedulerStateTest, GetReadyTasksBatchDrainsSharedQueue) { ASSERT_TRUE(sched.release_fanin_and_check_ready(slot_b)); ChipTaskSlotState *out[4]; - int count = sched.get_ready_tasks_batch(sched.ready_queues, PTO2ResourceShape::AIC, out, 4); + int count = sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 4); EXPECT_EQ(count, 2); // Shared queue is FIFO, so slot_a (pushed first) comes first. @@ -394,7 +394,7 @@ TEST(CoreTrackerTest, MixPartiallyRunningClusterAdmittedAsPerCorePlacement) { // Not all used cores are idle, so the IDLE phase skips this cluster; it is // consumed by the PENDING phase. - auto idle = tracker.get_idle_core_offset_states(PTO2ResourceShape::MIX); + auto idle = tracker.get_idle_core_offset_states(ResourceShape::MIX); EXPECT_FALSE(idle.has_value()); } @@ -411,7 +411,7 @@ TEST(CoreTrackerTest, MixPendingAcceptsFullyRunningClusterWithFreePendingSlots) tracker.clear_pending_occupied(cluster_offset + 1); tracker.clear_pending_occupied(cluster_offset + 2); - auto pending = tracker.get_pending_core_offset_states(PTO2ResourceShape::MIX); + auto pending = tracker.get_pending_core_offset_states(ResourceShape::MIX); EXPECT_TRUE(pending.has_value()); EXPECT_EQ(pending.count(), 1); } @@ -427,7 +427,7 @@ TEST(CoreTrackerTest, MixPendingRejectsFullyRunningClusterWithOccupiedPendingSlo tracker.change_core_state(cluster_offset + 2); tracker.set_pending_occupied(cluster_offset + 1); - auto pending = tracker.get_pending_core_offset_states(PTO2ResourceShape::MIX); + auto pending = tracker.get_pending_core_offset_states(ResourceShape::MIX); EXPECT_FALSE(pending.has_value()); } @@ -436,11 +436,11 @@ TEST(CoreTrackerTest, MixIdleAndPendingDoNotDoubleAdmitFullyIdleCluster) { tracker.init(1); tracker.set_cluster(0, 0, 1, 2); - auto idle = tracker.get_idle_core_offset_states(PTO2ResourceShape::MIX); + auto idle = tracker.get_idle_core_offset_states(ResourceShape::MIX); EXPECT_TRUE(idle.has_value()); EXPECT_EQ(idle.count(), 1); - auto pending = tracker.get_pending_core_offset_states(PTO2ResourceShape::MIX); + auto pending = tracker.get_pending_core_offset_states(ResourceShape::MIX); EXPECT_FALSE(pending.has_value()); } @@ -509,7 +509,7 @@ TEST(CoreTrackerTest, MixRunningClusterHelpersUseActiveMask) { constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; - EXPECT_EQ(tracker.get_idle_core_offset_states(PTO2ResourceShape::MIX).count(), 0); + EXPECT_EQ(tracker.get_idle_core_offset_states(ResourceShape::MIX).count(), 0); EXPECT_EQ(tracker.count_mix_running_clusters(used_mask), 2); EXPECT_EQ(tracker.get_mix_running_cluster_offset_states(used_mask).count(), 2); } @@ -536,11 +536,11 @@ TEST(CoreTrackerTest, CountAvailableBlocksAivIncludesFreePendingSlots) { constexpr uint8_t aiv_mask = PTO2_SUBTASK_MASK_AIV0; constexpr int32_t exact_fit = 4; - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::AIV, aiv_mask, false), 2); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::AIV, aiv_mask, true), exact_fit); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, false), 2); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, true), exact_fit); tracker.set_pending_occupied(4); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::AIV, aiv_mask, true), exact_fit - 1); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, true), exact_fit - 1); } TEST(CoreTrackerTest, CountAvailableBlocksMixCountsLogicalClusters) { @@ -552,8 +552,8 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixCountsLogicalClusters) { tracker.change_core_state(4); constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0 | PTO2_SUBTASK_MASK_AIV1; - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, false), 1); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, true), 2); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, false), 1); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, true), 2); } TEST(CoreTrackerTest, CountAvailableBlocksMixRejectsOccupiedUsedPendingSlot) { @@ -565,8 +565,8 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixRejectsOccupiedUsedPendingSlot) { tracker.change_core_state(1); tracker.set_pending_occupied(1); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, false), 0); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, true), 0); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, false), 0); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, true), 0); } TEST(CoreTrackerTest, CountAvailableBlocksMixIgnoresUnavailableUnusedCore) { @@ -578,6 +578,6 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixIgnoresUnavailableUnusedCore) { tracker.change_core_state(2); tracker.set_pending_occupied(2); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, false), 1); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, true), 1); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, false), 1); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, true), 1); } diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index 9c74dabe06..23f3486b35 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -140,7 +140,7 @@ TEST_F(WiringTest, NoFaninTaskBecomesReady) { EXPECT_EQ(task_slot.fanin_refcount.load(), 1); // Task should be in ready queue - PTO2ResourceShape shape = task_slot.active_mask.to_shape(); + ResourceShape shape = task_slot.active_mask.to_shape(); auto *popped = sched.ready_queues[static_cast(shape)].pop(); EXPECT_EQ(popped, &task_slot); } @@ -178,7 +178,7 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { EXPECT_GE(task_slot.fanin_refcount.load(), task_slot.fanin_count); // Task should be in ready queue - PTO2ResourceShape shape = task_slot.active_mask.to_shape(); + ResourceShape shape = task_slot.active_mask.to_shape(); auto *popped = sched.ready_queues[static_cast(shape)].pop(); EXPECT_EQ(popped, &task_slot); } @@ -215,7 +215,7 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { EXPECT_LT(task_slot.fanin_refcount.load(), task_slot.fanin_count); // Ready queue should be empty - PTO2ResourceShape shape = task_slot.active_mask.to_shape(); + ResourceShape shape = task_slot.active_mask.to_shape(); auto *popped = sched.ready_queues[static_cast(shape)].pop(); EXPECT_EQ(popped, nullptr); @@ -463,7 +463,7 @@ TEST_F(WiringTest, EarlyDispatchQueueOverflowFallsBackToNormalDispatch) { init_slot(filler, PTO2_TASK_PENDING, 0, 1); init_slot(consumer, PTO2_TASK_PENDING, 1, 1); - PTO2ResourceShape shape = consumer.active_mask.to_shape(); + ResourceShape shape = consumer.active_mask.to_shape(); auto &queue = sched.early_dispatch_queues[static_cast(shape)]; for (uint64_t i = 0; i < queue.capacity; i++) { ASSERT_TRUE(queue.push_tagged(&filler, i)); @@ -497,7 +497,7 @@ TEST_F(WiringTest, EarlyDispatchSyncStartQueueOverflowFallsBackToSyncReadyQueue) // A sync_start cohort that never reached its drain falls back to the // shape's sync ready queue, not the plain one. - PTO2ResourceShape shape = consumer.active_mask.to_shape(); + ResourceShape shape = consumer.active_mask.to_shape(); EXPECT_TRUE(sched.route_ready_once(consumer)); EXPECT_EQ(sched.ready_sync_queues[static_cast(shape)].pop(), &consumer); } @@ -620,7 +620,7 @@ TEST_F(WiringTest, PartialStagedReleaseRoutesRemainderToReadyQueue) { EXPECT_EQ(consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); EXPECT_EQ(consumer.next_block_idx.load(), 2); - PTO2ResourceShape shape = consumer.active_mask.to_shape(); + ResourceShape shape = consumer.active_mask.to_shape(); EXPECT_EQ(sched.ready_queues[static_cast(shape)].pop(), &consumer); int32_t remaining_start = -1; @@ -1127,7 +1127,7 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { EXPECT_EQ(consumer2.fanin_refcount.load(), 2); // Both consumers should be ready (fanin_refcount == fanin_count) - PTO2ResourceShape shape = consumer1.active_mask.to_shape(); + ResourceShape shape = consumer1.active_mask.to_shape(); auto *r1 = sched.ready_queues[static_cast(shape)].pop(); auto *r2 = sched.ready_queues[static_cast(shape)].pop(); EXPECT_TRUE((r1 == &consumer1 && r2 == &consumer2) || (r1 == &consumer2 && r2 == &consumer1)); diff --git a/tests/ut/cpp/a5/test_graph_activation.cpp b/tests/ut/cpp/a5/test_graph_activation.cpp index 2555fbbfef..c55180a193 100644 --- a/tests/ut/cpp/a5/test_graph_activation.cpp +++ b/tests/ut/cpp/a5/test_graph_activation.cpp @@ -89,7 +89,7 @@ TEST_F(GraphActivationTest, WakeRoutesConsumerWhenProducerCompletedBeforeRegiste sched.register_graph_wake(exec, &nodes[0].slot, &nodes[1].slot); ChipTaskSlotState *out[2]; - ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, PTO2ResourceShape::AIC, out, 2), 1) + ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 2), 1) << "consumer must route to ready, not hang on the SENTINEL wake list"; EXPECT_EQ(out[0], &nodes[1].slot); } @@ -115,13 +115,13 @@ TEST_F(GraphActivationTest, IncrementalPublishRoutesCompletedDepsAndWakeChainsPe EXPECT_EQ(exec.published_nodes.load(), 4); ChipTaskSlotState *out[4]; - ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, PTO2ResourceShape::AIC, out, 4), 1) + ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 4), 1) << "only the consumer whose producers are all COMPLETED routes at publish time"; EXPECT_EQ(out[0], &nodes[2].slot); nodes[1].slot.task_state.store(PTO2_TASK_COMPLETED); sched.drain_graph_wake_list(exec, nodes[1].slot); - ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, PTO2ResourceShape::AIC, out, 4), 1) + ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 4), 1) << "the wake-chained consumer must route once its pending producer completes"; EXPECT_EQ(out[0], &nodes[3].slot); } diff --git a/tests/ut/cpp/a5/test_scheduler_state.cpp b/tests/ut/cpp/a5/test_scheduler_state.cpp index 7f0aac1a10..1dca08f8e4 100644 --- a/tests/ut/cpp/a5/test_scheduler_state.cpp +++ b/tests/ut/cpp/a5/test_scheduler_state.cpp @@ -417,7 +417,7 @@ TEST_F(SchedulerStateTest, GetReadyTasksBatchDrainsSharedQueue) { ASSERT_TRUE(sched.release_fanin_and_check_ready(slot_b)); ChipTaskSlotState *out[4]; - int count = sched.get_ready_tasks_batch(sched.ready_queues, PTO2ResourceShape::AIC, out, 4); + int count = sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 4); EXPECT_EQ(count, 2); // Shared queue is FIFO, so slot_a (pushed first) comes first. @@ -433,8 +433,8 @@ TEST_F(SchedulerStateTest, SyncStartRoutesToDedicatedReadyQueue) { ASSERT_TRUE(sched.release_fanin_and_check_ready(slot)); ChipTaskSlotState *out[1]; - EXPECT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, PTO2ResourceShape::AIC, out, 1), 0); - ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_sync_queues, PTO2ResourceShape::AIC, out, 1), 1); + EXPECT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 1), 0); + ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_sync_queues, ResourceShape::AIC, out, 1), 1); EXPECT_EQ(out[0], &slot); } @@ -460,7 +460,7 @@ TEST(CoreTrackerTest, MixPartiallyRunningClusterAdmittedAsPerCorePlacement) { // Not all used cores are idle, so the IDLE phase skips this cluster; it is // consumed by the PENDING phase. - auto idle = tracker.get_idle_core_offset_states(PTO2ResourceShape::MIX); + auto idle = tracker.get_idle_core_offset_states(ResourceShape::MIX); EXPECT_FALSE(idle.has_value()); } @@ -477,7 +477,7 @@ TEST(CoreTrackerTest, MixPendingAcceptsFullyRunningClusterWithFreePendingSlots) tracker.clear_pending_occupied(cluster_offset + 1); tracker.clear_pending_occupied(cluster_offset + 2); - auto pending = tracker.get_pending_core_offset_states(PTO2ResourceShape::MIX); + auto pending = tracker.get_pending_core_offset_states(ResourceShape::MIX); EXPECT_TRUE(pending.has_value()); EXPECT_EQ(pending.count(), 1); } @@ -493,7 +493,7 @@ TEST(CoreTrackerTest, MixPendingRejectsFullyRunningClusterWithOccupiedPendingSlo tracker.change_core_state(cluster_offset + 2); tracker.set_pending_occupied(cluster_offset + 1); - auto pending = tracker.get_pending_core_offset_states(PTO2ResourceShape::MIX); + auto pending = tracker.get_pending_core_offset_states(ResourceShape::MIX); EXPECT_FALSE(pending.has_value()); } @@ -502,11 +502,11 @@ TEST(CoreTrackerTest, MixIdleAndPendingDoNotDoubleAdmitFullyIdleCluster) { tracker.init(1); tracker.set_cluster(0, 0, 1, 2); - auto idle = tracker.get_idle_core_offset_states(PTO2ResourceShape::MIX); + auto idle = tracker.get_idle_core_offset_states(ResourceShape::MIX); EXPECT_TRUE(idle.has_value()); EXPECT_EQ(idle.count(), 1); - auto pending = tracker.get_pending_core_offset_states(PTO2ResourceShape::MIX); + auto pending = tracker.get_pending_core_offset_states(ResourceShape::MIX); EXPECT_FALSE(pending.has_value()); } @@ -575,7 +575,7 @@ TEST(CoreTrackerTest, MixRunningClusterHelpersUseActiveMask) { constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; - EXPECT_EQ(tracker.get_idle_core_offset_states(PTO2ResourceShape::MIX).count(), 0); + EXPECT_EQ(tracker.get_idle_core_offset_states(ResourceShape::MIX).count(), 0); EXPECT_EQ(tracker.count_mix_running_clusters(used_mask), 2); EXPECT_EQ(tracker.get_mix_running_cluster_offset_states(used_mask).count(), 2); } @@ -602,11 +602,11 @@ TEST(CoreTrackerTest, CountAvailableBlocksAivIncludesFreePendingSlots) { constexpr uint8_t aiv_mask = PTO2_SUBTASK_MASK_AIV0; constexpr int32_t exact_fit = 4; - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::AIV, aiv_mask, false), 2); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::AIV, aiv_mask, true), exact_fit); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, false), 2); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, true), exact_fit); tracker.set_pending_occupied(4); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::AIV, aiv_mask, true), exact_fit - 1); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, true), exact_fit - 1); } TEST(CoreTrackerTest, CountAvailableBlocksMixCountsLogicalClusters) { @@ -618,8 +618,8 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixCountsLogicalClusters) { tracker.change_core_state(4); constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0 | PTO2_SUBTASK_MASK_AIV1; - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, false), 1); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, true), 2); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, false), 1); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, true), 2); } TEST(CoreTrackerTest, CountAvailableBlocksMixRejectsOccupiedUsedPendingSlot) { @@ -631,8 +631,8 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixRejectsOccupiedUsedPendingSlot) { tracker.change_core_state(1); tracker.set_pending_occupied(1); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, false), 0); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, true), 0); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, false), 0); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, true), 0); } TEST(CoreTrackerTest, CountAvailableBlocksMixIgnoresUnavailableUnusedCore) { @@ -644,6 +644,6 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixIgnoresUnavailableUnusedCore) { tracker.change_core_state(2); tracker.set_pending_occupied(2); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, false), 1); - EXPECT_EQ(tracker.count_available_blocks(PTO2ResourceShape::MIX, used_mask, true), 1); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, false), 1); + EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, true), 1); } diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index d9449f2cbc..83c8d7e278 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -156,7 +156,7 @@ TEST_F(WiringTest, NoFaninTaskBecomesReady) { EXPECT_EQ(task_slot.fanin_refcount.load(), 1); // Task should be in ready queue - PTO2ResourceShape shape = task_slot.active_mask.to_shape(); + ResourceShape shape = task_slot.active_mask.to_shape(); auto *popped = sched.ready_queues[static_cast(shape)].pop(); EXPECT_EQ(popped, &task_slot); } @@ -194,7 +194,7 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { EXPECT_GE(task_slot.fanin_refcount.load(), task_slot.fanin_count); // Task should be in ready queue - PTO2ResourceShape shape = task_slot.active_mask.to_shape(); + ResourceShape shape = task_slot.active_mask.to_shape(); auto *popped = sched.ready_queues[static_cast(shape)].pop(); EXPECT_EQ(popped, &task_slot); } @@ -231,7 +231,7 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { EXPECT_LT(task_slot.fanin_refcount.load(), task_slot.fanin_count); // Ready queue should be empty - PTO2ResourceShape shape = task_slot.active_mask.to_shape(); + ResourceShape shape = task_slot.active_mask.to_shape(); auto *popped = sched.ready_queues[static_cast(shape)].pop(); EXPECT_EQ(popped, nullptr); @@ -446,7 +446,7 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { EXPECT_EQ(consumer2.fanin_refcount.load(), 2); // Both consumers should be ready (fanin_refcount == fanin_count) - PTO2ResourceShape shape = consumer1.active_mask.to_shape(); + ResourceShape shape = consumer1.active_mask.to_shape(); auto *r1 = sched.ready_queues[static_cast(shape)].pop(); auto *r2 = sched.ready_queues[static_cast(shape)].pop(); EXPECT_TRUE((r1 == &consumer1 && r2 == &consumer2) || (r1 == &consumer2 && r2 == &consumer1)); @@ -994,7 +994,7 @@ TEST_F(WiringTest, EarlyDispatchQueueOverflowFallsBackToNormalDispatch) { init_slot(filler, PTO2_TASK_PENDING, 0, 1); init_slot(consumer, PTO2_TASK_PENDING, 1, 1); - PTO2ResourceShape shape = consumer.active_mask.to_shape(); + ResourceShape shape = consumer.active_mask.to_shape(); auto &queue = sched.early_dispatch_queues[static_cast(shape)]; for (uint64_t i = 0; i < queue.capacity; i++) { ASSERT_TRUE(queue.push_tagged(&filler, i)); @@ -1028,7 +1028,7 @@ TEST_F(WiringTest, EarlyDispatchSyncStartQueueOverflowFallsBackToSyncReadyQueue) // A sync_start cohort that never reached its drain falls back to the // shape's sync ready queue, not the plain one. - PTO2ResourceShape shape = consumer.active_mask.to_shape(); + ResourceShape shape = consumer.active_mask.to_shape(); EXPECT_TRUE(sched.route_ready_once(consumer)); EXPECT_EQ(sched.ready_sync_queues[static_cast(shape)].pop(), &consumer); } diff --git a/tests/ut/cpp/common/test_hbg_scheduler_drain.cpp b/tests/ut/cpp/common/test_hbg_scheduler_drain.cpp index 768ac03c61..65bd836c02 100644 --- a/tests/ut/cpp/common/test_hbg_scheduler_drain.cpp +++ b/tests/ut/cpp/common/test_hbg_scheduler_drain.cpp @@ -28,7 +28,7 @@ extern "C" uint64_t get_platform_pmu_reg_addrs() { return 0; } extern "C" uint64_t get_platform_regs() { return 0; } int SchedulerContext::prepare_block_for_dispatch( - int32_t, int32_t, ChipTaskSlotState &, PTO2ResourceShape, bool, int32_t, PublishHandle *, bool + int32_t, int32_t, ChipTaskSlotState &, ResourceShape, bool, int32_t, PublishHandle *, bool ) { return 0; } From d84786f62796749427083d062449e8ee7e001b88 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:15:17 -0700 Subject: [PATCH 02/22] Refactor: retire the PTO2 prefix on the wire task structs `PTO2TaskDescriptor` -> `TaskDescriptor`, `PTO2TaskPayload` -> `TaskPayload`, `PTO2PreparedTask` -> `PreparedTask`, and the seven `PTO2_TASKPAYLOAD_*` field offsets lose the prefix with them. All plain strips: no declaration anywhere in the tree, in the CANN headers or in pto-isa holds any of these names. 561 occurrences across 81 files. The offset macros describe the layout of the payload the AICore reads, so they belong to `TaskPayload` and move with it rather than in a commit of their own. The shared-memory layout diagrams in `runtime/shared_memory.h` and `docs/RUNTIME_LOGIC.md` already write `TaskDescriptor[]` and `TaskPayload[]`, so two of their three rows become literally true here. They are corrected once, in the commit that renames `SharedMemoryHeader`, when every name in the box is a name that exists. Co-Authored-By: Claude Opus 5 (1M context) --- docs/buffer-abi.md | 4 +- ...026-07-aicore-fills-all-args-ready-path.md | 4 +- src/a2a3/docs/runtimes.md | 4 +- .../aicore/aicore_executor.cpp | 13 ++- .../host_build_graph/docs/RUNTIME_LOGIC.md | 4 +- .../docs/SUBMIT_BY_CLUSTER.md | 4 +- .../runtime/dispatch_payload.h | 18 ++-- .../orchestrator_core/orchestrator.cpp | 34 ++++---- .../host_build_graph/runtime/runtime_types.h | 40 ++++----- .../runtime/scheduler/scheduler.h | 14 +-- .../runtime/scheduler/scheduler_dispatch.cpp | 12 +-- .../runtime/shared/shared_memory.cpp | 8 +- .../host_build_graph/runtime/shared_memory.h | 32 ++++--- .../runtime/host_build_graph/runtime/types.h | 4 +- .../aicore/aicore_executor.cpp | 14 ++- .../docs/MULTI_RING.md | 16 ++-- .../docs/RUNTIME_LOGIC.md | 14 +-- .../docs/SUBMIT_BY_CLUSTER.md | 6 +- .../runtime/dispatch_payload.h | 18 ++-- .../runtime/orchestrator.cpp | 24 +++--- .../runtime/ring_buffer.cpp | 2 +- .../runtime/ring_buffer.h | 8 +- .../runtime/runtime_types.h | 36 ++++---- .../runtime/scheduler/scheduler.h | 14 +-- .../runtime/scheduler/scheduler_cold_path.cpp | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 12 +-- .../runtime/shared/shared_memory.cpp | 8 +- .../runtime/shared_memory.h | 26 +++--- .../tensormap_and_ringbuffer/runtime/types.h | 2 +- src/a5/docs/runtimes.md | 2 +- .../aicore/aicore_executor.cpp | 13 ++- .../host_build_graph/docs/RUNTIME_LOGIC.md | 4 +- .../docs/SUBMIT_BY_CLUSTER.md | 4 +- .../runtime/dispatch_payload.h | 18 ++-- .../orchestrator_core/orchestrator.cpp | 34 ++++---- .../host_build_graph/runtime/runtime_types.h | 40 ++++----- .../runtime/scheduler/scheduler.h | 14 +-- .../runtime/scheduler/scheduler_dispatch.cpp | 12 +-- .../runtime/shared/shared_memory.cpp | 8 +- .../host_build_graph/runtime/shared_memory.h | 32 ++++--- .../runtime/host_build_graph/runtime/types.h | 4 +- .../aicore/aicore_executor.cpp | 14 ++- .../docs/MULTI_RING.md | 16 ++-- .../docs/RUNTIME_LOGIC.md | 14 +-- .../docs/SUBMIT_BY_CLUSTER.md | 6 +- .../runtime/dispatch_payload.h | 18 ++-- .../runtime/orchestrator.cpp | 22 ++--- .../runtime/ring_buffer.cpp | 2 +- .../runtime/ring_buffer.h | 8 +- .../runtime/runtime_types.h | 36 ++++---- .../runtime/scheduler/scheduler.h | 14 +-- .../runtime/scheduler/scheduler_cold_path.cpp | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 12 +-- .../runtime/shared/shared_memory.cpp | 8 +- .../runtime/shared_memory.h | 26 +++--- .../tensormap_and_ringbuffer/runtime/types.h | 2 +- .../host_build_graph/graph_execution.cpp | 10 +-- src/common/host_build_graph/graph_execution.h | 4 +- .../platform/include/common/args_dump.h | 2 +- src/common/task_interface/data_type.h | 2 +- src/common/task_interface/tensor.h | 2 +- tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp | 16 ++-- tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp | 2 +- tests/ut/cpp/a2a3/test_scheduler_state.cpp | 8 +- tests/ut/cpp/a2a3/test_shared_memory.cpp | 4 +- tests/ut/cpp/a2a3/test_task_allocator.cpp | 8 +- tests/ut/cpp/a2a3/test_task_state.cpp | 4 +- tests/ut/cpp/a2a3/test_task_timing_slots.cpp | 4 +- tests/ut/cpp/a2a3/test_wiring.cpp | 86 +++++++++---------- tests/ut/cpp/a5/test_hbg_submit_poison.cpp | 16 ++-- tests/ut/cpp/a5/test_orchestrator_fanin.cpp | 2 +- tests/ut/cpp/a5/test_scheduler_state.cpp | 8 +- tests/ut/cpp/a5/test_shared_memory.cpp | 4 +- tests/ut/cpp/a5/test_task_allocator.cpp | 8 +- tests/ut/cpp/a5/test_task_state.cpp | 4 +- tests/ut/cpp/a5/test_wiring.cpp | 48 +++++------ tests/ut/cpp/common/test_hbg_graph_cache.cpp | 10 +-- .../common/test_hbg_graph_submit_failure.cpp | 6 +- .../cpp/common/test_hbg_self_relative_ptr.cpp | 22 ++--- .../ut/cpp/common/test_hbg_sm_compaction.cpp | 30 +++---- .../common/test_scope_deadlock_detection.cpp | 6 +- 81 files changed, 535 insertions(+), 563 deletions(-) diff --git a/docs/buffer-abi.md b/docs/buffer-abi.md index 3942fdc1b7..a0329f1dbf 100644 --- a/docs/buffer-abi.md +++ b/docs/buffer-abi.md @@ -56,8 +56,8 @@ That is self-consistent, but it charges the device for host-side fields: - **Device cost.** `sizeof(ChipTensor)` is pinned at **128 B** by `static_assert(… == 128, "ChipTensor must be exactly 2 cache lines (128 bytes)")` and by - `PTO2_TASKPAYLOAD_TENSOR_STRIDE`, which the AICPU scheduler strides the payload - with. A merged struct is ~192–216 B, taking `PTO2TaskPayload` from 4864 B to + `TASKPAYLOAD_TENSOR_STRIDE`, which the AICPU scheduler strides the payload + with. A merged struct is ~192–216 B, taking `TaskPayload` from 4864 B to 6912–7680 B — **+42% to +58% per task slot** — because the payload embeds `ChipTensor tensors[MAX_TENSOR_ARGS]` **by value** in the device task ring. - **Zero return for that cost.** Most of what a merge would add has no reader on diff --git a/docs/investigations/2026-07-aicore-fills-all-args-ready-path.md b/docs/investigations/2026-07-aicore-fills-all-args-ready-path.md index 25077fbbd2..68c725f39d 100644 --- a/docs/investigations/2026-07-aicore-fills-all-args-ready-path.md +++ b/docs/investigations/2026-07-aicore-fills-all-args-ready-path.md @@ -29,7 +29,7 @@ An experiment build (not merged) that: filling from source, `src_payload` is always non-zero and can no longer double as the gate flag (the shipped design folds `not_ready` into `src_payload == 0`). -- AICPU `build_payload` writes only `src_payload = &PTO2TaskPayload` and +- AICPU `build_payload` writes only `src_payload = &TaskPayload` and `gated`, and **never** fills `args[]`. - AICore `aicore_executor` fills `args[0..num_args)` from `src_payload` for **every** task (moved out of the gate branch), then gates only when @@ -51,7 +51,7 @@ tasks, a2a3 silicon) and read each task's AICore setup — Correctness held (`paged_attention_unroll` passes), so the AICore fill is functionally equivalent — it just costs ~1 µs. The fill is AICore GM work: after the per-task whole-cache `dcci(ENTIRE_DATA_CACHE)`, the first reads of -the source `PTO2TaskPayload` (counts, scalars) miss to HBM, then the loop +the source `TaskPayload` (counts, scalars) miss to HBM, then the loop computes `&tensors[i]` and writes `args[]` — all on a core whose GM-access latency is high. On the **ready** path there is no doorbell wait to overlap it with, so the whole ~1 µs lands on the `receive → start` setup. diff --git a/src/a2a3/docs/runtimes.md b/src/a2a3/docs/runtimes.md index 986cf98aef..85941234e1 100644 --- a/src/a2a3/docs/runtimes.md +++ b/src/a2a3/docs/runtimes.md @@ -7,7 +7,7 @@ Two runtime implementations live under `src/a2a3/runtime/`, each providing a dif | Feature | host_build_graph | tensormap_and_ringbuffer | | ------- | ---------------- | ------------------------ | | Graph built on | Host CPU | AICPU (device) | -| Task storage | Fixed `Task[]` array | Ring buffer (`PTO2TaskDescriptor[]`) | +| Task storage | Fixed `Task[]` array | Ring buffer (`TaskDescriptor[]`) | | Dependencies | Explicit edges | Auto-derived via TensorMap | | Memory management | Host-side | Ring buffer heap (GM) | | Concurrent build+schedule | No | Yes (always) | @@ -30,7 +30,7 @@ See [host_build_graph/docs/RUNTIME_LOGIC.md](../runtime/host_build_graph/docs/RU The primary production runtime. Uses ring buffers for task slots and output memory, with a TensorMap for automatic dependency tracking. -- Task storage: `PTO2TaskDescriptor[]` in shared memory ring buffer +- Task storage: `TaskDescriptor[]` in shared memory ring buffer - Memory: GM Heap ring for output buffer allocation - Dependencies: automatically derived from tensor read/write patterns via TensorMap - Thread model: 3 scheduler threads + 1 orchestrator thread on AICPU diff --git a/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp b/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp index f69ed3c82b..61da824b68 100644 --- a/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp +++ b/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp @@ -181,7 +181,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // execute_task() below — strictly AFTER this gate — so predecessor // outputs are visible. src_payload == 0 (the common path) skips this; // a non-zero src_payload is both the gate flag and the source - // PTO2TaskPayload. + // TaskPayload. if (exec_payload->src_payload != 0) { // AICPU staged only src_payload, not the arg vector — fill // args[0..num_args) ourselves now, while we are idle waiting for @@ -192,20 +192,19 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // pre-filled once at init() and never changed; tensor and scalar // counts are set by the orchestrator, so num_args never reaches them. __gm__ char *src = reinterpret_cast<__gm__ char *>(exec_payload->src_payload); - int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET); - int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET); + int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_TENSOR_COUNT_OFFSET); + int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_SCALAR_COUNT_OFFSET); // Each region is named by an int32 delta from the naming field's own // address, so resolve the field, then add what it holds. - __gm__ char *tensors_field = src + PTO2_TASKPAYLOAD_TENSORS_DELTA_OFFSET; + __gm__ char *tensors_field = src + TASKPAYLOAD_TENSORS_DELTA_OFFSET; __gm__ char *src_tensors = tensors_field + *reinterpret_cast<__gm__ int32_t *>(tensors_field); - __gm__ char *scalars_field = src + PTO2_TASKPAYLOAD_SCALARS_DELTA_OFFSET; + __gm__ char *scalars_field = src + TASKPAYLOAD_SCALARS_DELTA_OFFSET; __gm__ uint64_t *src_scalars = reinterpret_cast<__gm__ uint64_t *>( scalars_field + *reinterpret_cast<__gm__ int32_t *>(scalars_field) ); int n = 0; for (int32_t i = 0; i < tensor_count; i++) { - exec_payload->args[n++] = - reinterpret_cast(src_tensors + i * PTO2_TASKPAYLOAD_TENSOR_STRIDE); + exec_payload->args[n++] = reinterpret_cast(src_tensors + i * TASKPAYLOAD_TENSOR_STRIDE); } for (int32_t i = 0; i < scalar_count; i++) { exec_payload->args[n++] = src_scalars[i]; diff --git a/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md index a52ca0e20d..c8ebfd16d3 100644 --- a/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md @@ -72,8 +72,8 @@ The shared image uses three per-slot structures: | Structure | Purpose | | --------- | ------- | -| `PTO2TaskDescriptor` | Full task ID, kernel IDs, packed-buffer addresses | -| `PTO2TaskPayload` | Argument counts, predicate, dispatch metadata, and a delta naming each of its tensor, scalar and fanin regions — the arguments themselves live in the pool segments, so the payload is a fixed three cache lines regardless of the argument caps | +| `TaskDescriptor` | Full task ID, kernel IDs, packed-buffer addresses | +| `TaskPayload` | Argument counts, predicate, dispatch metadata, and a delta naming each of its tensor, scalar and fanin regions — the arguments themselves live in the pool segments, so the payload is a fixed three cache lines regardless of the argument caps | | `ChipTaskSlotState` | Active mask, attributes, block/subtask counters, completion state, task/payload bindings | The host/device boundary is POD and position-independent. Fanins are integer diff --git a/src/a2a3/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md b/src/a2a3/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md index e5b98f7a5d..0839f42e9f 100644 --- a/src/a2a3/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a2a3/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md @@ -26,9 +26,9 @@ same mixed-task contract. The shared graph image separates stable task identity from scheduling state: -- `PTO2TaskDescriptor` contains the task ID, kernel IDs, and packed-buffer +- `TaskDescriptor` contains the task ID, kernel IDs, and packed-buffer addresses. -- `PTO2TaskPayload` contains the argument counts, predicate, and a delta naming +- `TaskPayload` contains the argument counts, predicate, and a delta naming each of its tensor, scalar and position-independent fanin-id regions. The arguments live in the pool segments, not in the payload. - `ChipTaskSlotState` contains the active mask, task attributes, logical block diff --git a/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h b/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h index 400bc202f9..554e797d17 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h +++ b/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h @@ -59,22 +59,22 @@ static_assert( "GLOBAL_CONTEXT_INDEX out of sync with intrinsic.h" ); -// Byte offsets into PTO2TaskPayload used by AICore to materialize args[] on the +// Byte offsets into TaskPayload used by AICore to materialize args[] on the // not_ready (early-dispatch) path. The AICore .o does not include // runtime_types.h, so it reads tensor_count / scalar_count / tensors[] / // scalars[] through these constants. The AICPU side (scheduler_dispatch.cpp) // static_asserts them against offsetof where the full struct is visible, so any -// PTO2TaskPayload layout drift fails the build rather than corrupting args[]. -constexpr uint32_t PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET = 0; -constexpr uint32_t PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET = 4; +// TaskPayload layout drift fails the build rather than corrupting args[]. +constexpr uint32_t TASKPAYLOAD_TENSOR_COUNT_OFFSET = 0; +constexpr uint32_t TASKPAYLOAD_SCALAR_COUNT_OFFSET = 4; // Offsets of the fields that NAME the argument regions, not of the regions themselves: // a region's address is (src + ) + . -constexpr uint32_t PTO2_TASKPAYLOAD_TENSORS_DELTA_OFFSET = 12; -constexpr uint32_t PTO2_TASKPAYLOAD_SCALARS_DELTA_OFFSET = 16; +constexpr uint32_t TASKPAYLOAD_TENSORS_DELTA_OFFSET = 12; +constexpr uint32_t TASKPAYLOAD_SCALARS_DELTA_OFFSET = 16; // Cache line 9 (byte 576) holds the AICPU-only DispatchPredicate. Scalars follow it, // then tensors — the tensor array is last because it is the only region whose used // extent varies per task, so a task's read set is a contiguous prefix. -constexpr uint32_t PTO2_TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) +constexpr uint32_t TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) /** * Per-core dispatch payload: function address + args[] + SPMD context. @@ -93,7 +93,7 @@ struct alignas(64) PTO2DispatchPayload { // capacity are cold (prefilled once at init) but ride this hot line for free. // Sized to exactly 64B so both dispatch paths write one control line: the // ready path (src_payload = 0) then also fills args[0..num_args); the gated - // path (src_payload = &PTO2TaskPayload) leaves args[] to the idle AICore. + // path (src_payload = &TaskPayload) leaves args[] to the idle AICore. uint64_t function_bin_addr; /**< Kernel entry address in GM (set by Scheduler). */ /** Per-dispatch context: block_idx/block_num (hot) + async_ctx (task_token hot, @@ -103,7 +103,7 @@ struct alignas(64) PTO2DispatchPayload { /** Early-dispatch gate AND source pointer, folded into one field. 0 = ready: * AICore executes on pickup (args[] already filled by the AICPU). Non-zero = - * gated: the value is the source PTO2TaskPayload address; the AICore fills + * gated: the value is the source TaskPayload address; the AICore fills * args[0..num_args) from it, then waits for the doorbell (DATA_MAIN_BASE * high 32 == this dispatch's reg_task_id) before executing. A payload address * is never 0, so the gate flag is lossless. */ diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 9e72bb384b..a028e1fda3 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1141,7 +1141,7 @@ static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { // (the consumer), used to bump each producer's last_consumer_local_id (the // reclaim gate the host wait_for_consumers polls via completed_watermark). struct PTO2FaninBuilder { - PTO2FaninBuilder(PTO2OrchestratorState *orch, PTO2TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : + PTO2FaninBuilder(PTO2OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : count(0), orch(orch), seen_epoch(seen_epoch), @@ -1152,7 +1152,7 @@ struct PTO2FaninBuilder { PTO2OrchestratorState *orch{nullptr}; uint32_t seen_epoch{0}; int32_t self_local{0}; - PTO2TaskPayload *payload{nullptr}; + TaskPayload *payload{nullptr}; // The payload's fanin region, resolved once: the appends below would otherwise // re-resolve the delta per producer. Requires the regions bound before construction. int32_t *slots{nullptr}; @@ -1216,11 +1216,11 @@ static bool append_fanin_or_fail( return true; } -struct PTO2PreparedTask { +struct PreparedTask { TaskId task_id = TaskId::invalid(); PTO2TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; - PTO2TaskDescriptor *task = nullptr; - PTO2TaskPayload *payload = nullptr; + TaskDescriptor *task = nullptr; + TaskPayload *payload = nullptr; ChipTaskSlotState *slot_state = nullptr; }; @@ -1240,7 +1240,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { static bool prepare_task( PTO2OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, - TaskAttrs task_attrs, PTO2PreparedTask *out + TaskAttrs task_attrs, PreparedTask *out ) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit task outside a scope"); uint8_t ring_id = 0; @@ -1308,7 +1308,7 @@ static bool prepare_task( out->slot_state->bind_buffers(out->payload, out->task); // prepare_task does NO payload writes: all payload content (tensors/scalars + - // early-dispatch fields) is initialized in PTO2TaskPayload::init, the + // early-dispatch fields) is initialized in TaskPayload::init, the // single payload-init point, which runs before Orch-side wiring publish. // Fields already zeroed by the reset_for_reuse() above: @@ -1456,14 +1456,14 @@ static TaskOutputTensors submit_task_common( ORCH_PHASE_START(); TaskOutputTensors result; PTO2OutputLayout layout = calculate_output_layout(args); - PTO2PreparedTask prepared; + PreparedTask prepared; if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; } PTO2SchedulerState *sched = orch->scheduler; TaskId task_id = prepared.task_id; - PTO2TaskDescriptor &task = *prepared.task; - PTO2TaskPayload &payload = *prepared.payload; + TaskDescriptor &task = *prepared.task; + TaskPayload &payload = *prepared.payload; result.set_task_id(task_id); // dep_gen capture point: open this task's graph entry before its dependency @@ -1734,7 +1734,7 @@ bool graph_boundary_matches(const GraphBoundary &boundary, const GraphTaskArgs & return true; } -void graph_reset_outer_payload(PTO2TaskPayload &payload) { +void graph_reset_outer_payload(TaskPayload &payload) { payload.tensor_count = 0; payload.scalar_count = 0; payload.fanin_count = 0; @@ -1797,8 +1797,8 @@ bool graph_submit_outer( } const TaskId task_id = TaskId::make(0, static_cast(allocation.task_id)); PTO2SharedMemoryRingHeader &ring = orch->sm_header->ring; - PTO2TaskDescriptor &task = ring.task_descriptors[allocation.slot]; - PTO2TaskPayload &payload = ring.task_payloads[allocation.slot]; + TaskDescriptor &task = ring.task_descriptors[allocation.slot]; + TaskPayload &payload = ring.task_payloads[allocation.slot]; ChipTaskSlotState &slot = ring.get_slot_state_by_slot(allocation.slot); // Init-on-write, as in prepare_task: this slot's dynamic scheduling fields and @@ -2025,7 +2025,7 @@ TaskOutputTensors graph_record_submit_node( node.record_packed_base = packed_base_addr; node.total_output_size = aligned_output; - // Build the tensor list exactly as PTO2TaskPayload::init: inputs/inouts copy + // Build the tensor list exactly as TaskPayload::init: inputs/inouts copy // the caller's ChipTensor; outputs materialize from the create-info onto the // scratch buffer and carry this node's owner id. const int32_t tensor_count = args.tensor_count(); @@ -2694,15 +2694,15 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) } PTO2OutputLayout layout = calculate_output_layout(args); - PTO2PreparedTask prepared; + PreparedTask prepared; // Kernel-less alloc task: no active subtasks, no dispatch-time attributes. The // early-dispatch hint is force-set below (see the flag-the-creator note). if (!prepare_task(orch, args, layout.total_output_size, ActiveMask{}, TaskAttrs{}, &prepared)) { return TaskOutputTensors{}; } - PTO2TaskDescriptor &task = *prepared.task; - PTO2TaskPayload &payload = *prepared.payload; + TaskDescriptor &task = *prepared.task; + TaskPayload &payload = *prepared.payload; CYCLE_COUNT_LAP(g_orch_alloc_cycle); diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index d3d6c7d0c8..57a868583a 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -140,7 +140,7 @@ inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HE #define PTO2_MAX_FANIN 128 // Alignment of every per-task region inside an argument pool. Each region starts -// and ends on a cache line so PTO2TaskPayload::init's round-up scalar memcpy stays +// and ends on a cache line so TaskPayload::init's round-up scalar memcpy stays // inside the task's own region — see its comment. ChipTensor is already 2 cache // lines, so only the fanin and scalar regions need the round-up. inline constexpr int32_t ARG_POOL_ALIGN = 64; @@ -232,7 +232,7 @@ struct ChipTaskSlotState; // Forward declaration (defined below) * * Fields set by Orchestrator at submission, read by Scheduler for dispatch. */ -struct PTO2TaskDescriptor { +struct TaskDescriptor { // Mixed-task identification (encodes ring_id in upper 32 bits) TaskId task_id; // raw: (ring_id << 32) | local_id @@ -246,8 +246,8 @@ struct PTO2TaskDescriptor { // A 4-byte alignment pad follows kernel_id[3]; the scheduler and shared-memory // ABI depend on the descriptor size and packed_buffer_base offset staying fixed. -static_assert(sizeof(PTO2TaskDescriptor) == 40, "PTO2TaskDescriptor size is part of the shared-memory ABI"); -static_assert(offsetof(PTO2TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged"); +static_assert(sizeof(TaskDescriptor) == 40, "TaskDescriptor size is part of the shared-memory ABI"); +static_assert(offsetof(TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged"); // ============================================================================= // Per-Slot Scheduling State @@ -260,7 +260,7 @@ static_assert(offsetof(PTO2TaskDescriptor, packed_buffer_base) == 24, "packed_bu * by bulk tensor and scalar data. Small fanins stay fully inline; larger * fanins spill into a per-ring ring buffer slice. */ -// Early-dispatch claim states for PTO2TaskPayload::early_dispatch_state. +// Early-dispatch claim states for TaskPayload::early_dispatch_state. enum PTO2EarlyDispatchState : uint8_t { PTO2_EARLY_DISPATCH_NONE = 0, // not pre-staged PTO2_EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress @@ -291,7 +291,7 @@ enum PTO2EarlySyncDrainState : uint8_t { // consumer can pre-stage all its idle cores. 2 words = 128 bits >= 72. inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; -struct PTO2TaskPayload { +struct TaskPayload { // === Cache line 0 (64B) — the dispatch path's own line === // sizeof is independent of PTO2_MAX_FANIN / MAX_TENSOR_ARGS / MAX_SCALAR_ARGS: // widening a cap costs pool bytes for the tasks that need them, not a control @@ -499,34 +499,32 @@ struct PTO2TaskPayload { } }; -// PTO2TaskPayload layout verification (offsetof requires complete type). The counts +// TaskPayload layout verification (offsetof requires complete type). The counts // and region deltas share the first cache line, the early-dispatch atomics own the // second, and the AICPU-only predicate + dump metadata own the third. -static_assert(offsetof(PTO2TaskPayload, tensors) == 12, "region deltas must follow the three counts"); +static_assert(offsetof(TaskPayload, tensors) == 12, "region deltas must follow the three counts"); static_assert( - offsetof(PTO2TaskPayload, fanin) + sizeof(simpler::hbg::SelfRelativePtr) <= 64, + offsetof(TaskPayload, fanin) + sizeof(simpler::hbg::SelfRelativePtr) <= 64, "counts + region deltas must fit the first cache line" ); static_assert( - offsetof(PTO2TaskPayload, staged_core_mask) == 64, + offsetof(TaskPayload, staged_core_mask) == 64, "the early-dispatch atomics own cache line 1: they are written during staging while " "line 0's counts and deltas are read at dispatch, so sharing a line would false-share" ); -static_assert(offsetof(PTO2TaskPayload, predicate) == 128, "dispatch predicate owns cache line 2"); +static_assert(offsetof(TaskPayload, predicate) == 128, "dispatch predicate owns cache line 2"); static_assert( - offsetof(PTO2TaskPayload, dump_metadata) + sizeof(ArgsDumpTaskMetadata) <= 192, + offsetof(TaskPayload, dump_metadata) + sizeof(ArgsDumpTaskMetadata) <= 192, "dump metadata must fit the predicate's cache line" ); -static_assert( - sizeof(PTO2TaskPayload) == 192, "PTO2TaskPayload is three cache lines and independent of every argument cap" -); +static_assert(sizeof(TaskPayload) == 192, "TaskPayload is three cache lines and independent of every argument cap"); // compact_live_image restacks the payload segment with one memcpy and the device // copy moves the whole image, so the payload has to be a POD wire struct. Deleting // SelfRelativePtr's copy operations does not cost it that — a deleted special member // is trivial — but only an assertion keeps a future member from doing so silently. static_assert( - std::is_trivially_copyable_v && std::is_standard_layout_v, - "PTO2TaskPayload crosses to the device by memcpy" + std::is_trivially_copyable_v && std::is_standard_layout_v, + "TaskPayload crosses to the device by memcpy" ); static_assert(sizeof(ChipTensor) == 128, "ChipTensor must be 2 cache lines"); @@ -560,8 +558,8 @@ struct alignas(64) ChipTaskSlotState { // --- Per-slot constant, re-bound by orch::prepare_task each submit --- // Self-relative, so the SM image needs no pointer fix-up on its way to the // device: both targets sit in the same block as this field. - simpler::hbg::SelfRelativePtr payload; - simpler::hbg::SelfRelativePtr task; + simpler::hbg::SelfRelativePtr payload; + simpler::hbg::SelfRelativePtr task; // --- Wake list: last-fanin notification (intrusive, lock-free) --- // A pending consumer whose fanin scan finds an unmet producer registers on @@ -619,7 +617,7 @@ struct alignas(64) ChipTaskSlotState { return 0; } - void bind_buffers(PTO2TaskPayload *p, PTO2TaskDescriptor *t) { + void bind_buffers(TaskPayload *p, TaskDescriptor *t) { payload.set(p); task.set(t); } @@ -655,7 +653,7 @@ struct alignas(64) ChipTaskSlotState { // rewritten in prepare_task on every reuse, so they are not reset here. // last_consumer_local_id is seeded in prepare_task once the id is known. // Payload early-dispatch/fanin fields are (re)initialized in - // PTO2TaskPayload::init on every submit, before the slot is visible. + // TaskPayload::init on every submit, before the slot is visible. } }; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h index 720c0c63bf..5ff913e8f9 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -585,7 +585,7 @@ struct PTO2SchedulerState { // lists. The decision is terminal: tasks are never re-polled; a producer's // completion re-scans its waiters via on_mixed_task_complete's wake drain. int classify_fanin_state(const ChipTaskSlotState *s) const { - const PTO2TaskPayload &p = *s->payload; + const TaskPayload &p = *s->payload; const PTO2SharedMemoryRingHeader &ring = *ring_sched_state.ring; const int32_t *fanin = p.fanin_data(); for (int32_t i = p.fanin_count - 1; i >= 0; i--) { @@ -757,7 +757,7 @@ struct PTO2SchedulerState { return true; } - static inline bool try_claim_early_dispatch_launch(PTO2TaskPayload &payload) { + static inline bool try_claim_early_dispatch_launch(TaskPayload &payload) { uint8_t expected = PTO2_EARLY_DISPATCH_LAUNCH_NONE; return payload.early_dispatch_launch_state.compare_exchange_strong( expected, PTO2_EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst @@ -785,22 +785,22 @@ struct PTO2SchedulerState { } } - static inline bool try_claim_early_sync_drain(PTO2TaskPayload &payload) { + static inline bool try_claim_early_sync_drain(TaskPayload &payload) { uint8_t expected = PTO2_EARLY_SYNC_DRAIN_NONE; return payload.early_sync_drain_state.compare_exchange_strong( expected, PTO2_EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst ); } - static inline bool owns_early_sync_drain(const PTO2TaskPayload &payload) { + static inline bool owns_early_sync_drain(const TaskPayload &payload) { return (payload.early_sync_drain_state.load(std::memory_order_acquire) & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; } - static inline void mark_early_sync_drain_armed(PTO2TaskPayload &payload) { + static inline void mark_early_sync_drain_armed(TaskPayload &payload) { payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); } - static inline bool publish_ready_to_early_sync_drain(PTO2TaskPayload &payload) { + static inline bool publish_ready_to_early_sync_drain(TaskPayload &payload) { uint8_t previous = payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); return (previous & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; @@ -819,7 +819,7 @@ struct PTO2SchedulerState { } } - static inline void finish_early_sync_drain(PTO2TaskPayload &payload) { + static inline void finish_early_sync_drain(TaskPayload &payload) { uint8_t state = payload.early_sync_drain_state.load(std::memory_order_seq_cst); while ((state & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0 && (state & PTO2_EARLY_SYNC_DRAIN_COMPLETE) == 0) { uint8_t desired = state | PTO2_EARLY_SYNC_DRAIN_COMPLETE; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 632f4b88e4..e6fcdcfdbf 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -38,13 +38,13 @@ #endif // AICore materializes args[] from src_payload on the gated path using the -// byte offsets in dispatch_payload.h (the AICore .o cannot see PTO2TaskPayload). +// byte offsets in dispatch_payload.h (the AICore .o cannot see TaskPayload). // Pin those constants to the real layout here, where the struct is fully visible. -static_assert(offsetof(PTO2TaskPayload, tensor_count) == PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET); -static_assert(offsetof(PTO2TaskPayload, scalar_count) == PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET); -static_assert(offsetof(PTO2TaskPayload, tensors) == PTO2_TASKPAYLOAD_TENSORS_DELTA_OFFSET); -static_assert(offsetof(PTO2TaskPayload, scalars) == PTO2_TASKPAYLOAD_SCALARS_DELTA_OFFSET); -static_assert(sizeof(ChipTensor) == PTO2_TASKPAYLOAD_TENSOR_STRIDE); +static_assert(offsetof(TaskPayload, tensor_count) == TASKPAYLOAD_TENSOR_COUNT_OFFSET); +static_assert(offsetof(TaskPayload, scalar_count) == TASKPAYLOAD_SCALAR_COUNT_OFFSET); +static_assert(offsetof(TaskPayload, tensors) == TASKPAYLOAD_TENSORS_DELTA_OFFSET); +static_assert(offsetof(TaskPayload, scalars) == TASKPAYLOAD_SCALARS_DELTA_OFFSET); +static_assert(sizeof(ChipTensor) == TASKPAYLOAD_TENSOR_STRIDE); // ============================================================================= // Dispatch helpers diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp index f8d03e6749..8e098d78f2 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -51,8 +51,8 @@ void PTO2SharedMemoryHandle::setup_pointers(uint64_t pitch) { // argument region through the payload's delta and needs no base at all. auto off = pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents({pitch, 0, 0, 0})); auto &ring = header->ring; - ring.task_descriptors = (PTO2TaskDescriptor *)(base + off.descriptors); - ring.task_payloads = (PTO2TaskPayload *)(base + off.payloads); + ring.task_descriptors = (TaskDescriptor *)(base + off.descriptors); + ring.task_payloads = (TaskPayload *)(base + off.payloads); ring.slot_states = (ChipTaskSlotState *)(base + off.slot_states); ring.completion_flags = (std::atomic *)(base + off.completion_flags); } @@ -153,8 +153,8 @@ void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size) { header->ring.task_window_size = task_window_size; header->ring.task_window_mask = static_cast(task_window_size - 1); header->ring.task_descriptors_offset = offset; - offset += PTO2_ALIGN_UP(task_window_size * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_size * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + offset += PTO2_ALIGN_UP(task_window_size * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + offset += PTO2_ALIGN_UP(task_window_size * sizeof(TaskPayload), PTO2_ALIGN_SIZE); offset += PTO2_ALIGN_UP(task_window_size * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); header->total_size = sm_size; diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h index deaa7e136e..32582ad0aa 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h @@ -92,8 +92,8 @@ struct alignas(64) PTO2SharedMemoryRingHeader { uint64_t task_descriptors_offset; // Offset from SM base, in bytes // Per-ring data pointers (host-side, set by setup_pointers) - PTO2TaskDescriptor *task_descriptors; - PTO2TaskPayload *task_payloads; + TaskDescriptor *task_descriptors; + TaskPayload *task_payloads; ChipTaskSlotState *slot_states; // Polling-completion state (device-addressed array, one byte per slot). @@ -139,11 +139,9 @@ struct alignas(64) PTO2SharedMemoryRingHeader { int32_t get_slot_by_task_id(int32_t local_task_id) { return local_task_id & task_window_mask; } - PTO2TaskDescriptor &get_task_by_slot(int32_t slot) { return task_descriptors[slot]; } + TaskDescriptor &get_task_by_slot(int32_t slot) { return task_descriptors[slot]; } - PTO2TaskDescriptor &get_task_by_task_id(int32_t local_id) { - return task_descriptors[get_slot_by_task_id(local_id)]; - } + TaskDescriptor &get_task_by_task_id(int32_t local_id) { return task_descriptors[get_slot_by_task_id(local_id)]; } // No get_payload_by_slot / get_payload_by_task_id here: a payload is reached // through its slot state's `payload` delta, which the image's restack rebinds to @@ -350,9 +348,9 @@ inline PTO2RingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) no uint64_t off = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); PTO2RingSegmentOffsets o{}; o.descriptors = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(e.slots * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); o.payloads = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(e.slots * sizeof(TaskPayload), PTO2_ALIGN_SIZE); o.slot_states = off; off += PTO2_ALIGN_UP(e.slots * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); o.completion_flags = off; @@ -371,7 +369,7 @@ inline PTO2RingSegmentOffsets ring_segment_offsets(uint64_t task_window_size) no return ring_segment_offsets(mirror_extents(task_window_size)); } -// Every per-task region starts on a cache line, which PTO2TaskPayload::init's +// Every per-task region starts on a cache line, which TaskPayload::init's // round-up scalar memcpy relies on. ChipTensor is 2 cache lines, so a tensor region // is aligned for any count; the fanin and scalar strides need it stated. static_assert( @@ -490,10 +488,10 @@ inline uint64_t compact_live_image( out_ring.completion_flags = nullptr; const uint64_t nt = used.submitted_tasks; - std::memcpy(out_base + to.descriptors, mirror_base + from.descriptors, nt * sizeof(PTO2TaskDescriptor)); - // One copy, not one per payload: PTO2TaskPayload is fixed-size, so the mirror and + std::memcpy(out_base + to.descriptors, mirror_base + from.descriptors, nt * sizeof(TaskDescriptor)); + // One copy, not one per payload: TaskPayload is fixed-size, so the mirror and // the image share a stride. Each pool is likewise one copy of its own prefix. - std::memcpy(out_base + to.payloads, mirror_base + from.payloads, nt * sizeof(PTO2TaskPayload)); + std::memcpy(out_base + to.payloads, mirror_base + from.payloads, nt * sizeof(TaskPayload)); std::memcpy(out_base + to.slot_states, mirror_base + from.slot_states, nt * sizeof(ChipTaskSlotState)); std::memcpy(out_base + to.completion_flags, mirror_base + from.completion_flags, nt * sizeof(std::atomic)); std::memcpy(out_base + to.fanin_pool, mirror_base + from.fanin_pool, used.fanin_elems * sizeof(int32_t)); @@ -501,9 +499,9 @@ inline uint64_t compact_live_image( std::memcpy(out_base + to.scalar_pool, mirror_base + from.scalar_pool, used.scalar_elems * sizeof(uint64_t)); auto *out_slots = reinterpret_cast(out_base + to.slot_states); - auto *out_descriptors = reinterpret_cast(out_base + to.descriptors); - auto *out_payloads = reinterpret_cast(out_base + to.payloads); - const auto *mirror_payloads = reinterpret_cast(mirror_base + from.payloads); + auto *out_descriptors = reinterpret_cast(out_base + to.descriptors); + auto *out_payloads = reinterpret_cast(out_base + to.payloads); + const auto *mirror_payloads = reinterpret_cast(mirror_base + from.payloads); auto *out_fanin = reinterpret_cast(out_base + to.fanin_pool); auto *out_tensors = reinterpret_cast(out_base + to.tensor_pool); auto *out_scalars = reinterpret_cast(out_base + to.scalar_pool); @@ -517,7 +515,7 @@ inline uint64_t compact_live_image( // re-derived. for (uint64_t i = 0; i < nt; ++i) { out_slots[i].bind_buffers(&out_payloads[i], &out_descriptors[i]); - const PTO2TaskPayload &src = mirror_payloads[i]; + const TaskPayload &src = mirror_payloads[i]; const ChipTensor *src_tensors = src.tensor_data(); const uint64_t *src_scalars = src.scalar_data(); const int32_t *src_fanin = src.fanin_data(); @@ -554,7 +552,7 @@ inline uint64_t compact_live_image( tensors[j].buffer.addr = rebased_heap_addr(tensors[j].buffer.addr, rebase); } } - PTO2TaskDescriptor &out_task = out_descriptors[i]; + TaskDescriptor &out_task = out_descriptors[i]; out_task.packed_buffer_base = reinterpret_cast( rebased_heap_addr(reinterpret_cast(out_task.packed_buffer_base), rebase) ); diff --git a/src/a2a3/runtime/host_build_graph/runtime/types.h b/src/a2a3/runtime/host_build_graph/runtime/types.h index ad105719c4..e7ca055336 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/types.h @@ -86,7 +86,7 @@ enum class PTO2ScopeMode : uint8_t { * * LIFETIME — single-pass only: * Internally this class stores pointers into the submitting task's tensor - * storage: the region named by PTO2TaskPayload::tensors for a plain submit, the + * storage: the region named by TaskPayload::tensors for a plain submit, the * GraphRecording node's tensors for a submit inside a Graph body. Both belong to * one orchestration pass, which the next bind rebuilds over the same bytes. * Therefore the TaskOutputTensors instance, the const ChipTensor& returned by @@ -692,7 +692,7 @@ inline constexpr uint32_t GRAPH_MAX_SCALAR_ARGS = 64; // outer GRAPH payload carries the whole boundary, while materialize stages only // one node's arguments at a time. The compact boundary values live in that // payload's argument-pool regions, so widening these caps costs pool bytes only -// for Graphs that use them; PTO2TaskPayload itself stays fixed-size. +// for Graphs that use them; TaskPayload itself stays fixed-size. using GraphTaskArgs = Arg; // ChipTaskArgs — chip-level entry-arg holding the orchestration entry's diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp index 3063025933..a08f71df99 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp @@ -172,7 +172,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // behind it. The kernel's own input dcci runs inside execute_task() // below — strictly AFTER this gate — so predecessor outputs are visible. // src_payload == 0 (the common ready path) skips this; a non-zero - // src_payload is both the gate flag and the source PTO2TaskPayload. + // src_payload is both the gate flag and the source TaskPayload. if (exec_payload->src_payload != 0) { // AICPU staged only src_payload, not the arg vector — fill // args[0..num_args) ourselves now, while we are idle waiting for @@ -182,15 +182,13 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // args[SPMD_LOCAL_CONTEXT_INDEX]/[SPMD_GLOBAL_CONTEXT_INDEX] are // still written by the AICPU (num_args <= 48 never reaches them). __gm__ char *src = reinterpret_cast<__gm__ char *>(exec_payload->src_payload); - int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET); - int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET); - __gm__ uint64_t *src_scalars = - reinterpret_cast<__gm__ uint64_t *>(src + PTO2_TASKPAYLOAD_SCALARS_OFFSET); + int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_TENSOR_COUNT_OFFSET); + int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_SCALAR_COUNT_OFFSET); + __gm__ uint64_t *src_scalars = reinterpret_cast<__gm__ uint64_t *>(src + TASKPAYLOAD_SCALARS_OFFSET); int n = 0; for (int32_t i = 0; i < tensor_count; i++) { - exec_payload->args[n++] = reinterpret_cast( - src + PTO2_TASKPAYLOAD_TENSORS_OFFSET + i * PTO2_TASKPAYLOAD_TENSOR_STRIDE - ); + exec_payload->args[n++] = + reinterpret_cast(src + TASKPAYLOAD_TENSORS_OFFSET + i * TASKPAYLOAD_TENSOR_STRIDE); } for (int32_t i = 0; i < scalar_count; i++) { exec_payload->args[n++] = src_scalars[i]; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index a8fc081d51..55d6d0675e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -40,7 +40,7 @@ Type changes: | Field | Before | After | | ----- | ------ | ----- | -| `PTO2TaskDescriptor.task_id` | `int32_t` | `TaskId` | +| `TaskDescriptor.task_id` | `int32_t` | `TaskId` | | `PTO2TensorMapEntry.producer_task_id` | `int32_t` | `TaskId` | | `ChipTaskSlotState.ring_id` | N/A | `uint8_t` (new, denormalized for fast access) | @@ -94,15 +94,15 @@ struct alignas(64) PTO2SharedMemoryRingHeader { uint64_t task_descriptors_offset; // Per-ring data pointers (host-side, set by setup_pointers) - PTO2TaskDescriptor *task_descriptors; - PTO2TaskPayload *task_payloads; + TaskDescriptor *task_descriptors; + TaskPayload *task_payloads; ChipTaskSlotState *slot_states; // Accessors (slot = local_id & task_window_mask) - PTO2TaskDescriptor &get_task_by_slot(int32_t slot); - PTO2TaskDescriptor &get_task_by_task_id(int32_t local_id); - PTO2TaskPayload &get_payload_by_slot(int32_t slot); - PTO2TaskPayload &get_payload_by_task_id(int32_t local_id); + TaskDescriptor &get_task_by_slot(int32_t slot); + TaskDescriptor &get_task_by_task_id(int32_t local_id); + TaskPayload &get_payload_by_slot(int32_t slot); + TaskPayload &get_payload_by_task_id(int32_t local_id); ChipTaskSlotState &get_slot_state_by_slot(int32_t slot); ChipTaskSlotState &get_slot_state_by_task_id(int32_t local_id); }; @@ -166,7 +166,7 @@ bool entry_valid(const PTO2TensorMapEntry& e) { | Structure | Reason | | --------- | ------ | | `PTO2DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | -| `PTO2TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | +| `TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | | `PTO2ReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | | `PTO2DispatchPayload` | Built per-dispatch, no ring state needed | diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 1ed0687425..a2fcd1610e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -49,7 +49,7 @@ The host builds the complete task graph before launching device execution. The o The primary production runtime. Uses ring buffers for task slots and output memory, with a TensorMap for automatic dependency tracking. -- **Task storage**: `PTO2TaskDescriptor[]` in shared memory ring buffer +- **Task storage**: `TaskDescriptor[]` in shared memory ring buffer - **Memory**: GM Heap ring for output buffer allocation - **Dependencies**: automatically derived from tensor read/write patterns via TensorMap - **Thread model**: 3 scheduler threads + 1 orchestrator thread on AICPU @@ -148,8 +148,8 @@ The orchestrator and schedulers communicate through a contiguous shared memory r │ PTO2SharedMemoryHeader │ (per-ring flow control + layout, global flags) ├─────────────────────────────┤ aligned │ Per-ring regions ×4: │ -│ PTO2TaskDescriptor[N] │ N = task_window_size per ring -│ PTO2TaskPayload[N] │ +│ TaskDescriptor[N] │ N = task_window_size per ring +│ TaskPayload[N] │ │ ChipTaskSlotState[N] │ └─────────────────────────────┘ ``` @@ -368,7 +368,7 @@ When `PTO2OrchestratorState::submit_task` processes parameters: ## 6. Task Descriptor and States -### 6.1 PTO2TaskDescriptor (Hot Path) +### 6.1 TaskDescriptor (Hot Path) | Field | Description | | ----- | ----------- | @@ -383,7 +383,7 @@ When `PTO2OrchestratorState::submit_task` processes parameters: | `packed_buffer_base` | Start of packed buffer in GM Heap | | `packed_buffer_end` | End of packed buffer (for heap reclamation) | -### 6.1b PTO2TaskPayload (Cold Path) +### 6.1b TaskPayload (Cold Path) | Field | Description | | ----- | ----------- | @@ -483,7 +483,7 @@ SIMPLER_SCOPE(rt) { **Output tensor lifetime — single-scope only.** `submit_task` returns a `TaskOutputTensors`, and `get_ref(i)` hands back a `const Tensor&`. Both are -backed by pointers into the submitting task's `PTO2TaskPayload::tensors[]`, +backed by pointers into the submitting task's `TaskPayload::tensors[]`, which lives in a ring-buffer slot. After `scope_end` the slot becomes eligible for reuse; once `advance_ring_pointers` reaches it, `reset_for_reuse()` runs and the next `submit_task` overwrites the same @@ -750,7 +750,7 @@ Instead of polling a shared-memory status flag, the production protocol uses har ### 9.3 PTO2DispatchPayload -Built by the scheduler from `PTO2TaskDescriptor`: +Built by the scheduler from `TaskDescriptor`: | Field | Description | | ----- | ----------- | diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md index b8ffeb52e0..6ac0f2f027 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md @@ -37,7 +37,7 @@ Legacy per-task submit (`kernel_id + worker_type`) cannot express atomic co-disp Design must preserve the current main runtime architecture: 1. Executor threading split (orchestrator thread vs scheduler threads); the orchestrator thread exits after the task graph is built while scheduler threads dispatch to completion. -2. Shared-memory hot/cold split (`PTO2TaskDescriptor` hot + `PTO2TaskPayload` cold). +2. Shared-memory hot/cold split (`TaskDescriptor` hot + `TaskPayload` cold). ## 5. Terminology @@ -86,7 +86,7 @@ Rules: ## 7. Data Model (Requirements + Design) -`PTO2TaskDescriptor` (hot path) carries mixed-task identity/state: +`TaskDescriptor` (hot path) carries mixed-task identity/state: 1. `task_id` 2. `active_mask` @@ -94,7 +94,7 @@ Rules: 4. `kernel_id[3]` for `(AIC, AIV0, AIV1)` 5. dependency heads/counters and packed-buffer metadata -`PTO2TaskPayload` (cold path) carries: +`TaskPayload` (cold path) carries: 1. shared args/tensors/scalars copied once per mixed submit 2. fanin mixed-task IDs diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h index 8fec3970d2..ac9a2b9008 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h @@ -60,18 +60,18 @@ static_assert( "GLOBAL_CONTEXT_INDEX out of sync with intrinsic.h" ); -// Byte offsets into PTO2TaskPayload used by AICore to materialize args[] on the +// Byte offsets into TaskPayload used by AICore to materialize args[] on the // not_ready (early-dispatch) path. The AICore .o does not include // runtime_types.h, so it reads tensor_count / scalar_count / tensors[] / // scalars[] through these constants. The AICPU side (scheduler_dispatch.cpp) // static_asserts them against offsetof where the full struct is visible, so any -// PTO2TaskPayload layout drift fails the build rather than corrupting args[]. -constexpr uint32_t PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET = 0; -constexpr uint32_t PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET = 4; +// TaskPayload layout drift fails the build rather than corrupting args[]. +constexpr uint32_t TASKPAYLOAD_TENSOR_COUNT_OFFSET = 0; +constexpr uint32_t TASKPAYLOAD_SCALAR_COUNT_OFFSET = 4; // Cache line 9 (byte 576) holds the AICPU-only DispatchPredicate; tensors follow it. -constexpr uint32_t PTO2_TASKPAYLOAD_TENSORS_OFFSET = 640; -constexpr uint32_t PTO2_TASKPAYLOAD_SCALARS_OFFSET = 4736; -constexpr uint32_t PTO2_TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) +constexpr uint32_t TASKPAYLOAD_TENSORS_OFFSET = 640; +constexpr uint32_t TASKPAYLOAD_SCALARS_OFFSET = 4736; +constexpr uint32_t TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) /** * Per-core dispatch payload: function address + args[] + SPMD context. @@ -90,7 +90,7 @@ struct alignas(64) PTO2DispatchPayload { // capacity are cold (prefilled once at init) but ride this hot line for free. // Sized to exactly 64B so both dispatch paths write one control line: the // ready path (src_payload = 0) then also fills args[0..num_args); the gated - // path (src_payload = &PTO2TaskPayload) leaves args[] to the idle AICore. + // path (src_payload = &TaskPayload) leaves args[] to the idle AICore. uint64_t function_bin_addr; /**< Kernel entry address in GM (set by Scheduler). */ /** Per-dispatch context: block_idx/block_num (hot) + async_ctx (task_token hot, @@ -100,7 +100,7 @@ struct alignas(64) PTO2DispatchPayload { /** Early-dispatch gate AND source pointer, folded into one field. 0 = ready: * AICore executes on pickup (args[] already filled by the AICPU). Non-zero = - * gated: the value is the source PTO2TaskPayload address; the AICore fills + * gated: the value is the source TaskPayload address; the AICore fills * args[0..num_args) from it, then waits for the doorbell (DATA_MAIN_BASE * high 32 == this dispatch's reg_task_id) before executing. A payload address * is never 0, so the gate flag is lossless. */ diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index f07b98dbf3..f7870d23e3 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -445,7 +445,7 @@ void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state void PTO2OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin) { PTO2SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; - PTO2TaskPayload *payload = slot_state.payload; + TaskPayload *payload = slot_state.payload; slot_state.fanin_count = wfanin + 1; int32_t completed_fanin = 0; @@ -537,11 +537,11 @@ static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, ChipTaskSlotS static void scope_tasks_push(PTO2OrchestratorState *orch, ChipTaskSlotState *task_slot_state); -struct PTO2PreparedTask { +struct PreparedTask { TaskId task_id = TaskId::invalid(); PTO2TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; - PTO2TaskDescriptor *task = nullptr; - PTO2TaskPayload *payload = nullptr; + TaskDescriptor *task = nullptr; + TaskPayload *payload = nullptr; ChipTaskSlotState *slot_state = nullptr; }; @@ -594,7 +594,7 @@ static bool check_scope_can_accept_task(PTO2OrchestratorState *orch, PTO2TaskAll static bool prepare_task( PTO2OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, - TaskAttrs task_attrs, PTO2PreparedTask *out + TaskAttrs task_attrs, PreparedTask *out ) { uint8_t ring_id = orch->current_ring_id(); auto &allocator = orch->rings[ring_id].task_allocator; @@ -641,7 +641,7 @@ static bool prepare_task( out->slot_state->bind_buffers(out->payload, out->task); // prepare_task does NO payload writes: all payload content (tensors/scalars + - // early-dispatch fields) is initialized in PTO2TaskPayload::init, the + // early-dispatch fields) is initialized in TaskPayload::init, the // single payload-init point, which runs before Orch-side wiring publish. // Fields already reset by advance_ring_pointers (eager reset after CONSUMED): @@ -888,7 +888,7 @@ static TaskOutputTensors submit_task_common( CYCLE_COUNT_START(); TaskOutputTensors result; PTO2OutputLayout layout = calculate_output_layout(args); - PTO2PreparedTask prepared; + PreparedTask prepared; if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; } @@ -897,8 +897,8 @@ static TaskOutputTensors submit_task_common( PTO2RingFlowControl &fc = orch->sm_header->rings[ring_id].fc; TaskId task_id = prepared.task_id; ChipTaskSlotState &cur_slot_state = *prepared.slot_state; - PTO2TaskDescriptor &task = *prepared.task; - PTO2TaskPayload &payload = *prepared.payload; + TaskDescriptor &task = *prepared.task; + TaskPayload &payload = *prepared.payload; result.set_task_id(task_id); // dep_gen capture point: snapshot the orch submit_task inputs while the @@ -1291,15 +1291,15 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) } PTO2OutputLayout layout = calculate_output_layout(args); - PTO2PreparedTask prepared; + PreparedTask prepared; // Kernel-less alloc task: no active subtasks, no dispatch-time markers. The // early-dispatch hint is force-set below (see the flag-the-creator note). if (!prepare_task(orch, args, layout.total_output_size, ActiveMask{}, TaskAttrs{}, &prepared)) { return TaskOutputTensors{}; } - PTO2TaskDescriptor &task = *prepared.task; - PTO2TaskPayload &payload = *prepared.payload; + TaskDescriptor &task = *prepared.task; + TaskPayload &payload = *prepared.payload; CYCLE_COUNT_LAP(g_orch_alloc_cycle); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index 8efb604553..6a5115e34d 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -39,7 +39,7 @@ void PTO2FaninPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_ta int32_t scan_end = sm_last_task_alive; for (int32_t task_id = reclaim_task_cursor; task_id < scan_end; ++task_id) { - PTO2TaskPayload &payload = ring.get_payload_by_task_id(task_id); + TaskPayload &payload = ring.get_payload_by_task_id(task_id); if (payload.fanin_spill_pool != this) { continue; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index b35cce9dc3..8652006c7f 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -90,7 +90,7 @@ class PTO2TaskAllocator { * exercise corner cases like task IDs near INT32_MAX. */ void init( - PTO2TaskDescriptor *descriptors, int32_t window_size, std::atomic *current_index_ptr, + TaskDescriptor *descriptors, int32_t window_size, std::atomic *current_index_ptr, std::atomic *last_alive_ptr, void *heap_base, uint64_t heap_size, std::atomic *error_code_ptr, ChipTaskSlotState *slot_states = nullptr, int32_t initial_local_task_id = 0, uint8_t ring_id = 0 ) { @@ -254,7 +254,7 @@ class PTO2TaskAllocator { private: // --- Task Ring --- - PTO2TaskDescriptor *descriptors_ = nullptr; + TaskDescriptor *descriptors_ = nullptr; // Parallel to descriptors_, indexed by task_id & window_mask_. Read-only here, // used by the deadlock detector to identify the head task's slot. ChipTaskSlotState *slot_states_ = nullptr; @@ -310,7 +310,7 @@ class PTO2TaskAllocator { } heap_rebase_anchor_task_id_ = -1; - PTO2TaskDescriptor &desc = descriptors_[(last_alive - 1) & window_mask_]; + TaskDescriptor &desc = descriptors_[(last_alive - 1) & window_mask_]; uint64_t old_tail = heap_tail_; heap_tail_ = static_cast(static_cast(desc.packed_buffer_end) - static_cast(heap_base_)); @@ -649,7 +649,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( } template -inline PTO2FaninForEachReturn for_each_fanin_slot_state(const PTO2TaskPayload &payload, Fn &&fn) { +inline PTO2FaninForEachReturn for_each_fanin_slot_state(const TaskPayload &payload, Fn &&fn) { return for_each_fanin_storage( payload.fanin_inline_edges, payload.fanin_actual_count, payload.fanin_spill_start, *payload.fanin_spill_pool, static_cast(fn) diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 46d63fb655..241281744b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -215,7 +215,7 @@ struct PTO2DepListEntry { * * Fields set by Orchestrator at submission, read by Scheduler for dispatch. */ -struct PTO2TaskDescriptor { +struct TaskDescriptor { // Mixed-task identification (encodes ring_id in upper 32 bits) TaskId task_id; // raw: (ring_id << 32) | local_id @@ -229,14 +229,14 @@ struct PTO2TaskDescriptor { // A 4-byte alignment pad follows kernel_id[3]; the scheduler and shared-memory // ABI depend on the descriptor size and packed_buffer_base offset staying fixed. -static_assert(sizeof(PTO2TaskDescriptor) == 40, "PTO2TaskDescriptor size is part of the shared-memory ABI"); -static_assert(offsetof(PTO2TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged"); +static_assert(sizeof(TaskDescriptor) == 40, "TaskDescriptor size is part of the shared-memory ABI"); +static_assert(offsetof(TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged"); // ============================================================================= // Per-Slot Scheduling State // ============================================================================= -// Early-dispatch claim states for PTO2TaskPayload::early_dispatch_state. +// Early-dispatch claim states for TaskPayload::early_dispatch_state. enum PTO2EarlyDispatchState : uint8_t { PTO2_EARLY_DISPATCH_NONE = 0, // not pre-staged PTO2_EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress @@ -274,7 +274,7 @@ inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; * by bulk tensor and scalar data. Small fanins stay fully inline; larger * fanins spill into a per-ring ring buffer slice. */ -struct PTO2TaskPayload { +struct TaskPayload { // === Cache lines 0-8 (576B) — metadata + inline fanin === int32_t tensor_count{0}; int32_t scalar_count{0}; @@ -413,23 +413,21 @@ struct PTO2TaskPayload { } }; -// PTO2TaskPayload layout verification (offsetof requires complete type). -static_assert(offsetof(PTO2TaskPayload, fanin_spill_pool) == 16, "spill pool pointer layout drift"); -static_assert(offsetof(PTO2TaskPayload, fanin_inline_edges) == 24, "inline fanin array must follow spill metadata"); +// TaskPayload layout verification (offsetof requires complete type). +static_assert(offsetof(TaskPayload, fanin_spill_pool) == 16, "spill pool pointer layout drift"); +static_assert(offsetof(TaskPayload, fanin_inline_edges) == 24, "inline fanin array must follow spill metadata"); static_assert( - offsetof(PTO2TaskPayload, predicate) == 576, + offsetof(TaskPayload, predicate) == 576, "dispatch predicate occupies cache line 9 at fixed byte 576 (before tensors, never moves)" ); +static_assert(offsetof(TaskPayload, tensors) == 640, "tensors must start at byte 640 (cache line 10, after predicate)"); static_assert( - offsetof(PTO2TaskPayload, tensors) == 640, "tensors must start at byte 640 (cache line 10, after predicate)" -); -static_assert( - offsetof(PTO2TaskPayload, scalars) == 640 + MAX_TENSOR_ARGS * sizeof(ChipTensor), + offsetof(TaskPayload, scalars) == 640 + MAX_TENSOR_ARGS * sizeof(ChipTensor), "scalars must immediately follow tensors" ); static_assert( - sizeof(PTO2TaskPayload) == 640 + MAX_TENSOR_ARGS * sizeof(ChipTensor) + MAX_SCALAR_ARGS * sizeof(uint64_t), - "PTO2TaskPayload size = metadata(576) + predicate cache line(64) + tensors + scalars" + sizeof(TaskPayload) == 640 + MAX_TENSOR_ARGS * sizeof(ChipTensor) + MAX_SCALAR_ARGS * sizeof(uint64_t), + "TaskPayload size = metadata(576) + predicate cache line(64) + tensors + scalars" ); /** @@ -495,8 +493,8 @@ struct alignas(64) ChipTaskSlotState { // but written here per-submit instead of in an O(window_size) init loop — // these are the only "scale-dependent" pointers in this struct, so moving // them out of init makes startup cost independent of task_window_size. - PTO2TaskPayload *payload; - PTO2TaskDescriptor *task; + TaskPayload *payload; + TaskDescriptor *task; // --- Set per-submit (depend on task inputs) --- ActiveMask active_mask; // Bitmask of active subtask slots (set once) @@ -552,7 +550,7 @@ struct alignas(64) ChipTaskSlotState { * the same 64B slot_state cache line that prepare_task is already * dirtying) to avoid the init-time per-slot loop. */ - void bind_buffers(PTO2TaskPayload *p, PTO2TaskDescriptor *t) { + void bind_buffers(TaskPayload *p, TaskDescriptor *t) { payload = p; task = t; } @@ -612,7 +610,7 @@ struct alignas(64) ChipTaskSlotState { lifecycle_flags.store(PTO2_LIFECYCLE_FLAGS_NONE, std::memory_order_relaxed); // Note: active_mask and task_attrs are per-submit-constant fields // rewritten in prepare_task on every reuse, so they are not reset here. - // Payload early-dispatch fields are initialized by PTO2TaskPayload::init + // Payload early-dispatch fields are initialized by TaskPayload::init // on every submit before the slot becomes scheduler-visible. } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index e6a508b66f..47ccf0e13a 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -807,7 +807,7 @@ struct PTO2SchedulerState { return true; } - static inline bool try_claim_early_dispatch_launch(PTO2TaskPayload &payload) { + static inline bool try_claim_early_dispatch_launch(TaskPayload &payload) { uint8_t expected = PTO2_EARLY_DISPATCH_LAUNCH_NONE; return payload.early_dispatch_launch_state.compare_exchange_strong( expected, PTO2_EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst @@ -835,22 +835,22 @@ struct PTO2SchedulerState { } } - static inline bool try_claim_early_sync_drain(PTO2TaskPayload &payload) { + static inline bool try_claim_early_sync_drain(TaskPayload &payload) { uint8_t expected = PTO2_EARLY_SYNC_DRAIN_NONE; return payload.early_sync_drain_state.compare_exchange_strong( expected, PTO2_EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst ); } - static inline bool owns_early_sync_drain(const PTO2TaskPayload &payload) { + static inline bool owns_early_sync_drain(const TaskPayload &payload) { return (payload.early_sync_drain_state.load(std::memory_order_acquire) & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; } - static inline void mark_early_sync_drain_armed(PTO2TaskPayload &payload) { + static inline void mark_early_sync_drain_armed(TaskPayload &payload) { payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); } - static inline bool publish_ready_to_early_sync_drain(PTO2TaskPayload &payload) { + static inline bool publish_ready_to_early_sync_drain(TaskPayload &payload) { uint8_t previous = payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); return (previous & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; @@ -869,7 +869,7 @@ struct PTO2SchedulerState { } } - static inline void finish_early_sync_drain(PTO2TaskPayload &payload) { + static inline void finish_early_sync_drain(TaskPayload &payload) { uint8_t state = payload.early_sync_drain_state.load(std::memory_order_seq_cst); while ((state & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0 && (state & PTO2_EARLY_SYNC_DRAIN_COMPLETE) == 0) { uint8_t desired = state | PTO2_EARLY_SYNC_DRAIN_COMPLETE; @@ -1339,7 +1339,7 @@ struct PTO2SchedulerState { #else int32_t on_task_release(ChipTaskSlotState &slot_state) { #endif - PTO2TaskPayload *payload = slot_state.payload; + TaskPayload *payload = slot_state.payload; int32_t released = 0; // Only DEP_RETAIN edges still hold a fanout pin at completion: an // ordering-only edge released its submit->wire pin at wiring, so releasing diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 3346a10a3f..d02801a3db 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -390,7 +390,7 @@ SchedulerContext::StallClassification SchedulerContext::classify_stall_reason() if (cnt_running == 0) { // Snapshot the non-atomic task pointer once: it can be null on a // torn slot, and a concurrent writer may flip it mid-read. - PTO2TaskDescriptor *task_ptr = slot_state.task; + TaskDescriptor *task_ptr = slot_state.task; cls.stuck_task_id = (task_ptr != nullptr) ? static_cast(task_ptr->task_id.raw) : -1; cls.stuck_core = run_core; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 3ab4d3ad57..88feeaa8bf 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -38,13 +38,13 @@ #endif // AICore materializes args[] from src_payload on the gated path using the -// byte offsets in dispatch_payload.h (the AICore .o cannot see PTO2TaskPayload). +// byte offsets in dispatch_payload.h (the AICore .o cannot see TaskPayload). // Pin those constants to the real layout here, where the struct is fully visible. -static_assert(offsetof(PTO2TaskPayload, tensor_count) == PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET); -static_assert(offsetof(PTO2TaskPayload, scalar_count) == PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET); -static_assert(offsetof(PTO2TaskPayload, tensors) == PTO2_TASKPAYLOAD_TENSORS_OFFSET); -static_assert(offsetof(PTO2TaskPayload, scalars) == PTO2_TASKPAYLOAD_SCALARS_OFFSET); -static_assert(sizeof(ChipTensor) == PTO2_TASKPAYLOAD_TENSOR_STRIDE); +static_assert(offsetof(TaskPayload, tensor_count) == TASKPAYLOAD_TENSOR_COUNT_OFFSET); +static_assert(offsetof(TaskPayload, scalar_count) == TASKPAYLOAD_SCALAR_COUNT_OFFSET); +static_assert(offsetof(TaskPayload, tensors) == TASKPAYLOAD_TENSORS_OFFSET); +static_assert(offsetof(TaskPayload, scalars) == TASKPAYLOAD_SCALARS_OFFSET); +static_assert(sizeof(ChipTensor) == TASKPAYLOAD_TENSOR_STRIDE); // ============================================================================= // Dispatch helpers diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index 30716f0603..7bb8da7fd8 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -55,8 +55,8 @@ void PTO2SharedMemoryHandle::setup_pointers_per_ring(const uint64_t task_window_ for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { auto off = pto2_sm_layout::ring_segment_offsets(task_window_sizes, r); auto &ring = header->rings[r]; - ring.task_descriptors = (PTO2TaskDescriptor *)(base + off.descriptors); - ring.task_payloads = (PTO2TaskPayload *)(base + off.payloads); + ring.task_descriptors = (TaskDescriptor *)(base + off.descriptors); + ring.task_payloads = (TaskPayload *)(base + off.payloads); ring.slot_states = (ChipTaskSlotState *)(base + off.slot_states); } } @@ -151,8 +151,8 @@ void PTO2SharedMemoryHandle::init_header_per_ring( header->rings[r].task_window_mask = static_cast(task_window_sizes[r] - 1); header->rings[r].heap_size = heap_sizes[r]; header->rings[r].task_descriptors_offset = offset; - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 8f072eabd4..421db9da1a 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -93,21 +93,19 @@ struct alignas(64) PTO2SharedMemoryRingHeader { uint64_t task_descriptors_offset; // Offset from SM base, in bytes // Per-ring data pointers (host-side, set by setup_pointers) - PTO2TaskDescriptor *task_descriptors; - PTO2TaskPayload *task_payloads; + TaskDescriptor *task_descriptors; + TaskPayload *task_payloads; ChipTaskSlotState *slot_states; int32_t get_slot_by_task_id(int32_t local_task_id) { return local_task_id & task_window_mask; } - PTO2TaskDescriptor &get_task_by_slot(int32_t slot) { return task_descriptors[slot]; } + TaskDescriptor &get_task_by_slot(int32_t slot) { return task_descriptors[slot]; } - PTO2TaskDescriptor &get_task_by_task_id(int32_t local_id) { - return task_descriptors[get_slot_by_task_id(local_id)]; - } + TaskDescriptor &get_task_by_task_id(int32_t local_id) { return task_descriptors[get_slot_by_task_id(local_id)]; } - PTO2TaskPayload &get_payload_by_slot(int32_t slot) { return task_payloads[slot]; } + TaskPayload &get_payload_by_slot(int32_t slot) { return task_payloads[slot]; } - PTO2TaskPayload &get_payload_by_task_id(int32_t local_id) { return task_payloads[get_slot_by_task_id(local_id)]; } + TaskPayload &get_payload_by_task_id(int32_t local_id) { return task_payloads[get_slot_by_task_id(local_id)]; } ChipTaskSlotState &get_slot_state_by_slot(int32_t slot) { return slot_states[slot]; } @@ -292,15 +290,15 @@ ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "pto2_sm_layout: ring_id out of range"); uint64_t off = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); for (int r = 0; r < ring_id; r++) { - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); } PTO2RingSegmentOffsets o{}; o.descriptors = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); o.payloads = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); o.slot_states = off; off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); o.end = off; @@ -308,10 +306,10 @@ ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int } // Device address of ring `ring_id`'s task_descriptors array. -inline PTO2TaskDescriptor *ring_task_descriptors_addr( +inline TaskDescriptor *ring_task_descriptors_addr( void *sm_dev_base, const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int ring_id ) noexcept { - return reinterpret_cast( + return reinterpret_cast( static_cast(sm_dev_base) + ring_segment_offsets(task_window_sizes, ring_id).descriptors ); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h index 74fd9b2f0a..b2b9149036 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h @@ -120,7 +120,7 @@ constexpr bool dep_has_retain(DepFlags f) { return (f & DEP_RETAIN) != DEP_NONE; * * LIFETIME — single-scope only: * Internally this class stores pointers into the submitting task's payload - * (PTO2TaskPayload::tensors[]), which lives in a ring-buffer slot. After + * (TaskPayload::tensors[]), which lives in a ring-buffer slot. After * scope_end the slot becomes eligible for reuse, and a later submit will * overwrite the same ChipTensor storage in place. Therefore the * TaskOutputTensors instance, the const ChipTensor& returned by get_ref(), and diff --git a/src/a5/docs/runtimes.md b/src/a5/docs/runtimes.md index c3097a0ee7..e2d7e295ac 100644 --- a/src/a5/docs/runtimes.md +++ b/src/a5/docs/runtimes.md @@ -7,7 +7,7 @@ Two runtime implementations live under `src/a5/runtime/`, each providing a diffe | Feature | host_build_graph | tensormap_and_ringbuffer | | ------- | ---------------- | ------------------------ | | Graph built on | Host CPU | AICPU (device) | -| Task storage | Fixed `Task[]` array | Ring buffer (`PTO2TaskDescriptor[]`) | +| Task storage | Fixed `Task[]` array | Ring buffer (`TaskDescriptor[]`) | | Dependencies | Explicit edges | Auto-derived via TensorMap | | Use case | Development, debugging | Production workloads | diff --git a/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp b/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp index f69ed3c82b..61da824b68 100644 --- a/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp +++ b/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp @@ -181,7 +181,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // execute_task() below — strictly AFTER this gate — so predecessor // outputs are visible. src_payload == 0 (the common path) skips this; // a non-zero src_payload is both the gate flag and the source - // PTO2TaskPayload. + // TaskPayload. if (exec_payload->src_payload != 0) { // AICPU staged only src_payload, not the arg vector — fill // args[0..num_args) ourselves now, while we are idle waiting for @@ -192,20 +192,19 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // pre-filled once at init() and never changed; tensor and scalar // counts are set by the orchestrator, so num_args never reaches them. __gm__ char *src = reinterpret_cast<__gm__ char *>(exec_payload->src_payload); - int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET); - int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET); + int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_TENSOR_COUNT_OFFSET); + int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_SCALAR_COUNT_OFFSET); // Each region is named by an int32 delta from the naming field's own // address, so resolve the field, then add what it holds. - __gm__ char *tensors_field = src + PTO2_TASKPAYLOAD_TENSORS_DELTA_OFFSET; + __gm__ char *tensors_field = src + TASKPAYLOAD_TENSORS_DELTA_OFFSET; __gm__ char *src_tensors = tensors_field + *reinterpret_cast<__gm__ int32_t *>(tensors_field); - __gm__ char *scalars_field = src + PTO2_TASKPAYLOAD_SCALARS_DELTA_OFFSET; + __gm__ char *scalars_field = src + TASKPAYLOAD_SCALARS_DELTA_OFFSET; __gm__ uint64_t *src_scalars = reinterpret_cast<__gm__ uint64_t *>( scalars_field + *reinterpret_cast<__gm__ int32_t *>(scalars_field) ); int n = 0; for (int32_t i = 0; i < tensor_count; i++) { - exec_payload->args[n++] = - reinterpret_cast(src_tensors + i * PTO2_TASKPAYLOAD_TENSOR_STRIDE); + exec_payload->args[n++] = reinterpret_cast(src_tensors + i * TASKPAYLOAD_TENSOR_STRIDE); } for (int32_t i = 0; i < scalar_count; i++) { exec_payload->args[n++] = src_scalars[i]; diff --git a/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md b/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md index a52ca0e20d..c8ebfd16d3 100644 --- a/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md @@ -72,8 +72,8 @@ The shared image uses three per-slot structures: | Structure | Purpose | | --------- | ------- | -| `PTO2TaskDescriptor` | Full task ID, kernel IDs, packed-buffer addresses | -| `PTO2TaskPayload` | Argument counts, predicate, dispatch metadata, and a delta naming each of its tensor, scalar and fanin regions — the arguments themselves live in the pool segments, so the payload is a fixed three cache lines regardless of the argument caps | +| `TaskDescriptor` | Full task ID, kernel IDs, packed-buffer addresses | +| `TaskPayload` | Argument counts, predicate, dispatch metadata, and a delta naming each of its tensor, scalar and fanin regions — the arguments themselves live in the pool segments, so the payload is a fixed three cache lines regardless of the argument caps | | `ChipTaskSlotState` | Active mask, attributes, block/subtask counters, completion state, task/payload bindings | The host/device boundary is POD and position-independent. Fanins are integer diff --git a/src/a5/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md b/src/a5/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md index e5b98f7a5d..0839f42e9f 100644 --- a/src/a5/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a5/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md @@ -26,9 +26,9 @@ same mixed-task contract. The shared graph image separates stable task identity from scheduling state: -- `PTO2TaskDescriptor` contains the task ID, kernel IDs, and packed-buffer +- `TaskDescriptor` contains the task ID, kernel IDs, and packed-buffer addresses. -- `PTO2TaskPayload` contains the argument counts, predicate, and a delta naming +- `TaskPayload` contains the argument counts, predicate, and a delta naming each of its tensor, scalar and position-independent fanin-id regions. The arguments live in the pool segments, not in the payload. - `ChipTaskSlotState` contains the active mask, task attributes, logical block diff --git a/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h b/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h index 400bc202f9..554e797d17 100644 --- a/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h +++ b/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h @@ -59,22 +59,22 @@ static_assert( "GLOBAL_CONTEXT_INDEX out of sync with intrinsic.h" ); -// Byte offsets into PTO2TaskPayload used by AICore to materialize args[] on the +// Byte offsets into TaskPayload used by AICore to materialize args[] on the // not_ready (early-dispatch) path. The AICore .o does not include // runtime_types.h, so it reads tensor_count / scalar_count / tensors[] / // scalars[] through these constants. The AICPU side (scheduler_dispatch.cpp) // static_asserts them against offsetof where the full struct is visible, so any -// PTO2TaskPayload layout drift fails the build rather than corrupting args[]. -constexpr uint32_t PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET = 0; -constexpr uint32_t PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET = 4; +// TaskPayload layout drift fails the build rather than corrupting args[]. +constexpr uint32_t TASKPAYLOAD_TENSOR_COUNT_OFFSET = 0; +constexpr uint32_t TASKPAYLOAD_SCALAR_COUNT_OFFSET = 4; // Offsets of the fields that NAME the argument regions, not of the regions themselves: // a region's address is (src + ) + . -constexpr uint32_t PTO2_TASKPAYLOAD_TENSORS_DELTA_OFFSET = 12; -constexpr uint32_t PTO2_TASKPAYLOAD_SCALARS_DELTA_OFFSET = 16; +constexpr uint32_t TASKPAYLOAD_TENSORS_DELTA_OFFSET = 12; +constexpr uint32_t TASKPAYLOAD_SCALARS_DELTA_OFFSET = 16; // Cache line 9 (byte 576) holds the AICPU-only DispatchPredicate. Scalars follow it, // then tensors — the tensor array is last because it is the only region whose used // extent varies per task, so a task's read set is a contiguous prefix. -constexpr uint32_t PTO2_TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) +constexpr uint32_t TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) /** * Per-core dispatch payload: function address + args[] + SPMD context. @@ -93,7 +93,7 @@ struct alignas(64) PTO2DispatchPayload { // capacity are cold (prefilled once at init) but ride this hot line for free. // Sized to exactly 64B so both dispatch paths write one control line: the // ready path (src_payload = 0) then also fills args[0..num_args); the gated - // path (src_payload = &PTO2TaskPayload) leaves args[] to the idle AICore. + // path (src_payload = &TaskPayload) leaves args[] to the idle AICore. uint64_t function_bin_addr; /**< Kernel entry address in GM (set by Scheduler). */ /** Per-dispatch context: block_idx/block_num (hot) + async_ctx (task_token hot, @@ -103,7 +103,7 @@ struct alignas(64) PTO2DispatchPayload { /** Early-dispatch gate AND source pointer, folded into one field. 0 = ready: * AICore executes on pickup (args[] already filled by the AICPU). Non-zero = - * gated: the value is the source PTO2TaskPayload address; the AICore fills + * gated: the value is the source TaskPayload address; the AICore fills * args[0..num_args) from it, then waits for the doorbell (DATA_MAIN_BASE * high 32 == this dispatch's reg_task_id) before executing. A payload address * is never 0, so the gate flag is lossless. */ diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 9e72bb384b..a028e1fda3 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1141,7 +1141,7 @@ static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { // (the consumer), used to bump each producer's last_consumer_local_id (the // reclaim gate the host wait_for_consumers polls via completed_watermark). struct PTO2FaninBuilder { - PTO2FaninBuilder(PTO2OrchestratorState *orch, PTO2TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : + PTO2FaninBuilder(PTO2OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : count(0), orch(orch), seen_epoch(seen_epoch), @@ -1152,7 +1152,7 @@ struct PTO2FaninBuilder { PTO2OrchestratorState *orch{nullptr}; uint32_t seen_epoch{0}; int32_t self_local{0}; - PTO2TaskPayload *payload{nullptr}; + TaskPayload *payload{nullptr}; // The payload's fanin region, resolved once: the appends below would otherwise // re-resolve the delta per producer. Requires the regions bound before construction. int32_t *slots{nullptr}; @@ -1216,11 +1216,11 @@ static bool append_fanin_or_fail( return true; } -struct PTO2PreparedTask { +struct PreparedTask { TaskId task_id = TaskId::invalid(); PTO2TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; - PTO2TaskDescriptor *task = nullptr; - PTO2TaskPayload *payload = nullptr; + TaskDescriptor *task = nullptr; + TaskPayload *payload = nullptr; ChipTaskSlotState *slot_state = nullptr; }; @@ -1240,7 +1240,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { static bool prepare_task( PTO2OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, - TaskAttrs task_attrs, PTO2PreparedTask *out + TaskAttrs task_attrs, PreparedTask *out ) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit task outside a scope"); uint8_t ring_id = 0; @@ -1308,7 +1308,7 @@ static bool prepare_task( out->slot_state->bind_buffers(out->payload, out->task); // prepare_task does NO payload writes: all payload content (tensors/scalars + - // early-dispatch fields) is initialized in PTO2TaskPayload::init, the + // early-dispatch fields) is initialized in TaskPayload::init, the // single payload-init point, which runs before Orch-side wiring publish. // Fields already zeroed by the reset_for_reuse() above: @@ -1456,14 +1456,14 @@ static TaskOutputTensors submit_task_common( ORCH_PHASE_START(); TaskOutputTensors result; PTO2OutputLayout layout = calculate_output_layout(args); - PTO2PreparedTask prepared; + PreparedTask prepared; if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; } PTO2SchedulerState *sched = orch->scheduler; TaskId task_id = prepared.task_id; - PTO2TaskDescriptor &task = *prepared.task; - PTO2TaskPayload &payload = *prepared.payload; + TaskDescriptor &task = *prepared.task; + TaskPayload &payload = *prepared.payload; result.set_task_id(task_id); // dep_gen capture point: open this task's graph entry before its dependency @@ -1734,7 +1734,7 @@ bool graph_boundary_matches(const GraphBoundary &boundary, const GraphTaskArgs & return true; } -void graph_reset_outer_payload(PTO2TaskPayload &payload) { +void graph_reset_outer_payload(TaskPayload &payload) { payload.tensor_count = 0; payload.scalar_count = 0; payload.fanin_count = 0; @@ -1797,8 +1797,8 @@ bool graph_submit_outer( } const TaskId task_id = TaskId::make(0, static_cast(allocation.task_id)); PTO2SharedMemoryRingHeader &ring = orch->sm_header->ring; - PTO2TaskDescriptor &task = ring.task_descriptors[allocation.slot]; - PTO2TaskPayload &payload = ring.task_payloads[allocation.slot]; + TaskDescriptor &task = ring.task_descriptors[allocation.slot]; + TaskPayload &payload = ring.task_payloads[allocation.slot]; ChipTaskSlotState &slot = ring.get_slot_state_by_slot(allocation.slot); // Init-on-write, as in prepare_task: this slot's dynamic scheduling fields and @@ -2025,7 +2025,7 @@ TaskOutputTensors graph_record_submit_node( node.record_packed_base = packed_base_addr; node.total_output_size = aligned_output; - // Build the tensor list exactly as PTO2TaskPayload::init: inputs/inouts copy + // Build the tensor list exactly as TaskPayload::init: inputs/inouts copy // the caller's ChipTensor; outputs materialize from the create-info onto the // scratch buffer and carry this node's owner id. const int32_t tensor_count = args.tensor_count(); @@ -2694,15 +2694,15 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) } PTO2OutputLayout layout = calculate_output_layout(args); - PTO2PreparedTask prepared; + PreparedTask prepared; // Kernel-less alloc task: no active subtasks, no dispatch-time attributes. The // early-dispatch hint is force-set below (see the flag-the-creator note). if (!prepare_task(orch, args, layout.total_output_size, ActiveMask{}, TaskAttrs{}, &prepared)) { return TaskOutputTensors{}; } - PTO2TaskDescriptor &task = *prepared.task; - PTO2TaskPayload &payload = *prepared.payload; + TaskDescriptor &task = *prepared.task; + TaskPayload &payload = *prepared.payload; CYCLE_COUNT_LAP(g_orch_alloc_cycle); diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 4e99114933..366baa3575 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -141,7 +141,7 @@ inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HE #define PTO2_MAX_FANIN 128 // Alignment of every per-task region inside an argument pool. Each region starts -// and ends on a cache line so PTO2TaskPayload::init's round-up scalar memcpy stays +// and ends on a cache line so TaskPayload::init's round-up scalar memcpy stays // inside the task's own region — see its comment. ChipTensor is already 2 cache // lines, so only the fanin and scalar regions need the round-up. inline constexpr int32_t ARG_POOL_ALIGN = 64; @@ -233,7 +233,7 @@ struct ChipTaskSlotState; // Forward declaration (defined below) * * Fields set by Orchestrator at submission, read by Scheduler for dispatch. */ -struct PTO2TaskDescriptor { +struct TaskDescriptor { // Mixed-task identification (encodes ring_id in upper 32 bits) TaskId task_id; // raw: (ring_id << 32) | local_id @@ -247,8 +247,8 @@ struct PTO2TaskDescriptor { // A 4-byte alignment pad follows kernel_id[3]; the scheduler and shared-memory // ABI depend on the descriptor size and packed_buffer_base offset staying fixed. -static_assert(sizeof(PTO2TaskDescriptor) == 40, "PTO2TaskDescriptor size is part of the shared-memory ABI"); -static_assert(offsetof(PTO2TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged"); +static_assert(sizeof(TaskDescriptor) == 40, "TaskDescriptor size is part of the shared-memory ABI"); +static_assert(offsetof(TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged"); // ============================================================================= // Per-Slot Scheduling State @@ -261,7 +261,7 @@ static_assert(offsetof(PTO2TaskDescriptor, packed_buffer_base) == 24, "packed_bu * by bulk tensor and scalar data. Small fanins stay fully inline; larger * fanins spill into a per-ring ring buffer slice. */ -// Early-dispatch claim states for PTO2TaskPayload::early_dispatch_state. +// Early-dispatch claim states for TaskPayload::early_dispatch_state. enum PTO2EarlyDispatchState : uint8_t { PTO2_EARLY_DISPATCH_NONE = 0, // not pre-staged PTO2_EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress @@ -292,7 +292,7 @@ enum PTO2EarlySyncDrainState : uint8_t { // consumer can pre-stage all its idle cores. 2 words = 128 bits >= 72. inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; -struct PTO2TaskPayload { +struct TaskPayload { // === Cache line 0 (64B) — the dispatch path's own line === // sizeof is independent of PTO2_MAX_FANIN / MAX_TENSOR_ARGS / MAX_SCALAR_ARGS: // widening a cap costs pool bytes for the tasks that need them, not a control @@ -500,34 +500,32 @@ struct PTO2TaskPayload { } }; -// PTO2TaskPayload layout verification (offsetof requires complete type). The counts +// TaskPayload layout verification (offsetof requires complete type). The counts // and region deltas share the first cache line, the early-dispatch atomics own the // second, and the AICPU-only predicate + dump metadata own the third. -static_assert(offsetof(PTO2TaskPayload, tensors) == 12, "region deltas must follow the three counts"); +static_assert(offsetof(TaskPayload, tensors) == 12, "region deltas must follow the three counts"); static_assert( - offsetof(PTO2TaskPayload, fanin) + sizeof(simpler::hbg::SelfRelativePtr) <= 64, + offsetof(TaskPayload, fanin) + sizeof(simpler::hbg::SelfRelativePtr) <= 64, "counts + region deltas must fit the first cache line" ); static_assert( - offsetof(PTO2TaskPayload, staged_core_mask) == 64, + offsetof(TaskPayload, staged_core_mask) == 64, "the early-dispatch atomics own cache line 1: they are written during staging while " "line 0's counts and deltas are read at dispatch, so sharing a line would false-share" ); -static_assert(offsetof(PTO2TaskPayload, predicate) == 128, "dispatch predicate owns cache line 2"); +static_assert(offsetof(TaskPayload, predicate) == 128, "dispatch predicate owns cache line 2"); static_assert( - offsetof(PTO2TaskPayload, dump_metadata) + sizeof(ArgsDumpTaskMetadata) <= 192, + offsetof(TaskPayload, dump_metadata) + sizeof(ArgsDumpTaskMetadata) <= 192, "dump metadata must fit the predicate's cache line" ); -static_assert( - sizeof(PTO2TaskPayload) == 192, "PTO2TaskPayload is three cache lines and independent of every argument cap" -); +static_assert(sizeof(TaskPayload) == 192, "TaskPayload is three cache lines and independent of every argument cap"); // compact_live_image restacks the payload segment with one memcpy and the device // copy moves the whole image, so the payload has to be a POD wire struct. Deleting // SelfRelativePtr's copy operations does not cost it that — a deleted special member // is trivial — but only an assertion keeps a future member from doing so silently. static_assert( - std::is_trivially_copyable_v && std::is_standard_layout_v, - "PTO2TaskPayload crosses to the device by memcpy" + std::is_trivially_copyable_v && std::is_standard_layout_v, + "TaskPayload crosses to the device by memcpy" ); static_assert(sizeof(ChipTensor) == 128, "ChipTensor must be 2 cache lines"); @@ -561,8 +559,8 @@ struct alignas(64) ChipTaskSlotState { // --- Per-slot constant, re-bound by orch::prepare_task each submit --- // Self-relative, so the SM image needs no pointer fix-up on its way to the // device: both targets sit in the same block as this field. - simpler::hbg::SelfRelativePtr payload; - simpler::hbg::SelfRelativePtr task; + simpler::hbg::SelfRelativePtr payload; + simpler::hbg::SelfRelativePtr task; // --- Wake list: last-fanin notification (intrusive, lock-free) --- // A pending consumer whose fanin scan finds an unmet producer registers on @@ -617,7 +615,7 @@ struct alignas(64) ChipTaskSlotState { return 0; } - void bind_buffers(PTO2TaskPayload *p, PTO2TaskDescriptor *t) { + void bind_buffers(TaskPayload *p, TaskDescriptor *t) { payload.set(p); task.set(t); } @@ -653,7 +651,7 @@ struct alignas(64) ChipTaskSlotState { // rewritten in prepare_task on every reuse, so they are not reset here. // last_consumer_local_id is seeded in prepare_task once the id is known. // Payload early-dispatch/fanin fields are (re)initialized in - // PTO2TaskPayload::init on every submit, before the slot is visible. + // TaskPayload::init on every submit, before the slot is visible. } }; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h index 720c0c63bf..5ff913e8f9 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -585,7 +585,7 @@ struct PTO2SchedulerState { // lists. The decision is terminal: tasks are never re-polled; a producer's // completion re-scans its waiters via on_mixed_task_complete's wake drain. int classify_fanin_state(const ChipTaskSlotState *s) const { - const PTO2TaskPayload &p = *s->payload; + const TaskPayload &p = *s->payload; const PTO2SharedMemoryRingHeader &ring = *ring_sched_state.ring; const int32_t *fanin = p.fanin_data(); for (int32_t i = p.fanin_count - 1; i >= 0; i--) { @@ -757,7 +757,7 @@ struct PTO2SchedulerState { return true; } - static inline bool try_claim_early_dispatch_launch(PTO2TaskPayload &payload) { + static inline bool try_claim_early_dispatch_launch(TaskPayload &payload) { uint8_t expected = PTO2_EARLY_DISPATCH_LAUNCH_NONE; return payload.early_dispatch_launch_state.compare_exchange_strong( expected, PTO2_EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst @@ -785,22 +785,22 @@ struct PTO2SchedulerState { } } - static inline bool try_claim_early_sync_drain(PTO2TaskPayload &payload) { + static inline bool try_claim_early_sync_drain(TaskPayload &payload) { uint8_t expected = PTO2_EARLY_SYNC_DRAIN_NONE; return payload.early_sync_drain_state.compare_exchange_strong( expected, PTO2_EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst ); } - static inline bool owns_early_sync_drain(const PTO2TaskPayload &payload) { + static inline bool owns_early_sync_drain(const TaskPayload &payload) { return (payload.early_sync_drain_state.load(std::memory_order_acquire) & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; } - static inline void mark_early_sync_drain_armed(PTO2TaskPayload &payload) { + static inline void mark_early_sync_drain_armed(TaskPayload &payload) { payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); } - static inline bool publish_ready_to_early_sync_drain(PTO2TaskPayload &payload) { + static inline bool publish_ready_to_early_sync_drain(TaskPayload &payload) { uint8_t previous = payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); return (previous & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; @@ -819,7 +819,7 @@ struct PTO2SchedulerState { } } - static inline void finish_early_sync_drain(PTO2TaskPayload &payload) { + static inline void finish_early_sync_drain(TaskPayload &payload) { uint8_t state = payload.early_sync_drain_state.load(std::memory_order_seq_cst); while ((state & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0 && (state & PTO2_EARLY_SYNC_DRAIN_COMPLETE) == 0) { uint8_t desired = state | PTO2_EARLY_SYNC_DRAIN_COMPLETE; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 632f4b88e4..e6fcdcfdbf 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -38,13 +38,13 @@ #endif // AICore materializes args[] from src_payload on the gated path using the -// byte offsets in dispatch_payload.h (the AICore .o cannot see PTO2TaskPayload). +// byte offsets in dispatch_payload.h (the AICore .o cannot see TaskPayload). // Pin those constants to the real layout here, where the struct is fully visible. -static_assert(offsetof(PTO2TaskPayload, tensor_count) == PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET); -static_assert(offsetof(PTO2TaskPayload, scalar_count) == PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET); -static_assert(offsetof(PTO2TaskPayload, tensors) == PTO2_TASKPAYLOAD_TENSORS_DELTA_OFFSET); -static_assert(offsetof(PTO2TaskPayload, scalars) == PTO2_TASKPAYLOAD_SCALARS_DELTA_OFFSET); -static_assert(sizeof(ChipTensor) == PTO2_TASKPAYLOAD_TENSOR_STRIDE); +static_assert(offsetof(TaskPayload, tensor_count) == TASKPAYLOAD_TENSOR_COUNT_OFFSET); +static_assert(offsetof(TaskPayload, scalar_count) == TASKPAYLOAD_SCALAR_COUNT_OFFSET); +static_assert(offsetof(TaskPayload, tensors) == TASKPAYLOAD_TENSORS_DELTA_OFFSET); +static_assert(offsetof(TaskPayload, scalars) == TASKPAYLOAD_SCALARS_DELTA_OFFSET); +static_assert(sizeof(ChipTensor) == TASKPAYLOAD_TENSOR_STRIDE); // ============================================================================= // Dispatch helpers diff --git a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp index f8d03e6749..8e098d78f2 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -51,8 +51,8 @@ void PTO2SharedMemoryHandle::setup_pointers(uint64_t pitch) { // argument region through the payload's delta and needs no base at all. auto off = pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents({pitch, 0, 0, 0})); auto &ring = header->ring; - ring.task_descriptors = (PTO2TaskDescriptor *)(base + off.descriptors); - ring.task_payloads = (PTO2TaskPayload *)(base + off.payloads); + ring.task_descriptors = (TaskDescriptor *)(base + off.descriptors); + ring.task_payloads = (TaskPayload *)(base + off.payloads); ring.slot_states = (ChipTaskSlotState *)(base + off.slot_states); ring.completion_flags = (std::atomic *)(base + off.completion_flags); } @@ -153,8 +153,8 @@ void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size) { header->ring.task_window_size = task_window_size; header->ring.task_window_mask = static_cast(task_window_size - 1); header->ring.task_descriptors_offset = offset; - offset += PTO2_ALIGN_UP(task_window_size * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_size * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + offset += PTO2_ALIGN_UP(task_window_size * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + offset += PTO2_ALIGN_UP(task_window_size * sizeof(TaskPayload), PTO2_ALIGN_SIZE); offset += PTO2_ALIGN_UP(task_window_size * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); header->total_size = sm_size; diff --git a/src/a5/runtime/host_build_graph/runtime/shared_memory.h b/src/a5/runtime/host_build_graph/runtime/shared_memory.h index deaa7e136e..32582ad0aa 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a5/runtime/host_build_graph/runtime/shared_memory.h @@ -92,8 +92,8 @@ struct alignas(64) PTO2SharedMemoryRingHeader { uint64_t task_descriptors_offset; // Offset from SM base, in bytes // Per-ring data pointers (host-side, set by setup_pointers) - PTO2TaskDescriptor *task_descriptors; - PTO2TaskPayload *task_payloads; + TaskDescriptor *task_descriptors; + TaskPayload *task_payloads; ChipTaskSlotState *slot_states; // Polling-completion state (device-addressed array, one byte per slot). @@ -139,11 +139,9 @@ struct alignas(64) PTO2SharedMemoryRingHeader { int32_t get_slot_by_task_id(int32_t local_task_id) { return local_task_id & task_window_mask; } - PTO2TaskDescriptor &get_task_by_slot(int32_t slot) { return task_descriptors[slot]; } + TaskDescriptor &get_task_by_slot(int32_t slot) { return task_descriptors[slot]; } - PTO2TaskDescriptor &get_task_by_task_id(int32_t local_id) { - return task_descriptors[get_slot_by_task_id(local_id)]; - } + TaskDescriptor &get_task_by_task_id(int32_t local_id) { return task_descriptors[get_slot_by_task_id(local_id)]; } // No get_payload_by_slot / get_payload_by_task_id here: a payload is reached // through its slot state's `payload` delta, which the image's restack rebinds to @@ -350,9 +348,9 @@ inline PTO2RingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) no uint64_t off = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); PTO2RingSegmentOffsets o{}; o.descriptors = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(e.slots * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); o.payloads = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(e.slots * sizeof(TaskPayload), PTO2_ALIGN_SIZE); o.slot_states = off; off += PTO2_ALIGN_UP(e.slots * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); o.completion_flags = off; @@ -371,7 +369,7 @@ inline PTO2RingSegmentOffsets ring_segment_offsets(uint64_t task_window_size) no return ring_segment_offsets(mirror_extents(task_window_size)); } -// Every per-task region starts on a cache line, which PTO2TaskPayload::init's +// Every per-task region starts on a cache line, which TaskPayload::init's // round-up scalar memcpy relies on. ChipTensor is 2 cache lines, so a tensor region // is aligned for any count; the fanin and scalar strides need it stated. static_assert( @@ -490,10 +488,10 @@ inline uint64_t compact_live_image( out_ring.completion_flags = nullptr; const uint64_t nt = used.submitted_tasks; - std::memcpy(out_base + to.descriptors, mirror_base + from.descriptors, nt * sizeof(PTO2TaskDescriptor)); - // One copy, not one per payload: PTO2TaskPayload is fixed-size, so the mirror and + std::memcpy(out_base + to.descriptors, mirror_base + from.descriptors, nt * sizeof(TaskDescriptor)); + // One copy, not one per payload: TaskPayload is fixed-size, so the mirror and // the image share a stride. Each pool is likewise one copy of its own prefix. - std::memcpy(out_base + to.payloads, mirror_base + from.payloads, nt * sizeof(PTO2TaskPayload)); + std::memcpy(out_base + to.payloads, mirror_base + from.payloads, nt * sizeof(TaskPayload)); std::memcpy(out_base + to.slot_states, mirror_base + from.slot_states, nt * sizeof(ChipTaskSlotState)); std::memcpy(out_base + to.completion_flags, mirror_base + from.completion_flags, nt * sizeof(std::atomic)); std::memcpy(out_base + to.fanin_pool, mirror_base + from.fanin_pool, used.fanin_elems * sizeof(int32_t)); @@ -501,9 +499,9 @@ inline uint64_t compact_live_image( std::memcpy(out_base + to.scalar_pool, mirror_base + from.scalar_pool, used.scalar_elems * sizeof(uint64_t)); auto *out_slots = reinterpret_cast(out_base + to.slot_states); - auto *out_descriptors = reinterpret_cast(out_base + to.descriptors); - auto *out_payloads = reinterpret_cast(out_base + to.payloads); - const auto *mirror_payloads = reinterpret_cast(mirror_base + from.payloads); + auto *out_descriptors = reinterpret_cast(out_base + to.descriptors); + auto *out_payloads = reinterpret_cast(out_base + to.payloads); + const auto *mirror_payloads = reinterpret_cast(mirror_base + from.payloads); auto *out_fanin = reinterpret_cast(out_base + to.fanin_pool); auto *out_tensors = reinterpret_cast(out_base + to.tensor_pool); auto *out_scalars = reinterpret_cast(out_base + to.scalar_pool); @@ -517,7 +515,7 @@ inline uint64_t compact_live_image( // re-derived. for (uint64_t i = 0; i < nt; ++i) { out_slots[i].bind_buffers(&out_payloads[i], &out_descriptors[i]); - const PTO2TaskPayload &src = mirror_payloads[i]; + const TaskPayload &src = mirror_payloads[i]; const ChipTensor *src_tensors = src.tensor_data(); const uint64_t *src_scalars = src.scalar_data(); const int32_t *src_fanin = src.fanin_data(); @@ -554,7 +552,7 @@ inline uint64_t compact_live_image( tensors[j].buffer.addr = rebased_heap_addr(tensors[j].buffer.addr, rebase); } } - PTO2TaskDescriptor &out_task = out_descriptors[i]; + TaskDescriptor &out_task = out_descriptors[i]; out_task.packed_buffer_base = reinterpret_cast( rebased_heap_addr(reinterpret_cast(out_task.packed_buffer_base), rebase) ); diff --git a/src/a5/runtime/host_build_graph/runtime/types.h b/src/a5/runtime/host_build_graph/runtime/types.h index ad105719c4..e7ca055336 100644 --- a/src/a5/runtime/host_build_graph/runtime/types.h +++ b/src/a5/runtime/host_build_graph/runtime/types.h @@ -86,7 +86,7 @@ enum class PTO2ScopeMode : uint8_t { * * LIFETIME — single-pass only: * Internally this class stores pointers into the submitting task's tensor - * storage: the region named by PTO2TaskPayload::tensors for a plain submit, the + * storage: the region named by TaskPayload::tensors for a plain submit, the * GraphRecording node's tensors for a submit inside a Graph body. Both belong to * one orchestration pass, which the next bind rebuilds over the same bytes. * Therefore the TaskOutputTensors instance, the const ChipTensor& returned by @@ -692,7 +692,7 @@ inline constexpr uint32_t GRAPH_MAX_SCALAR_ARGS = 64; // outer GRAPH payload carries the whole boundary, while materialize stages only // one node's arguments at a time. The compact boundary values live in that // payload's argument-pool regions, so widening these caps costs pool bytes only -// for Graphs that use them; PTO2TaskPayload itself stays fixed-size. +// for Graphs that use them; TaskPayload itself stays fixed-size. using GraphTaskArgs = Arg; // ChipTaskArgs — chip-level entry-arg holding the orchestration entry's diff --git a/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp b/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp index f9ba4b4687..173bfa9775 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp @@ -167,18 +167,16 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // deferred until AFTER the gate so the scheduler keeps the core // off-limits (pending_occupied stays set) while the task is gated. // src_payload == 0 (ready path) skips this; a non-zero src_payload is - // both the gate flag and the source PTO2TaskPayload. + // both the gate flag and the source TaskPayload. if (exec_payload->src_payload != 0) { __gm__ char *src = reinterpret_cast<__gm__ char *>(exec_payload->src_payload); - int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET); - int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET); - __gm__ uint64_t *src_scalars = - reinterpret_cast<__gm__ uint64_t *>(src + PTO2_TASKPAYLOAD_SCALARS_OFFSET); + int32_t tensor_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_TENSOR_COUNT_OFFSET); + int32_t scalar_count = *reinterpret_cast<__gm__ int32_t *>(src + TASKPAYLOAD_SCALAR_COUNT_OFFSET); + __gm__ uint64_t *src_scalars = reinterpret_cast<__gm__ uint64_t *>(src + TASKPAYLOAD_SCALARS_OFFSET); int n = 0; for (int32_t i = 0; i < tensor_count; i++) { - exec_payload->args[n++] = reinterpret_cast( - src + PTO2_TASKPAYLOAD_TENSORS_OFFSET + i * PTO2_TASKPAYLOAD_TENSOR_STRIDE - ); + exec_payload->args[n++] = + reinterpret_cast(src + TASKPAYLOAD_TENSORS_OFFSET + i * TASKPAYLOAD_TENSOR_STRIDE); } for (int32_t i = 0; i < scalar_count; i++) { exec_payload->args[n++] = src_scalars[i]; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index d0e2dfee5a..24b185d855 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -40,7 +40,7 @@ Type changes: | Field | Before | After | | ----- | ------ | ----- | -| `PTO2TaskDescriptor.task_id` | `int32_t` | `TaskId` | +| `TaskDescriptor.task_id` | `int32_t` | `TaskId` | | `PTO2TensorMapEntry.producer_task_id` | `int32_t` | `TaskId` | | `ChipTaskSlotState.ring_id` | N/A | `uint8_t` (new, denormalized for fast access) | @@ -94,15 +94,15 @@ struct alignas(64) PTO2SharedMemoryRingHeader { uint64_t task_descriptors_offset; // Per-ring data pointers (host-side, set by setup_pointers) - PTO2TaskDescriptor *task_descriptors; - PTO2TaskPayload *task_payloads; + TaskDescriptor *task_descriptors; + TaskPayload *task_payloads; ChipTaskSlotState *slot_states; // Accessors (slot = local_id & task_window_mask) - PTO2TaskDescriptor &get_task_by_slot(int32_t slot); - PTO2TaskDescriptor &get_task_by_task_id(int32_t local_id); - PTO2TaskPayload &get_payload_by_slot(int32_t slot); - PTO2TaskPayload &get_payload_by_task_id(int32_t local_id); + TaskDescriptor &get_task_by_slot(int32_t slot); + TaskDescriptor &get_task_by_task_id(int32_t local_id); + TaskPayload &get_payload_by_slot(int32_t slot); + TaskPayload &get_payload_by_task_id(int32_t local_id); ChipTaskSlotState &get_slot_state_by_slot(int32_t slot); ChipTaskSlotState &get_slot_state_by_task_id(int32_t local_id); }; @@ -166,7 +166,7 @@ bool entry_valid(const PTO2TensorMapEntry& e) { | Structure | Reason | | --------- | ------ | | `PTO2DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | -| `PTO2TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | +| `TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | | `PTO2ReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | | `PTO2DispatchPayload` | Built per-dispatch, no ring state needed | diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index b2158cfe71..bfdfa8e73f 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -49,7 +49,7 @@ The host builds the complete task graph before launching device execution. The o The primary production runtime. Uses ring buffers for task slots and output memory, with a TensorMap for automatic dependency tracking. -- **Task storage**: `PTO2TaskDescriptor[]` in shared memory ring buffer +- **Task storage**: `TaskDescriptor[]` in shared memory ring buffer - **Memory**: GM Heap ring for output buffer allocation - **Dependencies**: automatically derived from tensor read/write patterns via TensorMap - **Thread model**: `aicpu_thread_num - 1` scheduler threads + 1 orchestrator @@ -149,8 +149,8 @@ The orchestrator and schedulers communicate through a contiguous shared memory r │ PTO2SharedMemoryHeader │ (per-ring flow control + layout, global flags) ├─────────────────────────────┤ aligned │ Per-ring regions ×4: │ -│ PTO2TaskDescriptor[N] │ N = task_window_size per ring -│ PTO2TaskPayload[N] │ +│ TaskDescriptor[N] │ N = task_window_size per ring +│ TaskPayload[N] │ │ ChipTaskSlotState[N] │ └─────────────────────────────┘ ``` @@ -378,7 +378,7 @@ When `PTO2OrchestratorState::submit_task` processes parameters: ## 6. Task Descriptor and States -### 6.1 PTO2TaskDescriptor (Hot Path) +### 6.1 TaskDescriptor (Hot Path) | Field | Description | | ----- | ----------- | @@ -393,7 +393,7 @@ When `PTO2OrchestratorState::submit_task` processes parameters: | `packed_buffer_base` | Start of packed buffer in GM Heap | | `packed_buffer_end` | End of packed buffer (for heap reclamation) | -### 6.1b PTO2TaskPayload (Cold Path) +### 6.1b TaskPayload (Cold Path) | Field | Description | | ----- | ----------- | @@ -493,7 +493,7 @@ SIMPLER_SCOPE(rt) { **Output tensor lifetime — single-scope only.** `submit_task` returns a `TaskOutputTensors`, and `get_ref(i)` hands back a `const Tensor&`. Both are -backed by pointers into the submitting task's `PTO2TaskPayload::tensors[]`, +backed by pointers into the submitting task's `TaskPayload::tensors[]`, which lives in a ring-buffer slot. After `scope_end` the slot becomes eligible for reuse; once `advance_ring_pointers` reaches it, `reset_for_reuse()` runs and the next `submit_task` overwrites the same @@ -772,7 +772,7 @@ Instead of polling a shared-memory status flag, the production protocol uses har ### 9.3 PTO2DispatchPayload -Built by the scheduler from `PTO2TaskDescriptor`: +Built by the scheduler from `TaskDescriptor`: | Field | Description | | ----- | ----------- | diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md index b8ffeb52e0..6ac0f2f027 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md @@ -37,7 +37,7 @@ Legacy per-task submit (`kernel_id + worker_type`) cannot express atomic co-disp Design must preserve the current main runtime architecture: 1. Executor threading split (orchestrator thread vs scheduler threads); the orchestrator thread exits after the task graph is built while scheduler threads dispatch to completion. -2. Shared-memory hot/cold split (`PTO2TaskDescriptor` hot + `PTO2TaskPayload` cold). +2. Shared-memory hot/cold split (`TaskDescriptor` hot + `TaskPayload` cold). ## 5. Terminology @@ -86,7 +86,7 @@ Rules: ## 7. Data Model (Requirements + Design) -`PTO2TaskDescriptor` (hot path) carries mixed-task identity/state: +`TaskDescriptor` (hot path) carries mixed-task identity/state: 1. `task_id` 2. `active_mask` @@ -94,7 +94,7 @@ Rules: 4. `kernel_id[3]` for `(AIC, AIV0, AIV1)` 5. dependency heads/counters and packed-buffer metadata -`PTO2TaskPayload` (cold path) carries: +`TaskPayload` (cold path) carries: 1. shared args/tensors/scalars copied once per mixed submit 2. fanin mixed-task IDs diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h index 6526a7f294..fa6b471f82 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h @@ -60,18 +60,18 @@ static_assert( "GLOBAL_CONTEXT_INDEX out of sync with intrinsic.h" ); -// Byte offsets into PTO2TaskPayload used by AICore to materialize args[] on the +// Byte offsets into TaskPayload used by AICore to materialize args[] on the // not_ready (early-dispatch) path. The AICore .o does not include // runtime_types.h, so it reads tensor_count / scalar_count / tensors[] / // scalars[] through these constants. The AICPU side (scheduler_dispatch.cpp) // static_asserts them against offsetof where the full struct is visible, so any -// PTO2TaskPayload layout drift fails the build rather than corrupting args[]. -constexpr uint32_t PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET = 0; -constexpr uint32_t PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET = 4; +// TaskPayload layout drift fails the build rather than corrupting args[]. +constexpr uint32_t TASKPAYLOAD_TENSOR_COUNT_OFFSET = 0; +constexpr uint32_t TASKPAYLOAD_SCALAR_COUNT_OFFSET = 4; // Cache line 9 (byte 576) holds the AICPU-only DispatchPredicate; tensors follow it. -constexpr uint32_t PTO2_TASKPAYLOAD_TENSORS_OFFSET = 640; -constexpr uint32_t PTO2_TASKPAYLOAD_SCALARS_OFFSET = 4736; -constexpr uint32_t PTO2_TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) +constexpr uint32_t TASKPAYLOAD_TENSORS_OFFSET = 640; +constexpr uint32_t TASKPAYLOAD_SCALARS_OFFSET = 4736; +constexpr uint32_t TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) /** * Per-core dispatch payload: function address + args[] + SPMD context. @@ -90,7 +90,7 @@ struct alignas(64) PTO2DispatchPayload { // capacity are cold (prefilled once at init) but ride this hot line for free. // Sized to exactly 64B so both dispatch paths write one control line: the // ready path (src_payload = 0) then also fills args[0..num_args); the gated - // path (src_payload = &PTO2TaskPayload) leaves args[] to the idle AICore. + // path (src_payload = &TaskPayload) leaves args[] to the idle AICore. uint64_t function_bin_addr; /**< Kernel entry address in GM (set by Scheduler). */ /** Per-dispatch context: s_block_idx/s_block_num (hot) + async_ctx (task_token hot, @@ -100,7 +100,7 @@ struct alignas(64) PTO2DispatchPayload { /** Early-dispatch gate AND source pointer, folded into one field. 0 = ready: * AICore executes on pickup (args[] already filled by the AICPU). Non-zero = - * gated: the value is the source PTO2TaskPayload address; the AICore fills + * gated: the value is the source TaskPayload address; the AICore fills * args[0..num_args) from it, then waits for the doorbell (DATA_MAIN_BASE * high 32 == this dispatch's reg_task_id) before executing. A payload address * is never 0, so the gate flag is lossless. */ diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index c2dd39548d..c47fcd1d99 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -438,7 +438,7 @@ void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state void PTO2OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin) { PTO2SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; - PTO2TaskPayload *payload = slot_state.payload; + TaskPayload *payload = slot_state.payload; slot_state.fanin_count = wfanin + 1; int32_t completed_fanin = 0; @@ -529,11 +529,11 @@ static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, ChipTaskSlotS static void scope_tasks_push(PTO2OrchestratorState *orch, ChipTaskSlotState *task_slot_state); -struct PTO2PreparedTask { +struct PreparedTask { TaskId task_id = TaskId::invalid(); PTO2TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; - PTO2TaskDescriptor *task = nullptr; - PTO2TaskPayload *payload = nullptr; + TaskDescriptor *task = nullptr; + TaskPayload *payload = nullptr; ChipTaskSlotState *slot_state = nullptr; }; @@ -586,7 +586,7 @@ static bool check_scope_can_accept_task(PTO2OrchestratorState *orch, PTO2TaskAll static bool prepare_task( PTO2OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, - TaskAttrs task_attrs, PTO2PreparedTask *out + TaskAttrs task_attrs, PreparedTask *out ) { uint8_t ring_id = orch->current_ring_id(); auto &allocator = orch->rings[ring_id].task_allocator; @@ -890,7 +890,7 @@ static TaskOutputTensors submit_task_common( CYCLE_COUNT_START(); TaskOutputTensors result; PTO2OutputLayout layout = calculate_output_layout(args); - PTO2PreparedTask prepared; + PreparedTask prepared; if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; } @@ -899,8 +899,8 @@ static TaskOutputTensors submit_task_common( PTO2RingFlowControl &fc = orch->sm_header->rings[ring_id].fc; TaskId task_id = prepared.task_id; ChipTaskSlotState &cur_slot_state = *prepared.slot_state; - PTO2TaskDescriptor &task = *prepared.task; - PTO2TaskPayload &payload = *prepared.payload; + TaskDescriptor &task = *prepared.task; + TaskPayload &payload = *prepared.payload; result.set_task_id(task_id); // dep_gen capture point: snapshot the orch submit_task inputs while the @@ -1293,15 +1293,15 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) } PTO2OutputLayout layout = calculate_output_layout(args); - PTO2PreparedTask prepared; + PreparedTask prepared; // Kernel-less alloc task: no active subtasks, no dispatch-time attributes. The // early-dispatch hint is force-set below (see the flag-the-creator note). if (!prepare_task(orch, args, layout.total_output_size, ActiveMask{}, TaskAttrs{}, &prepared)) { return TaskOutputTensors{}; } - PTO2TaskDescriptor &task = *prepared.task; - PTO2TaskPayload &payload = *prepared.payload; + TaskDescriptor &task = *prepared.task; + TaskPayload &payload = *prepared.payload; CYCLE_COUNT_LAP(g_orch_alloc_cycle); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index 9539e2e0d0..d731d53da4 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -39,7 +39,7 @@ void PTO2FaninPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_ta int32_t scan_end = sm_last_task_alive; for (int32_t task_id = reclaim_task_cursor; task_id < scan_end; ++task_id) { - PTO2TaskPayload &payload = ring.get_payload_by_task_id(task_id); + TaskPayload &payload = ring.get_payload_by_task_id(task_id); if (payload.fanin_spill_pool != this) { continue; } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 3ac9e95296..ec6a0ed111 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -134,7 +134,7 @@ class PTO2TaskAllocator { * exercise corner cases like task IDs near INT32_MAX. */ void init( - PTO2TaskDescriptor *descriptors, int32_t window_size, std::atomic *current_index_ptr, + TaskDescriptor *descriptors, int32_t window_size, std::atomic *current_index_ptr, std::atomic *last_alive_ptr, void *heap_base, uint64_t heap_size, std::atomic *error_code_ptr, ChipTaskSlotState *slot_states = nullptr, int32_t initial_local_task_id = 0, uint8_t ring_id = 0 ) { @@ -326,7 +326,7 @@ class PTO2TaskAllocator { private: // --- Task Ring --- - PTO2TaskDescriptor *descriptors_ = nullptr; + TaskDescriptor *descriptors_ = nullptr; // Parallel to descriptors_, indexed by task_id & window_mask_. Read-only here, // used by the deadlock detector to identify the head task's slot. ChipTaskSlotState *slot_states_ = nullptr; @@ -384,7 +384,7 @@ class PTO2TaskAllocator { } heap_rebase_anchor_task_id_ = -1; - PTO2TaskDescriptor &desc = descriptors_[(last_alive - 1) & window_mask_]; + TaskDescriptor &desc = descriptors_[(last_alive - 1) & window_mask_]; uint64_t old_tail = heap_tail_; heap_tail_ = static_cast(static_cast(desc.packed_buffer_end) - static_cast(heap_base_)); @@ -742,7 +742,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( } template -inline PTO2FaninForEachReturn for_each_fanin_slot_state(const PTO2TaskPayload &payload, Fn &&fn) { +inline PTO2FaninForEachReturn for_each_fanin_slot_state(const TaskPayload &payload, Fn &&fn) { return for_each_fanin_storage( payload.fanin_inline_edges, payload.fanin_actual_count, payload.fanin_spill_start, *payload.fanin_spill_pool, static_cast(fn) diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 22a9557c13..f20d90e8ba 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -215,7 +215,7 @@ struct PTO2DepListEntry { * * Fields set by Orchestrator at submission, read by Scheduler for dispatch. */ -struct PTO2TaskDescriptor { +struct TaskDescriptor { // Mixed-task identification (encodes ring_id in upper 32 bits) TaskId task_id; // raw: (ring_id << 32) | local_id @@ -229,14 +229,14 @@ struct PTO2TaskDescriptor { // A 4-byte alignment pad follows kernel_id[3]; the scheduler and shared-memory // ABI depend on the descriptor size and packed_buffer_base offset staying fixed. -static_assert(sizeof(PTO2TaskDescriptor) == 40, "PTO2TaskDescriptor size is part of the shared-memory ABI"); -static_assert(offsetof(PTO2TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged"); +static_assert(sizeof(TaskDescriptor) == 40, "TaskDescriptor size is part of the shared-memory ABI"); +static_assert(offsetof(TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged"); // ============================================================================= // Per-Slot Scheduling State // ============================================================================= -// Early-dispatch claim states for PTO2TaskPayload::early_dispatch_state. +// Early-dispatch claim states for TaskPayload::early_dispatch_state. enum PTO2EarlyDispatchState : uint8_t { PTO2_EARLY_DISPATCH_NONE = 0, // not pre-staged PTO2_EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress @@ -274,7 +274,7 @@ inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; * by bulk tensor and scalar data. Small fanins stay fully inline; larger * fanins spill into a per-ring ring buffer slice. */ -struct PTO2TaskPayload { +struct TaskPayload { // === Cache lines 0-8 (576B) — metadata + inline fanin === int32_t tensor_count{0}; int32_t scalar_count{0}; @@ -413,23 +413,21 @@ struct PTO2TaskPayload { } }; -// PTO2TaskPayload layout verification (offsetof requires complete type). -static_assert(offsetof(PTO2TaskPayload, fanin_spill_pool) == 16, "spill pool pointer layout drift"); -static_assert(offsetof(PTO2TaskPayload, fanin_inline_edges) == 24, "inline fanin array must follow spill metadata"); +// TaskPayload layout verification (offsetof requires complete type). +static_assert(offsetof(TaskPayload, fanin_spill_pool) == 16, "spill pool pointer layout drift"); +static_assert(offsetof(TaskPayload, fanin_inline_edges) == 24, "inline fanin array must follow spill metadata"); static_assert( - offsetof(PTO2TaskPayload, predicate) == 576, + offsetof(TaskPayload, predicate) == 576, "dispatch predicate occupies cache line 9 at fixed byte 576 (before tensors, never moves)" ); +static_assert(offsetof(TaskPayload, tensors) == 640, "tensors must start at byte 640 (cache line 10, after predicate)"); static_assert( - offsetof(PTO2TaskPayload, tensors) == 640, "tensors must start at byte 640 (cache line 10, after predicate)" -); -static_assert( - offsetof(PTO2TaskPayload, scalars) == 640 + MAX_TENSOR_ARGS * sizeof(ChipTensor), + offsetof(TaskPayload, scalars) == 640 + MAX_TENSOR_ARGS * sizeof(ChipTensor), "scalars must immediately follow tensors" ); static_assert( - sizeof(PTO2TaskPayload) == 640 + MAX_TENSOR_ARGS * sizeof(ChipTensor) + MAX_SCALAR_ARGS * sizeof(uint64_t), - "PTO2TaskPayload size = metadata(576) + predicate cache line(64) + tensors + scalars" + sizeof(TaskPayload) == 640 + MAX_TENSOR_ARGS * sizeof(ChipTensor) + MAX_SCALAR_ARGS * sizeof(uint64_t), + "TaskPayload size = metadata(576) + predicate cache line(64) + tensors + scalars" ); /** @@ -495,8 +493,8 @@ struct alignas(64) ChipTaskSlotState { // but written here per-submit instead of in an O(window_size) init loop — // these are the only "scale-dependent" pointers in this struct, so moving // them out of init makes startup cost independent of task_window_size. - PTO2TaskPayload *payload; - PTO2TaskDescriptor *task; + TaskPayload *payload; + TaskDescriptor *task; // --- Set per-submit (depend on task inputs) --- ActiveMask active_mask; // Bitmask of active subtask slots (set once) @@ -552,7 +550,7 @@ struct alignas(64) ChipTaskSlotState { * the same 64B slot_state cache line that prepare_task is already * dirtying) to avoid the init-time per-slot loop. */ - void bind_buffers(PTO2TaskPayload *p, PTO2TaskDescriptor *t) { + void bind_buffers(TaskPayload *p, TaskDescriptor *t) { payload = p; task = t; } @@ -612,7 +610,7 @@ struct alignas(64) ChipTaskSlotState { lifecycle_flags.store(PTO2_LIFECYCLE_FLAGS_NONE, std::memory_order_relaxed); // Note: active_mask and task_attrs are per-submit-constant fields // rewritten in prepare_task on every reuse, so they are not reset here. - // Payload early-dispatch fields are initialized by PTO2TaskPayload::init + // Payload early-dispatch fields are initialized by TaskPayload::init // on every submit before the slot becomes scheduler-visible. } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 37f74e50c3..b74aee70c6 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -807,7 +807,7 @@ struct PTO2SchedulerState { return true; } - static inline bool try_claim_early_dispatch_launch(PTO2TaskPayload &payload) { + static inline bool try_claim_early_dispatch_launch(TaskPayload &payload) { uint8_t expected = PTO2_EARLY_DISPATCH_LAUNCH_NONE; return payload.early_dispatch_launch_state.compare_exchange_strong( expected, PTO2_EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst @@ -832,22 +832,22 @@ struct PTO2SchedulerState { } } - static inline bool try_claim_early_sync_drain(PTO2TaskPayload &payload) { + static inline bool try_claim_early_sync_drain(TaskPayload &payload) { uint8_t expected = PTO2_EARLY_SYNC_DRAIN_NONE; return payload.early_sync_drain_state.compare_exchange_strong( expected, PTO2_EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst ); } - static inline bool owns_early_sync_drain(const PTO2TaskPayload &payload) { + static inline bool owns_early_sync_drain(const TaskPayload &payload) { return (payload.early_sync_drain_state.load(std::memory_order_acquire) & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; } - static inline void mark_early_sync_drain_armed(PTO2TaskPayload &payload) { + static inline void mark_early_sync_drain_armed(TaskPayload &payload) { payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); } - static inline bool publish_ready_to_early_sync_drain(PTO2TaskPayload &payload) { + static inline bool publish_ready_to_early_sync_drain(TaskPayload &payload) { uint8_t previous = payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); return (previous & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; @@ -866,7 +866,7 @@ struct PTO2SchedulerState { } } - static inline void finish_early_sync_drain(PTO2TaskPayload &payload) { + static inline void finish_early_sync_drain(TaskPayload &payload) { uint8_t state = payload.early_sync_drain_state.load(std::memory_order_seq_cst); while ((state & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0 && (state & PTO2_EARLY_SYNC_DRAIN_COMPLETE) == 0) { uint8_t desired = state | PTO2_EARLY_SYNC_DRAIN_COMPLETE; @@ -1246,7 +1246,7 @@ struct PTO2SchedulerState { #else int32_t on_task_release(ChipTaskSlotState &slot_state) { #endif - PTO2TaskPayload *payload = slot_state.payload; + TaskPayload *payload = slot_state.payload; int32_t released = 0; // Only DEP_RETAIN edges still hold a fanout pin at completion: an // ordering-only edge released its submit->wire pin at wiring, so releasing diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 7fa809cf0f..2fcc440b05 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -396,7 +396,7 @@ SchedulerContext::StallClassification SchedulerContext::classify_stall_reason() if (cnt_running == 0) { // Snapshot the non-atomic task pointer once: it can be null on a // torn slot, and a concurrent writer may flip it mid-read. - PTO2TaskDescriptor *task_ptr = slot_state.task; + TaskDescriptor *task_ptr = slot_state.task; cls.stuck_task_id = (task_ptr != nullptr) ? static_cast(task_ptr->task_id.raw) : -1; cls.stuck_core = run_core; } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 1f09e12464..6e798ccd04 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -42,12 +42,12 @@ inline constexpr int32_t PTO2_DEFERRED_RELEASE_CAP = 256; } // AICore materializes args[] from src_payload on the gated path using these -// offsets; pin them against the live PTO2TaskPayload layout. -static_assert(offsetof(PTO2TaskPayload, tensor_count) == PTO2_TASKPAYLOAD_TENSOR_COUNT_OFFSET); -static_assert(offsetof(PTO2TaskPayload, scalar_count) == PTO2_TASKPAYLOAD_SCALAR_COUNT_OFFSET); -static_assert(offsetof(PTO2TaskPayload, tensors) == PTO2_TASKPAYLOAD_TENSORS_OFFSET); -static_assert(offsetof(PTO2TaskPayload, scalars) == PTO2_TASKPAYLOAD_SCALARS_OFFSET); -static_assert(sizeof(ChipTensor) == PTO2_TASKPAYLOAD_TENSOR_STRIDE); +// offsets; pin them against the live TaskPayload layout. +static_assert(offsetof(TaskPayload, tensor_count) == TASKPAYLOAD_TENSOR_COUNT_OFFSET); +static_assert(offsetof(TaskPayload, scalar_count) == TASKPAYLOAD_SCALAR_COUNT_OFFSET); +static_assert(offsetof(TaskPayload, tensors) == TASKPAYLOAD_TENSORS_OFFSET); +static_assert(offsetof(TaskPayload, scalars) == TASKPAYLOAD_SCALARS_OFFSET); +static_assert(sizeof(ChipTensor) == TASKPAYLOAD_TENSOR_STRIDE); static_assert(RUNTIME_MAX_WORKER <= PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64); const char *SchedulerContext::shape_name(ResourceShape shape) { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index 30716f0603..7bb8da7fd8 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -55,8 +55,8 @@ void PTO2SharedMemoryHandle::setup_pointers_per_ring(const uint64_t task_window_ for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { auto off = pto2_sm_layout::ring_segment_offsets(task_window_sizes, r); auto &ring = header->rings[r]; - ring.task_descriptors = (PTO2TaskDescriptor *)(base + off.descriptors); - ring.task_payloads = (PTO2TaskPayload *)(base + off.payloads); + ring.task_descriptors = (TaskDescriptor *)(base + off.descriptors); + ring.task_payloads = (TaskPayload *)(base + off.payloads); ring.slot_states = (ChipTaskSlotState *)(base + off.slot_states); } } @@ -151,8 +151,8 @@ void PTO2SharedMemoryHandle::init_header_per_ring( header->rings[r].task_window_mask = static_cast(task_window_sizes[r] - 1); header->rings[r].heap_size = heap_sizes[r]; header->rings[r].task_descriptors_offset = offset; - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 8f072eabd4..421db9da1a 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -93,21 +93,19 @@ struct alignas(64) PTO2SharedMemoryRingHeader { uint64_t task_descriptors_offset; // Offset from SM base, in bytes // Per-ring data pointers (host-side, set by setup_pointers) - PTO2TaskDescriptor *task_descriptors; - PTO2TaskPayload *task_payloads; + TaskDescriptor *task_descriptors; + TaskPayload *task_payloads; ChipTaskSlotState *slot_states; int32_t get_slot_by_task_id(int32_t local_task_id) { return local_task_id & task_window_mask; } - PTO2TaskDescriptor &get_task_by_slot(int32_t slot) { return task_descriptors[slot]; } + TaskDescriptor &get_task_by_slot(int32_t slot) { return task_descriptors[slot]; } - PTO2TaskDescriptor &get_task_by_task_id(int32_t local_id) { - return task_descriptors[get_slot_by_task_id(local_id)]; - } + TaskDescriptor &get_task_by_task_id(int32_t local_id) { return task_descriptors[get_slot_by_task_id(local_id)]; } - PTO2TaskPayload &get_payload_by_slot(int32_t slot) { return task_payloads[slot]; } + TaskPayload &get_payload_by_slot(int32_t slot) { return task_payloads[slot]; } - PTO2TaskPayload &get_payload_by_task_id(int32_t local_id) { return task_payloads[get_slot_by_task_id(local_id)]; } + TaskPayload &get_payload_by_task_id(int32_t local_id) { return task_payloads[get_slot_by_task_id(local_id)]; } ChipTaskSlotState &get_slot_state_by_slot(int32_t slot) { return slot_states[slot]; } @@ -292,15 +290,15 @@ ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "pto2_sm_layout: ring_id out of range"); uint64_t off = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); for (int r = 0; r < ring_id; r++) { - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); } PTO2RingSegmentOffsets o{}; o.descriptors = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(PTO2TaskDescriptor), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); o.payloads = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(PTO2TaskPayload), PTO2_ALIGN_SIZE); + off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); o.slot_states = off; off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); o.end = off; @@ -308,10 +306,10 @@ ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int } // Device address of ring `ring_id`'s task_descriptors array. -inline PTO2TaskDescriptor *ring_task_descriptors_addr( +inline TaskDescriptor *ring_task_descriptors_addr( void *sm_dev_base, const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int ring_id ) noexcept { - return reinterpret_cast( + return reinterpret_cast( static_cast(sm_dev_base) + ring_segment_offsets(task_window_sizes, ring_id).descriptors ); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h index 165bb8808b..2fb26df9d7 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h @@ -121,7 +121,7 @@ constexpr bool dep_has_retain(DepFlags f) { return (f & DEP_RETAIN) != DEP_NONE; * * LIFETIME — single-scope only: * Internally this class stores pointers into the submitting task's payload - * (PTO2TaskPayload::tensors[]), which lives in a ring-buffer slot. After + * (TaskPayload::tensors[]), which lives in a ring-buffer slot. After * scope_end the slot becomes eligible for reuse, and a later submit will * overwrite the same ChipTensor storage in place. Therefore the * TaskOutputTensors instance, the const ChipTensor& returned by get_ref(), and diff --git a/src/common/host_build_graph/graph_execution.cpp b/src/common/host_build_graph/graph_execution.cpp index ee95caf778..9786afd73b 100644 --- a/src/common/host_build_graph/graph_execution.cpp +++ b/src/common/host_build_graph/graph_execution.cpp @@ -42,7 +42,7 @@ GraphExecution *acquire_execution_storage( return execution; } -void reset_graph_payload(PTO2TaskPayload &payload) { +void reset_graph_payload(TaskPayload &payload) { payload.fanin_count = 0; payload.predicate = DispatchPredicate{}; payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_NONE, std::memory_order_relaxed); @@ -236,7 +236,7 @@ bool graph_rebind_tensor( (!own_output && producer_index == node_index)) { return false; } - PTO2TaskDescriptor &producer = execution.node_at(producer_index).task; + TaskDescriptor &producer = execution.node_at(producer_index).task; const uint64_t producer_bytes = static_cast(nodes[producer_index].total_output_size); const uintptr_t producer_base = reinterpret_cast(producer.packed_buffer_base); if (ref.packed_offset > producer_bytes || rebound.buffer_size > producer_bytes - ref.packed_offset || @@ -313,7 +313,7 @@ GraphExecution *graph_execution_localize(ChipTaskSlotState &outer_slot) { auto *definition_header = reinterpret_cast(definition_addr - sizeof(GraphDefinitionHeader)); const GraphDefinition *definition = graph_definition_object_verified(*definition_header); - PTO2TaskPayload &payload = *outer_slot.payload; + TaskPayload &payload = *outer_slot.payload; if (definition == nullptr || definition->total_bytes == 0 || definition->task_count == 0 || definition->task_count > GRAPH_MAX_NODES || payload.tensor_count != static_cast(definition->boundary_count) || @@ -434,8 +434,8 @@ GraphMaterializeResult graph_execution_materialize_slice( storage = new (storage) GraphNodeStorage; execution.constructed_nodes++; } - PTO2TaskDescriptor &task = storage->task; - PTO2TaskPayload &payload = storage->payload; + TaskDescriptor &task = storage->task; + TaskPayload &payload = storage->payload; ChipTaskSlotState &slot = storage->slot; const uint32_t synthetic_local = diff --git a/src/common/host_build_graph/graph_execution.h b/src/common/host_build_graph/graph_execution.h index 8bd4dcc286..caff55de64 100644 --- a/src/common/host_build_graph/graph_execution.h +++ b/src/common/host_build_graph/graph_execution.h @@ -409,13 +409,13 @@ enum class GraphMaterializeResult : uint8_t { }; struct alignas(64) GraphNodeStorage { - PTO2TaskDescriptor task; + TaskDescriptor task; ChipTaskSlotState slot; // The payload carries its argument regions as deltas into pools past the node // array, so its size is the same for every node and the slot names it by a delta // from the slot's own address. Field order here therefore constrains nothing, and // node_at strides the storage by this type. - PTO2TaskPayload payload; + TaskPayload payload; }; inline constexpr uint8_t GRAPH_EXECUTION_STATE_MASK = 0x7; diff --git a/src/common/platform/include/common/args_dump.h b/src/common/platform/include/common/args_dump.h index efe078a1cc..3744e74910 100644 --- a/src/common/platform/include/common/args_dump.h +++ b/src/common/platform/include/common/args_dump.h @@ -263,7 +263,7 @@ struct DumpDataHeader { /** * Caller fills this struct from runtime-specific tensor types. - * Platform layer is agnostic to runtime-specific types (ChipTensor, PTO2TaskPayload, etc.). + * Platform layer is agnostic to runtime-specific types (ChipTensor, TaskPayload, etc.). */ struct ArgsDumpInfo { uint64_t task_id; diff --git a/src/common/task_interface/data_type.h b/src/common/task_interface/data_type.h index d38243e01b..4c7a5816fd 100644 --- a/src/common/task_interface/data_type.h +++ b/src/common/task_interface/data_type.h @@ -175,7 +175,7 @@ inline const char *get_dtype_name(DataType dtype) { * Map a C++ type to its DataType enum value. * * Used to preserve original scalar type information through the - * Arg -> PTO2TaskPayload -> args dump pipeline. + * Arg -> TaskPayload -> args dump pipeline. */ template inline constexpr uint8_t dtype_of() { diff --git a/src/common/task_interface/tensor.h b/src/common/task_interface/tensor.h index 30fd6161de..910aa07f5c 100644 --- a/src/common/task_interface/tensor.h +++ b/src/common/task_interface/tensor.h @@ -470,7 +470,7 @@ struct alignas(64) ChipTensor { } // Friends that need to construct ChipTensors - friend struct PTO2TaskPayload; + friend struct TaskPayload; friend inline ChipTensor make_tensor_external( void *addr, const uint32_t shapes[], uint32_t ndims, DataType dtype, bool manual_dep, int32_t version, AddressSpace address_space diff --git a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp index 8086671672..09bbcf54fd 100644 --- a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp @@ -13,7 +13,7 @@ * * host_build_graph no longer zero-fills the shared-memory task window (init-on-write): * init_header_per_ring writes only the header, and each slot's device-read fields are - * written per task at submit (prepare_task + submit_task_common + PTO2TaskPayload::init). + * written per task at submit (prepare_task + submit_task_common + TaskPayload::init). * Nothing else clears the window, so a device-read field a submit forgets to write would * read as 0 only by allocator accident — passing every zero-backed test and failing * non-deterministically on device. @@ -80,8 +80,8 @@ class HbgSubmitPoisonTest : public ::testing::Test { void poison_window() { auto &ring = sm_handle->header->ring; // host_build_graph is single-ring const size_t n = static_cast(ring.task_window_mask) + 1; - std::memset(ring.task_descriptors, POISON, n * sizeof(PTO2TaskDescriptor)); - std::memset(ring.task_payloads, POISON, n * sizeof(PTO2TaskPayload)); + std::memset(ring.task_descriptors, POISON, n * sizeof(TaskDescriptor)); + std::memset(ring.task_payloads, POISON, n * sizeof(TaskPayload)); std::memset(ring.slot_states, POISON, n * sizeof(ChipTaskSlotState)); std::memset(ring.completion_flags, POISON, n * sizeof(std::atomic)); } @@ -135,8 +135,8 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { for (int32_t local = 0; local < total; local++) { SCOPED_TRACE(testing::Message() << "slot local_id=" << local); const int32_t slot = ring.get_slot_by_task_id(local); - const PTO2TaskDescriptor &desc = ring.task_descriptors[slot]; - const PTO2TaskPayload &pl = ring.task_payloads[slot]; + const TaskDescriptor &desc = ring.task_descriptors[slot]; + const TaskPayload &pl = ring.task_payloads[slot]; const ChipTaskSlotState &st = ring.slot_states[slot]; // Descriptor: the task id is written to this exact local id. @@ -164,8 +164,8 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { } // Field-specific coverage on the real task: tensors, scalar, packed output buffer. - const PTO2TaskDescriptor &root_desc = ring.task_descriptors[ring.get_slot_by_task_id(root.task_id().local())]; - const PTO2TaskPayload &root_pl = ring.task_payloads[ring.get_slot_by_task_id(root.task_id().local())]; + const TaskDescriptor &root_desc = ring.task_descriptors[ring.get_slot_by_task_id(root.task_id().local())]; + const TaskPayload &root_pl = ring.task_payloads[ring.get_slot_by_task_id(root.task_id().local())]; EXPECT_EQ(root_pl.tensor_count, 1); EXPECT_EQ(root_pl.scalar_count, 1); EXPECT_EQ(root_pl.dump_metadata.dump_arg_mask, (uint64_t{1} << 0) | (uint64_t{1} << 1)); @@ -176,7 +176,7 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { EXPECT_EQ(root_desc.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)], 0); // The consumer's fanin is written: two duplicate deps dedupe to one. - const PTO2TaskPayload &cons_pl = ring.task_payloads[ring.get_slot_by_task_id(consumer.task_id().local())]; + const TaskPayload &cons_pl = ring.task_payloads[ring.get_slot_by_task_id(consumer.task_id().local())]; EXPECT_EQ(cons_pl.fanin_count, 1); EXPECT_EQ(cons_pl.fanin_data()[0], static_cast(root.task_id().local())); } diff --git a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp index 7028159b8f..2eb7fff030 100644 --- a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp @@ -179,7 +179,7 @@ TEST_F(OrchestratorFaninTest, DuplicateProducerInSpillRegionDedups) { auto &consumer_slot = sm_handle->header->rings[consumer.task_id().ring()].get_slot_state_by_task_id(consumer.task_id().local()); ASSERT_NE(consumer_slot.payload, nullptr); - PTO2TaskPayload *payload = consumer_slot.payload; + TaskPayload *payload = consumer_slot.payload; EXPECT_EQ(payload->fanin_actual_count, kProducers); // duplicate folded, not 66 TaskId dup = producers.back().task_id(); diff --git a/tests/ut/cpp/a2a3/test_scheduler_state.cpp b/tests/ut/cpp/a2a3/test_scheduler_state.cpp index 447490f1b0..c4ad461157 100644 --- a/tests/ut/cpp/a2a3/test_scheduler_state.cpp +++ b/tests/ut/cpp/a2a3/test_scheduler_state.cpp @@ -62,7 +62,7 @@ class SchedulerStateTest : public ::testing::Test { // mirroring orch::prepare_task's bind_buffers: every production slot has a // payload, and the scheduler's release/propagate paths dereference it. static constexpr int kSlotPayloadPoolSize = 16; - PTO2TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; + TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; int slot_payload_pool_idx_ = 0; void SetUp() override { @@ -96,7 +96,7 @@ class SchedulerStateTest : public ::testing::Test { slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; - PTO2TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; + TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; memset(&slot_pl, 0, sizeof(slot_pl)); slot.payload = &slot_pl; } @@ -106,8 +106,8 @@ class SchedulerStateTest : public ::testing::Test { uint32_t fanout_count = 1, uint32_t fanout_refcount = 1 ) { ChipTaskSlotState &slot = ring.get_slot_state_by_task_id(task_id); - PTO2TaskPayload &payload = ring.get_payload_by_task_id(task_id); - PTO2TaskDescriptor &task = ring.get_task_by_task_id(task_id); + TaskPayload &payload = ring.get_payload_by_task_id(task_id); + TaskDescriptor &task = ring.get_task_by_task_id(task_id); memset(&slot, 0, sizeof(slot)); memset(&payload, 0, sizeof(payload)); memset(&task, 0, sizeof(task)); diff --git a/tests/ut/cpp/a2a3/test_shared_memory.cpp b/tests/ut/cpp/a2a3/test_shared_memory.cpp index abae53add3..aa45f780a0 100644 --- a/tests/ut/cpp/a2a3/test_shared_memory.cpp +++ b/tests/ut/cpp/a2a3/test_shared_memory.cpp @@ -170,14 +170,14 @@ TEST(SharedMemoryLayout, RegionsNonOverlapping) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { uintptr_t desc_start = (uintptr_t)h->header->rings[r].task_descriptors; - uintptr_t desc_end = desc_start + 64 * sizeof(PTO2TaskDescriptor); + uintptr_t desc_end = desc_start + 64 * sizeof(TaskDescriptor); uintptr_t payload_start = (uintptr_t)h->header->rings[r].task_payloads; EXPECT_GE(payload_start, desc_end) << "Ring " << r << ": payload region should not overlap descriptors"; } for (int r = 0; r < CHIP_MAX_RING_DEPTH - 1; r++) { - uintptr_t this_payload_end = (uintptr_t)h->header->rings[r].task_payloads + 64 * sizeof(PTO2TaskPayload); + uintptr_t this_payload_end = (uintptr_t)h->header->rings[r].task_payloads + 64 * sizeof(TaskPayload); uintptr_t next_desc_start = (uintptr_t)h->header->rings[r + 1].task_descriptors; EXPECT_GE(next_desc_start, this_payload_end) << "Ring " << r << " and " << (r + 1) << " should not overlap"; } diff --git a/tests/ut/cpp/a2a3/test_task_allocator.cpp b/tests/ut/cpp/a2a3/test_task_allocator.cpp index c700e0efcc..b693bb4c1a 100644 --- a/tests/ut/cpp/a2a3/test_task_allocator.cpp +++ b/tests/ut/cpp/a2a3/test_task_allocator.cpp @@ -56,8 +56,8 @@ // ============================================================================= static void consume_up_to( - std::vector &descriptors, std::atomic &last_alive, void *heap_base, - int32_t window_size, int32_t new_last_alive, uint64_t heap_tail_offset + std::vector &descriptors, std::atomic &last_alive, void *heap_base, int32_t window_size, + int32_t new_last_alive, uint64_t heap_tail_offset ) { int32_t last_consumed = new_last_alive - 1; descriptors[last_consumed & (window_size - 1)].packed_buffer_end = @@ -74,7 +74,7 @@ class TaskAllocatorTest : public ::testing::Test { static constexpr int32_t WINDOW_SIZE = 16; static constexpr uint64_t HEAP_SIZE = 4096; - std::vector descriptors; + std::vector descriptors; alignas(64) uint8_t heap_buf[HEAP_SIZE]{}; std::atomic current_index{0}; std::atomic last_alive{0}; @@ -82,7 +82,7 @@ class TaskAllocatorTest : public ::testing::Test { PTO2TaskAllocator allocator{}; void SetUp() override { - descriptors.assign(WINDOW_SIZE, PTO2TaskDescriptor{}); + descriptors.assign(WINDOW_SIZE, TaskDescriptor{}); std::memset(heap_buf, 0, sizeof(heap_buf)); current_index.store(0); last_alive.store(0); diff --git a/tests/ut/cpp/a2a3/test_task_state.cpp b/tests/ut/cpp/a2a3/test_task_state.cpp index 12473e280e..5a60677149 100644 --- a/tests/ut/cpp/a2a3/test_task_state.cpp +++ b/tests/ut/cpp/a2a3/test_task_state.cpp @@ -42,7 +42,7 @@ class TaskStateTest : public ::testing::Test { // mirroring orch::prepare_task's bind_buffers: every production slot has a // payload, and the scheduler's release/propagate paths dereference it. static constexpr int kSlotPayloadPoolSize = 16; - PTO2TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; + TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; int slot_payload_pool_idx_ = 0; void SetUp() override { @@ -74,7 +74,7 @@ class TaskStateTest : public ::testing::Test { slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; - PTO2TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; + TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; memset(&slot_pl, 0, sizeof(slot_pl)); slot.payload = &slot_pl; } diff --git a/tests/ut/cpp/a2a3/test_task_timing_slots.cpp b/tests/ut/cpp/a2a3/test_task_timing_slots.cpp index 8c33db349f..9b7689a88d 100644 --- a/tests/ut/cpp/a2a3/test_task_timing_slots.cpp +++ b/tests/ut/cpp/a2a3/test_task_timing_slots.cpp @@ -208,8 +208,8 @@ TEST(TaskTimingSlots, ResolveZeroesIncompleteAndUntagged) { TEST(TaskTimingSlots, DescriptorDoesNotGrow) { // The timing tag moved off the descriptor onto TaskAttrs; the descriptor // (shared-memory ABI) keeps its size and packed_buffer_base offset. - EXPECT_EQ(sizeof(PTO2TaskDescriptor), 40u); - EXPECT_EQ(offsetof(PTO2TaskDescriptor, packed_buffer_base), 24u); + EXPECT_EQ(sizeof(TaskDescriptor), 40u); + EXPECT_EQ(offsetof(TaskDescriptor, packed_buffer_base), 24u); } TEST(TaskTimingSlots, MarkersTimingRoundTrip) { diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index 23f3486b35..17ce61ae8b 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -53,8 +53,8 @@ class WiringTest : public ::testing::Test { // mirroring orch::prepare_task's bind_buffers: every production slot has a // payload, and the scheduler's release/propagate paths dereference it. static constexpr int kSlotPayloadPoolSize = 16; - PTO2TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; - PTO2TaskDescriptor slot_task_pool_[kSlotPayloadPoolSize]; + TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; + TaskDescriptor slot_task_pool_[kSlotPayloadPoolSize]; int slot_payload_pool_idx_ = 0; void SetUp() override { @@ -91,10 +91,10 @@ class WiringTest : public ::testing::Test { slot.total_required_subtasks = 1; slot.logical_block_num = 1; slot.dep_pool_mark = 0; - PTO2TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; + TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; memset(&slot_pl, 0, sizeof(slot_pl)); slot.payload = &slot_pl; - PTO2TaskDescriptor &slot_task = slot_task_pool_[(slot_payload_pool_idx_ - 1) % kSlotPayloadPoolSize]; + TaskDescriptor &slot_task = slot_task_pool_[(slot_payload_pool_idx_ - 1) % kSlotPayloadPoolSize]; memset(&slot_task, 0, sizeof(slot_task)); slot.task = &slot_task; } @@ -123,9 +123,9 @@ class WiringTest : public ::testing::Test { TEST_F(WiringTest, NoFaninTaskBecomesReady) { // A task with 0 actual fanins should immediately be pushed to ready queue alignas(64) ChipTaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); payload.fanin_actual_count = 0; @@ -152,9 +152,9 @@ TEST_F(WiringTest, NoFaninTaskBecomesReady) { TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producer_slots[2]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // Set up 2 producers that are already COMPLETED for (int i = 0; i < 2; i++) { @@ -190,9 +190,9 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producer_slots[2]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // Producers are PENDING (not yet completed) for (int i = 0; i < 2; i++) { @@ -253,9 +253,9 @@ TEST_F(WiringTest, DispatchPropagationMarkerPreservesLifecycleFlagsAndResets) { TEST_F(WiringTest, WireTaskMixedProducerStates) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producers[3]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(producers[0], PTO2_TASK_COMPLETED, 1, 2); // early finished init_slot(producers[1], PTO2_TASK_PENDING, 1, 2); // in flight (< COMPLETED) @@ -295,9 +295,9 @@ TEST_F(WiringTest, WireTaskMixedProducerStates) { TEST_F(WiringTest, WireTaskAllFlaggedPrecompletedSeedsDispatchFanin) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producer_slots[2]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; for (int i = 0; i < 2; i++) { init_slot(producer_slots[i], PTO2_TASK_COMPLETED, 1, 2); @@ -325,9 +325,9 @@ TEST_F(WiringTest, WireTaskUnflaggedPrecompletedProducerDoesNotSeed) { // the consumer become an early-dispatch candidate it should stay off. alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producer; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(producer, PTO2_TASK_COMPLETED, 1, 1); producer.task_attrs.set_early_resolve(false); // unflagged @@ -348,9 +348,9 @@ TEST_F(WiringTest, WireTaskUnflaggedPrecompletedProducerDoesNotSeed) { TEST_F(WiringTest, WireTaskOneUnflaggedProducerDisqualifiesSeed) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producers[2]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(producers[0], PTO2_TASK_COMPLETED, 1, 2); producers[0].task_attrs.set_early_resolve(true); // flagged @@ -375,9 +375,9 @@ TEST_F(WiringTest, EarlyDispatchWaitsForAllProducerBlocksPublished) { // to fanin_actual_count and makes it an early-dispatch candidate. alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producer; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(producer, PTO2_TASK_PENDING, 1, 1); producer.task_attrs.set_early_resolve(true); @@ -504,9 +504,9 @@ TEST_F(WiringTest, EarlyDispatchSyncStartQueueOverflowFallsBackToSyncReadyQueue) TEST_F(WiringTest, LateWiredFullyPublishedProducerStillSeedsEarlyDispatch) { alignas(64) ChipTaskSlotState producer, consumer; - alignas(64) PTO2TaskPayload consumer_payload; + alignas(64) TaskPayload consumer_payload; memset(&consumer_payload, 0, sizeof(consumer_payload)); - PTO2TaskDescriptor consumer_desc{}; + TaskDescriptor consumer_desc{}; init_slot(producer, PTO2_TASK_PENDING, 1, 1); producer.task_attrs.set_early_resolve(true); @@ -531,9 +531,9 @@ TEST_F(WiringTest, LateWiredFullyPublishedProducerStillSeedsEarlyDispatch) { TEST_F(WiringTest, WiringSeedEnqueuesAfterConcurrentPropagation) { alignas(64) ChipTaskSlotState producers[3], consumer; - alignas(64) PTO2TaskPayload consumer_payload; + alignas(64) TaskPayload consumer_payload; memset(&consumer_payload, 0, sizeof(consumer_payload)); - PTO2TaskDescriptor consumer_desc{}; + TaskDescriptor consumer_desc{}; init_slot(producers[0], PTO2_TASK_COMPLETED, 1, 1); init_slot(producers[1], PTO2_TASK_PENDING, 1, 1); @@ -663,7 +663,7 @@ TEST_F(WiringTest, EarlyDispatchClaimStaysGatedAfterRelease) { } TEST_F(WiringTest, EarlyDispatchLaunchHasSingleOwner) { - alignas(64) PTO2TaskPayload payload{}; + alignas(64) TaskPayload payload{}; std::atomic start{false}; bool won[2] = {false, false}; @@ -769,7 +769,7 @@ TEST_F(WiringTest, EarlyDispatchReleaseConsumesDoorbellMask) { } TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { - PTO2TaskPayload payload{}; + TaskPayload payload{}; for (int iteration = 0; iteration < 1000; iteration++) { payload.early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); @@ -886,7 +886,7 @@ TEST_F(WiringTest, ArmedEarlySyncDrainOwnsFinalReadyRoute) { } TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { - alignas(64) PTO2TaskPayload payload{}; + alignas(64) TaskPayload payload{}; payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); ASSERT_TRUE(PTO2SchedulerState::try_claim_early_sync_drain(payload)); @@ -922,7 +922,7 @@ TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { for (bool was_gated : gated) EXPECT_TRUE(was_gated); - alignas(64) PTO2TaskPayload normal_ready{}; + alignas(64) TaskPayload normal_ready{}; normal_ready.early_dispatch_state.store(PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_relaxed); EXPECT_FALSE(PTO2SchedulerState::publish_ready_to_early_sync_drain(normal_ready)); EXPECT_FALSE(PTO2SchedulerState::owns_early_sync_drain(normal_ready)); @@ -998,9 +998,9 @@ TEST_F(WiringTest, ProducerReleaseTransfersReadyRouteToCancellingDrain) { TEST_F(WiringTest, EarlyDispatchBlockedByUnflaggedProducer) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState p_flagged, q_unflagged; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(p_flagged, PTO2_TASK_PENDING, 1, 1); p_flagged.task_attrs.set_early_resolve(true); @@ -1028,7 +1028,7 @@ TEST_F(WiringTest, UnflaggedProducerDoesNotPropagate) { // consumers' dispatch_fanin stays untouched even after it dispatches, and the // once-guard is not even consumed (the gate returns first). alignas(64) ChipTaskSlotState producer, consumer; - alignas(64) PTO2TaskPayload prod_payload, cons_payload; + alignas(64) TaskPayload prod_payload, cons_payload; memset(&prod_payload, 0, sizeof(prod_payload)); memset(&cons_payload, 0, sizeof(cons_payload)); @@ -1059,9 +1059,9 @@ TEST_F(WiringTest, FlaggedPrecompletedCreatorTransparentToEarlyDispatch) { // consumer reaches fanin_actual_count and becomes an early-dispatch candidate. alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState creator, compute; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(creator, PTO2_TASK_COMPLETED, 1, 1); creator.task_attrs.set_early_resolve(true); // alloc creator: flagged -> transparent @@ -1090,9 +1090,9 @@ TEST_F(WiringTest, FlaggedPrecompletedCreatorTransparentToEarlyDispatch) { TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { alignas(64) ChipTaskSlotState producer; alignas(64) ChipTaskSlotState consumer1, consumer2; - alignas(64) PTO2TaskPayload prod_payload; + alignas(64) TaskPayload prod_payload; memset(&prod_payload, 0, sizeof(prod_payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // Producer in flight (PENDING, not yet COMPLETED) with 2 consumers in fanout chain init_slot(producer, PTO2_TASK_PENDING, 1, 1); @@ -1140,9 +1140,9 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { TEST_F(WiringTest, OnTaskReleaseReleasesProducers) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producers[2]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // 2 producers, each COMPLETED with fanout_count=1 for (int i = 0; i < 2; i++) { @@ -1184,9 +1184,9 @@ TEST_F(WiringTest, OrderingOnlyReleasedAtWiringRetentionHeldUntilRelease) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState wait_producer; // DEP_WAIT only (modifier) alignas(64) ChipTaskSlotState retain_producer; // DEP_WAIT|DEP_RETAIN (creator) - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // Both live (PENDING) with a single submit pin (fanout_count = 1). init_slot(wait_producer, PTO2_TASK_PENDING, 1, 1); @@ -1226,9 +1226,9 @@ TEST_F(WiringTest, ReleaseHonorsRetainFlagInSpillRegion) { alignas(64) ChipTaskSlotState filler; // 64 inline DEP_WAIT-only edges alignas(64) ChipTaskSlotState spill_retain; // 1 spilled DEP_RETAIN edge alignas(64) ChipTaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // filler carries a large fanout_count so releasing it can never consume it. init_slot(filler, PTO2_TASK_COMPLETED, 1, 100); @@ -1327,9 +1327,9 @@ TEST_F(WiringTest, AdvanceRingPointersResetsSlots) { TEST_F(WiringTest, NoEdgePublishRecordsDepPoolMark) { alignas(64) ChipTaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); payload.fanin_actual_count = 0; diff --git a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp index 8086671672..09bbcf54fd 100644 --- a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp @@ -13,7 +13,7 @@ * * host_build_graph no longer zero-fills the shared-memory task window (init-on-write): * init_header_per_ring writes only the header, and each slot's device-read fields are - * written per task at submit (prepare_task + submit_task_common + PTO2TaskPayload::init). + * written per task at submit (prepare_task + submit_task_common + TaskPayload::init). * Nothing else clears the window, so a device-read field a submit forgets to write would * read as 0 only by allocator accident — passing every zero-backed test and failing * non-deterministically on device. @@ -80,8 +80,8 @@ class HbgSubmitPoisonTest : public ::testing::Test { void poison_window() { auto &ring = sm_handle->header->ring; // host_build_graph is single-ring const size_t n = static_cast(ring.task_window_mask) + 1; - std::memset(ring.task_descriptors, POISON, n * sizeof(PTO2TaskDescriptor)); - std::memset(ring.task_payloads, POISON, n * sizeof(PTO2TaskPayload)); + std::memset(ring.task_descriptors, POISON, n * sizeof(TaskDescriptor)); + std::memset(ring.task_payloads, POISON, n * sizeof(TaskPayload)); std::memset(ring.slot_states, POISON, n * sizeof(ChipTaskSlotState)); std::memset(ring.completion_flags, POISON, n * sizeof(std::atomic)); } @@ -135,8 +135,8 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { for (int32_t local = 0; local < total; local++) { SCOPED_TRACE(testing::Message() << "slot local_id=" << local); const int32_t slot = ring.get_slot_by_task_id(local); - const PTO2TaskDescriptor &desc = ring.task_descriptors[slot]; - const PTO2TaskPayload &pl = ring.task_payloads[slot]; + const TaskDescriptor &desc = ring.task_descriptors[slot]; + const TaskPayload &pl = ring.task_payloads[slot]; const ChipTaskSlotState &st = ring.slot_states[slot]; // Descriptor: the task id is written to this exact local id. @@ -164,8 +164,8 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { } // Field-specific coverage on the real task: tensors, scalar, packed output buffer. - const PTO2TaskDescriptor &root_desc = ring.task_descriptors[ring.get_slot_by_task_id(root.task_id().local())]; - const PTO2TaskPayload &root_pl = ring.task_payloads[ring.get_slot_by_task_id(root.task_id().local())]; + const TaskDescriptor &root_desc = ring.task_descriptors[ring.get_slot_by_task_id(root.task_id().local())]; + const TaskPayload &root_pl = ring.task_payloads[ring.get_slot_by_task_id(root.task_id().local())]; EXPECT_EQ(root_pl.tensor_count, 1); EXPECT_EQ(root_pl.scalar_count, 1); EXPECT_EQ(root_pl.dump_metadata.dump_arg_mask, (uint64_t{1} << 0) | (uint64_t{1} << 1)); @@ -176,7 +176,7 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { EXPECT_EQ(root_desc.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)], 0); // The consumer's fanin is written: two duplicate deps dedupe to one. - const PTO2TaskPayload &cons_pl = ring.task_payloads[ring.get_slot_by_task_id(consumer.task_id().local())]; + const TaskPayload &cons_pl = ring.task_payloads[ring.get_slot_by_task_id(consumer.task_id().local())]; EXPECT_EQ(cons_pl.fanin_count, 1); EXPECT_EQ(cons_pl.fanin_data()[0], static_cast(root.task_id().local())); } diff --git a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp index 422f612ba0..c703974977 100644 --- a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp @@ -191,7 +191,7 @@ TEST_F(OrchestratorFaninTest, DuplicateProducerInSpillRegionDedups) { auto &consumer_slot = sm_handle->header->rings[consumer.task_id().ring()].get_slot_state_by_task_id(consumer.task_id().local()); ASSERT_NE(consumer_slot.payload, nullptr); - PTO2TaskPayload *payload = consumer_slot.payload; + TaskPayload *payload = consumer_slot.payload; EXPECT_EQ(payload->fanin_actual_count, kProducers); // duplicate folded, not 66 TaskId dup = producers.back().task_id(); diff --git a/tests/ut/cpp/a5/test_scheduler_state.cpp b/tests/ut/cpp/a5/test_scheduler_state.cpp index 1dca08f8e4..1f5f14f48f 100644 --- a/tests/ut/cpp/a5/test_scheduler_state.cpp +++ b/tests/ut/cpp/a5/test_scheduler_state.cpp @@ -62,7 +62,7 @@ class SchedulerStateTest : public ::testing::Test { // mirroring orch::prepare_task's bind_buffers: every production slot has a // payload, and the scheduler's release/propagate paths dereference it. static constexpr int kSlotPayloadPoolSize = 16; - PTO2TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; + TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; int slot_payload_pool_idx_ = 0; void SetUp() override { @@ -96,7 +96,7 @@ class SchedulerStateTest : public ::testing::Test { slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; - PTO2TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; + TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; memset(&slot_pl, 0, sizeof(slot_pl)); slot.payload = &slot_pl; } @@ -106,8 +106,8 @@ class SchedulerStateTest : public ::testing::Test { uint32_t fanout_count = 1, uint32_t fanout_refcount = 1 ) { ChipTaskSlotState &slot = ring.get_slot_state_by_task_id(task_id); - PTO2TaskPayload &payload = ring.get_payload_by_task_id(task_id); - PTO2TaskDescriptor &task = ring.get_task_by_task_id(task_id); + TaskPayload &payload = ring.get_payload_by_task_id(task_id); + TaskDescriptor &task = ring.get_task_by_task_id(task_id); memset(&slot, 0, sizeof(slot)); memset(&payload, 0, sizeof(payload)); memset(&task, 0, sizeof(task)); diff --git a/tests/ut/cpp/a5/test_shared_memory.cpp b/tests/ut/cpp/a5/test_shared_memory.cpp index 27878a869e..ba83ffbad7 100644 --- a/tests/ut/cpp/a5/test_shared_memory.cpp +++ b/tests/ut/cpp/a5/test_shared_memory.cpp @@ -170,14 +170,14 @@ TEST(SharedMemoryLayout, RegionsNonOverlapping) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { uintptr_t desc_start = (uintptr_t)h->header->rings[r].task_descriptors; - uintptr_t desc_end = desc_start + 64 * sizeof(PTO2TaskDescriptor); + uintptr_t desc_end = desc_start + 64 * sizeof(TaskDescriptor); uintptr_t payload_start = (uintptr_t)h->header->rings[r].task_payloads; EXPECT_GE(payload_start, desc_end) << "Ring " << r << ": payload region should not overlap descriptors"; } for (int r = 0; r < CHIP_MAX_RING_DEPTH - 1; r++) { - uintptr_t this_payload_end = (uintptr_t)h->header->rings[r].task_payloads + 64 * sizeof(PTO2TaskPayload); + uintptr_t this_payload_end = (uintptr_t)h->header->rings[r].task_payloads + 64 * sizeof(TaskPayload); uintptr_t next_desc_start = (uintptr_t)h->header->rings[r + 1].task_descriptors; EXPECT_GE(next_desc_start, this_payload_end) << "Ring " << r << " and " << (r + 1) << " should not overlap"; } diff --git a/tests/ut/cpp/a5/test_task_allocator.cpp b/tests/ut/cpp/a5/test_task_allocator.cpp index c700e0efcc..b693bb4c1a 100644 --- a/tests/ut/cpp/a5/test_task_allocator.cpp +++ b/tests/ut/cpp/a5/test_task_allocator.cpp @@ -56,8 +56,8 @@ // ============================================================================= static void consume_up_to( - std::vector &descriptors, std::atomic &last_alive, void *heap_base, - int32_t window_size, int32_t new_last_alive, uint64_t heap_tail_offset + std::vector &descriptors, std::atomic &last_alive, void *heap_base, int32_t window_size, + int32_t new_last_alive, uint64_t heap_tail_offset ) { int32_t last_consumed = new_last_alive - 1; descriptors[last_consumed & (window_size - 1)].packed_buffer_end = @@ -74,7 +74,7 @@ class TaskAllocatorTest : public ::testing::Test { static constexpr int32_t WINDOW_SIZE = 16; static constexpr uint64_t HEAP_SIZE = 4096; - std::vector descriptors; + std::vector descriptors; alignas(64) uint8_t heap_buf[HEAP_SIZE]{}; std::atomic current_index{0}; std::atomic last_alive{0}; @@ -82,7 +82,7 @@ class TaskAllocatorTest : public ::testing::Test { PTO2TaskAllocator allocator{}; void SetUp() override { - descriptors.assign(WINDOW_SIZE, PTO2TaskDescriptor{}); + descriptors.assign(WINDOW_SIZE, TaskDescriptor{}); std::memset(heap_buf, 0, sizeof(heap_buf)); current_index.store(0); last_alive.store(0); diff --git a/tests/ut/cpp/a5/test_task_state.cpp b/tests/ut/cpp/a5/test_task_state.cpp index 12473e280e..5a60677149 100644 --- a/tests/ut/cpp/a5/test_task_state.cpp +++ b/tests/ut/cpp/a5/test_task_state.cpp @@ -42,7 +42,7 @@ class TaskStateTest : public ::testing::Test { // mirroring orch::prepare_task's bind_buffers: every production slot has a // payload, and the scheduler's release/propagate paths dereference it. static constexpr int kSlotPayloadPoolSize = 16; - PTO2TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; + TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; int slot_payload_pool_idx_ = 0; void SetUp() override { @@ -74,7 +74,7 @@ class TaskStateTest : public ::testing::Test { slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; - PTO2TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; + TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; memset(&slot_pl, 0, sizeof(slot_pl)); slot.payload = &slot_pl; } diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index 83c8d7e278..c3a3cab4bc 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -49,8 +49,8 @@ class WiringTest : public ::testing::Test { // mirroring orch::prepare_task's bind_buffers: every production slot has a // payload, and the scheduler's release/propagate paths dereference it. static constexpr int kSlotPayloadPoolSize = 16; - PTO2TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; - PTO2TaskDescriptor slot_task_pool_[kSlotPayloadPoolSize]; + TaskPayload slot_payload_pool_[kSlotPayloadPoolSize]; + TaskDescriptor slot_task_pool_[kSlotPayloadPoolSize]; int slot_payload_pool_idx_ = 0; void SetUp() override { @@ -88,10 +88,10 @@ class WiringTest : public ::testing::Test { slot.total_required_subtasks = 1; slot.logical_block_num = 1; slot.dep_pool_mark = 0; - PTO2TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; + TaskPayload &slot_pl = slot_payload_pool_[slot_payload_pool_idx_++ % kSlotPayloadPoolSize]; memset(&slot_pl, 0, sizeof(slot_pl)); slot.payload = &slot_pl; - PTO2TaskDescriptor &slot_task = slot_task_pool_[(slot_payload_pool_idx_ - 1) % kSlotPayloadPoolSize]; + TaskDescriptor &slot_task = slot_task_pool_[(slot_payload_pool_idx_ - 1) % kSlotPayloadPoolSize]; memset(&slot_task, 0, sizeof(slot_task)); slot.task = &slot_task; } @@ -121,7 +121,7 @@ class WiringTest : public ::testing::Test { }; TEST(ReclaimHeadMatchTest, ComparesExactRingAndLocalTaskId) { - PTO2TaskDescriptor descriptor{}; + TaskDescriptor descriptor{}; ChipTaskSlotState slot{}; slot.task = &descriptor; descriptor.task_id = TaskId::make(0, 16); @@ -139,9 +139,9 @@ TEST(ReclaimHeadMatchTest, ComparesExactRingAndLocalTaskId) { TEST_F(WiringTest, NoFaninTaskBecomesReady) { // A task with 0 actual fanins should immediately be pushed to ready queue alignas(64) ChipTaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); payload.fanin_actual_count = 0; @@ -168,9 +168,9 @@ TEST_F(WiringTest, NoFaninTaskBecomesReady) { TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producer_slots[2]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // Set up 2 producers that are already COMPLETED for (int i = 0; i < 2; i++) { @@ -206,9 +206,9 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producer_slots[2]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // Producers are PENDING (not yet completed) for (int i = 0; i < 2; i++) { @@ -249,9 +249,9 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { TEST_F(WiringTest, WireTaskMixedProducerStates) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producers[3]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(producers[0], PTO2_TASK_COMPLETED, 1, 2); // early finished init_slot(producers[1], PTO2_TASK_PENDING, 1, 2); // in flight (< COMPLETED) @@ -280,7 +280,7 @@ TEST_F(WiringTest, WireTaskMixedProducerStates) { } TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { - PTO2TaskPayload payload{}; + TaskPayload payload{}; for (int iteration = 0; iteration < 1000; iteration++) { payload.early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); @@ -404,9 +404,9 @@ TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { alignas(64) ChipTaskSlotState producer; alignas(64) ChipTaskSlotState consumer1, consumer2; - alignas(64) PTO2TaskPayload prod_payload; + alignas(64) TaskPayload prod_payload; memset(&prod_payload, 0, sizeof(prod_payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // Producer in flight (PENDING, not yet COMPLETED) with 2 consumers in fanout chain init_slot(producer, PTO2_TASK_PENDING, 1, 1); @@ -459,9 +459,9 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { TEST_F(WiringTest, OnTaskReleaseReleasesProducers) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState producers[2]; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // 2 producers, each COMPLETED with fanout_count=1 for (int i = 0; i < 2; i++) { @@ -503,9 +503,9 @@ TEST_F(WiringTest, OrderingOnlyReleasedAtWiringRetentionHeldUntilRelease) { alignas(64) ChipTaskSlotState task_slot; alignas(64) ChipTaskSlotState wait_producer; // DEP_WAIT only (modifier) alignas(64) ChipTaskSlotState retain_producer; // DEP_WAIT|DEP_RETAIN (creator) - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // Both live (PENDING) with a single submit pin (fanout_count = 1). init_slot(wait_producer, PTO2_TASK_PENDING, 1, 1); @@ -545,9 +545,9 @@ TEST_F(WiringTest, ReleaseHonorsRetainFlagInSpillRegion) { alignas(64) ChipTaskSlotState filler; // 64 inline DEP_WAIT-only edges alignas(64) ChipTaskSlotState spill_retain; // 1 spilled DEP_RETAIN edge alignas(64) ChipTaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; // filler carries a large fanout_count so releasing it can never consume it. init_slot(filler, PTO2_TASK_COMPLETED, 1, 100); @@ -939,9 +939,9 @@ TEST_F(WiringTest, AdvanceRingPointersResetsSlots) { TEST_F(WiringTest, NoEdgePublishRecordsDepPoolMark) { alignas(64) ChipTaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; + alignas(64) TaskPayload payload; memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; + TaskDescriptor desc{}; init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); payload.fanin_actual_count = 0; diff --git a/tests/ut/cpp/common/test_hbg_graph_cache.cpp b/tests/ut/cpp/common/test_hbg_graph_cache.cpp index 0327b9bddf..0d3c1d1e90 100644 --- a/tests/ut/cpp/common/test_hbg_graph_cache.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_cache.cpp @@ -252,8 +252,8 @@ class OuterHeap { private: size_t heap_bytes_{0}; std::unique_ptr storage_; - PTO2TaskDescriptor task_{}; - PTO2TaskPayload payload_{}; + TaskDescriptor task_{}; + TaskPayload payload_{}; ChipTaskSlotState slot_{}; std::array boundary_tensors_{}; std::array boundary_scalars_{}; @@ -452,7 +452,7 @@ TEST(GraphExecutionReplay, ResubmissionRebuildsFromDefinition) { heap.initialize_execution(definition_object, reinterpret_cast(first_boundary.data()), 17); ASSERT_NE(execution, nullptr); - PTO2TaskDescriptor outer_task{}; + TaskDescriptor outer_task{}; outer_task.task_id = TaskId::make(1, 7); outer_task.packed_buffer_base = heap.base(); outer_task.packed_buffer_end = heap.end(); @@ -528,7 +528,7 @@ TEST(GraphExecutionReplay, MaterializesBoundaryScalarPoolWiderThanTaskPayload) { heap.initialize_execution(definition_object, reinterpret_cast(boundary.data()), 21); ASSERT_NE(execution, nullptr); - PTO2TaskDescriptor outer_task{}; + TaskDescriptor outer_task{}; outer_task.task_id = TaskId::make(1, 7); outer_task.packed_buffer_base = heap.base(); outer_task.packed_buffer_end = heap.end(); @@ -735,7 +735,7 @@ TEST(GraphExecutionMaterialize, DirtyStorageYieldsValidExecution) { heap.initialize_execution(definition_object, reinterpret_cast(boundary.data()), 17); ASSERT_NE(execution, nullptr); - PTO2TaskDescriptor outer_task{}; + TaskDescriptor outer_task{}; outer_task.task_id = TaskId::make(1, 5); outer_task.packed_buffer_base = heap.base(); outer_task.packed_buffer_end = heap.end(); diff --git a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp index f87f5cbf1e..2d6b1d1dff 100644 --- a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp @@ -743,7 +743,7 @@ TEST_F(HbgGraphSubmitFailureTest, AnOrdinaryAllocationInterleavesWithADeferredSh << "the shell's block sits above the ordinary task's, not before it"; PTO2SharedMemoryRingHeader &ring = sm_handle->header->ring; const int32_t shell_slot = ring.get_slot_by_task_id(static_cast(graph.task_id.local())); - const PTO2TaskDescriptor *shell = ring.slot_states[shell_slot].task.get(); + const TaskDescriptor *shell = ring.slot_states[shell_slot].task.get(); ASSERT_NE(shell, nullptr); ASSERT_NE(shell->packed_buffer_base, nullptr); EXPECT_GE( @@ -863,14 +863,14 @@ TEST_F(HbgGraphSubmitFailureTest, RecordsAGraphWhoseBoundaryLivesInTheHeapWindow EXPECT_LT(reinterpret_cast(heap_resident) + nbytes, GRAPH_RECORD_VIRTUAL_BASE); } -// A hidden-alloc task's payload passes through PTO2TaskPayload::init() and nothing +// A hidden-alloc task's payload passes through TaskPayload::init() and nothing // else — unlike an ordinary ring task, no dispatch-predicate assignment follows it, // and unlike an outer GRAPH task, no graph_reset_outer_payload precedes it. So // init() is where its predicate has to acquire a defined value: the ring's payload // storage is reused raw memory that no constructor runs over, and compact_live_image // translates every submitted slot's predicate.addr as a graph-heap address. TEST_F(HbgGraphSubmitFailureTest, AHiddenAllocTaskLeavesItsDispatchPredicateDefined) { - PTO2TaskPayload *payloads = sm_handle->header->ring.task_payloads; + TaskPayload *payloads = sm_handle->header->ring.task_payloads; ASSERT_NE(payloads, nullptr); // 0x4A repeated has 01 as its top two bits, so read as an address it lands // inside [HEAP_VIRTUAL_BASE, GRAPH_RECORD_VIRTUAL_BASE) — the quarter of the diff --git a/tests/ut/cpp/common/test_hbg_self_relative_ptr.cpp b/tests/ut/cpp/common/test_hbg_self_relative_ptr.cpp index c281d0ccdd..4e8e8b8eb7 100644 --- a/tests/ut/cpp/common/test_hbg_self_relative_ptr.cpp +++ b/tests/ut/cpp/common/test_hbg_self_relative_ptr.cpp @@ -32,17 +32,17 @@ using simpler::hbg::SelfRelativePtr; // its targets in one contiguous span, with a target on each side of the field so // both delta signs are covered. struct Block { - PTO2TaskPayload before_payload; - SelfRelativePtr to_before; - SelfRelativePtr to_after; - PTO2TaskPayload after_payload; + TaskPayload before_payload; + SelfRelativePtr to_before; + SelfRelativePtr to_after; + TaskPayload after_payload; }; } // namespace TEST(HbgSelfRelativePtr, ZeroedMemoryReadsAsUnbound) { - std::vector storage(sizeof(SelfRelativePtr), std::byte{0}); - auto *pointer = reinterpret_cast *>(storage.data()); + std::vector storage(sizeof(SelfRelativePtr), std::byte{0}); + auto *pointer = reinterpret_cast *>(storage.data()); EXPECT_EQ(pointer->get(), nullptr); EXPECT_TRUE(*pointer == nullptr); @@ -98,8 +98,8 @@ TEST(HbgSelfRelativePtr, SurvivesABlockCopyToAnotherAddress) { // part of the same image as its payload and descriptor. TEST(HbgSelfRelativePtr, SlotStateBindingSurvivesTheImageCopy) { struct Image { - PTO2TaskDescriptor descriptor; - PTO2TaskPayload payload; + TaskDescriptor descriptor; + TaskPayload payload; ChipTaskSlotState slot; }; @@ -125,7 +125,7 @@ TEST(HbgSelfRelativePtr, SlotStateBindingSurvivesTheImageCopy) { // tail padding inside the same 64. TEST(HbgSelfRelativePtr, KeepsTheSharedMemoryAbiSizes) { EXPECT_EQ(sizeof(ChipTaskSlotState), 64u); - EXPECT_EQ(sizeof(PTO2TaskDescriptor), 40u); - EXPECT_EQ(sizeof(SelfRelativePtr), 4u); - EXPECT_EQ(offsetof(PTO2TaskDescriptor, packed_buffer_base), 24u); + EXPECT_EQ(sizeof(TaskDescriptor), 40u); + EXPECT_EQ(sizeof(SelfRelativePtr), 4u); + EXPECT_EQ(offsetof(TaskDescriptor, packed_buffer_base), 24u); } diff --git a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp index b0d09abbd4..baea7c9631 100644 --- a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp +++ b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp @@ -133,15 +133,13 @@ class Mirror { const char *base() const { return image_.base(); } - PTO2TaskDescriptor *descriptors() { - return reinterpret_cast( + TaskDescriptor *descriptors() { + return reinterpret_cast( image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).descriptors ); } - PTO2TaskPayload *payloads() { - return reinterpret_cast( - image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).payloads - ); + TaskPayload *payloads() { + return reinterpret_cast(image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).payloads); } ChipTensor *tensor_pool() { return reinterpret_cast(image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).tensor_pool); @@ -200,13 +198,9 @@ struct Compacted { return pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents(usage_for(submitted))); } - PTO2TaskPayload *payload_at(uint64_t i) { - return reinterpret_cast(image.base() + off().payloads) + i; - } - PTO2TaskDescriptor *descriptors() { - return reinterpret_cast(image.base() + off().descriptors); - } - PTO2TaskPayload *payloads() { return payload_at(0); } + TaskPayload *payload_at(uint64_t i) { return reinterpret_cast(image.base() + off().payloads) + i; } + TaskDescriptor *descriptors() { return reinterpret_cast(image.base() + off().descriptors); } + TaskPayload *payloads() { return payload_at(0); } ChipTaskSlotState *slot_states() { return reinterpret_cast(image.base() + off().slot_states); } ChipTensor *tensor_pool() { return reinterpret_cast(image.base() + off().tensor_pool); } std::atomic *completion_flags() { @@ -272,8 +266,8 @@ TEST(HbgSmCompaction, BindingsSurviveTheCopyToTheDevice) { // stop being true the moment a pool moved ahead of them. const auto off = compacted.off(); auto *slots = reinterpret_cast(landed.base() + off.slot_states); - auto *payloads = reinterpret_cast(landed.base() + off.payloads); - auto *descriptors = reinterpret_cast(landed.base() + off.descriptors); + auto *payloads = reinterpret_cast(landed.base() + off.payloads); + auto *descriptors = reinterpret_cast(landed.base() + off.descriptors); for (uint64_t i = 0; i < SUBMITTED; ++i) { EXPECT_EQ(slots[i].payload.get(), &payloads[i]) << "slot " << i; @@ -321,7 +315,7 @@ TEST(HbgSmCompaction, PackedPoolsShipFewerBytesAndStillResolve) { EXPECT_LT(compacted.bytes, pto2_sm_layout::ring_segment_offsets(WINDOW).end); for (uint64_t i = 0; i < SUBMITTED; ++i) { - PTO2TaskPayload *shipped = compacted.payload_at(i); + TaskPayload *shipped = compacted.payload_at(i); EXPECT_EQ(shipped->tensor_count, TENSORS_PER_TASK) << "slot " << i; EXPECT_EQ(shipped->scalar_count, SCALARS_PER_TASK) << "slot " << i; EXPECT_EQ(shipped->fanin_count, FANIN_PER_TASK) << "slot " << i; @@ -358,7 +352,7 @@ TEST(HbgSmCompaction, RebindsEveryArgumentRegionInsideTheImage) { const char *fanin_end = fanin_begin + compacted.used.fanin_elems * sizeof(int32_t); for (uint64_t i = 0; i < SUBMITTED; ++i) { - PTO2TaskPayload *shipped = compacted.payload_at(i); + TaskPayload *shipped = compacted.payload_at(i); const char *t = reinterpret_cast(shipped->tensor_data()); const char *s = reinterpret_cast(shipped->scalar_data()); const char *f = reinterpret_cast(shipped->fanin_data()); @@ -385,7 +379,7 @@ TEST(HbgSmCompaction, UnboundRegionsStayUnbound) { Compacted compacted(mirror); - PTO2TaskPayload *shipped = compacted.payload_at(GRAPH_SLOT); + TaskPayload *shipped = compacted.payload_at(GRAPH_SLOT); EXPECT_EQ(shipped->tensor_data(), nullptr); EXPECT_EQ(shipped->scalar_data(), nullptr); // Its fanin region still resolves, and still inside the image. diff --git a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp index 32fe595fc1..5d99a298fc 100644 --- a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp +++ b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp @@ -23,7 +23,7 @@ constexpr int32_t WINDOW_SIZE = 16; constexpr int32_t POOL_CAPACITY = 8; void make_head_match_old_structural_predicate( - ChipTaskSlotState &head, PTO2TaskDescriptor &descriptor, uint8_t ring_id, uint32_t local_task_id + ChipTaskSlotState &head, TaskDescriptor &descriptor, uint8_t ring_id, uint32_t local_task_id ) { descriptor.task_id = TaskId::make(ring_id, local_task_id); head.task = &descriptor; @@ -44,7 +44,7 @@ TEST(ScopeDeadlockDetectionTest, DepPoolUsesTimeoutForDifferentScopeHead) { } alignas(64) ChipTaskSlotState slot_states[WINDOW_SIZE]{}; - PTO2TaskDescriptor task_descriptors[WINDOW_SIZE]{}; + TaskDescriptor task_descriptors[WINDOW_SIZE]{}; PTO2SharedMemoryRingHeader ring{}; ring.fc.init(); ring.task_window_size = WINDOW_SIZE; @@ -76,7 +76,7 @@ TEST(ScopeDeadlockDetectionTest, DepPoolRejectsCurrentScopeHeadStructurally) { } alignas(64) ChipTaskSlotState slot_states[WINDOW_SIZE]{}; - PTO2TaskDescriptor task_descriptors[WINDOW_SIZE]{}; + TaskDescriptor task_descriptors[WINDOW_SIZE]{}; PTO2SharedMemoryRingHeader ring{}; ring.fc.init(); ring.task_window_size = WINDOW_SIZE; From d6dcc64c2ea02d4791e42e79dc5025ff91a2ba52 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:17:05 -0700 Subject: [PATCH 03/22] Refactor: retire the PTO2 prefix on the shared-memory structs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PTO2SharedMemoryHeader` -> `SharedMemoryHeader`, `PTO2SharedMemoryRingHeader` -> `SharedMemoryRingHeader`, `PTO2SharedMemoryHandle` -> `SharedMemoryHandle`, `PTO2SharedMemory` -> `SharedMemory`, and the `pto2_sm_layout` namespace becomes `sm_layout`. All plain strips; 602 occurrences across 82 files. The namespace is the case rule 9 has in mind when it asks for "clear names or a `namespace`" in place of the prefix: `sm_layout::ring_segment_offsets` is already qualified, so the `pto2_` on the namespace itself was pure noise. This is also where the shared-memory layout diagrams become true, which is what the `ChipTaskSlotState` commit deferred them for. Every row of those boxes now names a type that exists under exactly that spelling — `SharedMemoryHeader`, `TaskDescriptor[]`, `TaskPayload[]`, `ChipTaskSlotState[]` — in all four `runtime/shared_memory.h` copies and in the `sizeof` formula in both `tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md` copies. The two multi-ring boxes widen by four columns because the corrected name no longer fits beside the `Ring N:` prefix. The bare `TaskSlotState` that remains in `docs/orchestrator.md`, `docs/callable-identity-registration.md` and `docs/hierarchical-level-runtime.md` is the host orchestrator's own type and stays as it is. Co-Authored-By: Claude Opus 5 (1M context) --- .../host_build_graph/aicpu/aicpu_executor.cpp | 5 +- .../host_build_graph/host/runtime_maker.cpp | 26 +++--- .../host_build_graph/runtime/orchestrator.h | 2 +- .../orchestrator_core/orchestrator.cpp | 8 +- .../orchestrator_core/runtime_core.cpp | 2 +- .../host_build_graph/runtime/runtime_core.h | 2 +- .../runtime/scheduler/scheduler.h | 10 +-- .../runtime/scheduler/scheduler_cold_path.cpp | 15 ++-- .../runtime/scheduler/scheduler_context.h | 6 +- .../runtime/scheduler/scheduler_dispatch.cpp | 4 +- .../runtime/shared/runtime_init.cpp | 16 ++-- .../runtime/shared/shared_memory.cpp | 44 +++++----- .../host_build_graph/runtime/shared_memory.h | 54 ++++++------ .../aicpu/aicpu_executor.cpp | 4 +- .../docs/MULTI_RING.md | 18 ++-- .../docs/RUNTIME_LOGIC.md | 8 +- .../host/runtime_maker.cpp | 8 +- .../runtime/orchestrator.cpp | 4 +- .../runtime/orchestrator.h | 2 +- .../runtime/ring_buffer.cpp | 12 ++- .../runtime/ring_buffer.h | 10 +-- .../runtime/runtime_core.h | 2 +- .../runtime/scheduler/scheduler.h | 4 +- .../runtime/scheduler/scheduler_cold_path.cpp | 15 ++-- .../runtime/scheduler/scheduler_context.h | 6 +- .../runtime/scheduler/scheduler_dispatch.cpp | 2 +- .../runtime/shared/runtime_init.cpp | 44 +++++----- .../runtime/shared/shared_memory.cpp | 50 ++++++----- .../runtime/shared_memory.h | 85 +++++++++---------- .../host_build_graph/aicpu/aicpu_executor.cpp | 5 +- .../host_build_graph/host/runtime_maker.cpp | 26 +++--- .../host_build_graph/runtime/orchestrator.h | 2 +- .../orchestrator_core/orchestrator.cpp | 8 +- .../orchestrator_core/runtime_core.cpp | 2 +- .../host_build_graph/runtime/runtime_core.h | 2 +- .../runtime/scheduler/scheduler.h | 10 +-- .../runtime/scheduler/scheduler_cold_path.cpp | 15 ++-- .../runtime/scheduler/scheduler_context.h | 6 +- .../runtime/scheduler/scheduler_dispatch.cpp | 4 +- .../runtime/shared/runtime_init.cpp | 16 ++-- .../runtime/shared/shared_memory.cpp | 44 +++++----- .../host_build_graph/runtime/shared_memory.h | 54 ++++++------ .../aicpu/aicpu_executor.cpp | 4 +- .../docs/MULTI_RING.md | 18 ++-- .../docs/RUNTIME_LOGIC.md | 8 +- .../host/runtime_maker.cpp | 8 +- .../runtime/orchestrator.cpp | 4 +- .../runtime/orchestrator.h | 2 +- .../runtime/ring_buffer.cpp | 12 ++- .../runtime/ring_buffer.h | 10 +-- .../runtime/runtime_core.h | 2 +- .../runtime/scheduler/scheduler.h | 4 +- .../runtime/scheduler/scheduler_cold_path.cpp | 15 ++-- .../runtime/scheduler/scheduler_context.h | 6 +- .../runtime/scheduler/scheduler_dispatch.cpp | 2 +- .../runtime/shared/runtime_init.cpp | 44 +++++----- .../runtime/shared/shared_memory.cpp | 50 ++++++----- .../runtime/shared_memory.h | 85 +++++++++---------- tests/ut/cpp/a2a3/test_dep_list_pool.cpp | 2 +- tests/ut/cpp/a2a3/test_fanin_pool.cpp | 2 +- tests/ut/cpp/a2a3/test_graph_activation.cpp | 4 +- tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp | 4 +- tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp | 4 +- tests/ut/cpp/a2a3/test_scheduler_state.cpp | 18 ++-- tests/ut/cpp/a2a3/test_shared_memory.cpp | 46 +++++----- tests/ut/cpp/a2a3/test_task_state.cpp | 4 +- tests/ut/cpp/a2a3/test_wiring.cpp | 4 +- tests/ut/cpp/a5/test_dep_list_pool.cpp | 2 +- tests/ut/cpp/a5/test_fanin_pool.cpp | 2 +- tests/ut/cpp/a5/test_graph_activation.cpp | 4 +- tests/ut/cpp/a5/test_hbg_submit_poison.cpp | 4 +- tests/ut/cpp/a5/test_orchestrator_fanin.cpp | 4 +- tests/ut/cpp/a5/test_scheduler_state.cpp | 22 ++--- tests/ut/cpp/a5/test_shared_memory.cpp | 48 +++++------ tests/ut/cpp/a5/test_task_state.cpp | 4 +- tests/ut/cpp/a5/test_wiring.cpp | 14 +-- tests/ut/cpp/common/test_hbg_graph_cache.cpp | 4 +- .../test_hbg_graph_definition_arena.cpp | 4 +- .../common/test_hbg_graph_submit_failure.cpp | 6 +- tests/ut/cpp/common/test_hbg_slot_claim.cpp | 4 +- .../ut/cpp/common/test_hbg_sm_compaction.cpp | 58 ++++++------- .../common/test_scope_deadlock_detection.cpp | 4 +- .../common/test_trb_runtime_temp_buffer.cpp | 2 +- 83 files changed, 592 insertions(+), 618 deletions(-) diff --git a/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp b/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp index efeb52a0a2..1ab6986a84 100644 --- a/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp +++ b/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp @@ -70,7 +70,7 @@ static int32_t read_pto2_runtime_status(Runtime *runtime) { return 0; } - auto *header = static_cast(sm); + auto *header = static_cast(sm); int32_t orch_error_code = header->orch_error_code.load(std::memory_order_acquire); int32_t sched_error_code = header->sched_error_code.load(std::memory_order_acquire); return runtime_status_from_error_codes(orch_error_code, sched_error_code); @@ -267,8 +267,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { // not to the ring capacity, and the device region holds exactly that // image — so its size comes from the same pitch. attach_populated // rejects a pitch outside (0, capacity] and a region too small for it. - const uint64_t live_slots = - pto2_sm_layout::live_slot_pitch(static_cast(runtime->host_total_tasks)); + const uint64_t live_slots = sm_layout::live_slot_pitch(static_cast(runtime->host_total_tasks)); const uint64_t sm_size = runtime->sm_image_bytes; // sm_handle and the scheduler state are the device-only zone: their // bytes never travel, so they start as whatever the pooled arena last diff --git a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp index e06a0b1e49..c937a90628 100644 --- a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp @@ -294,7 +294,7 @@ static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t // A slot state reaches its payload and descriptor through a 32-bit // self-relative delta, so every pair of addresses in the shared-memory // image must be within INT32_MAX of each other. - const uint64_t sm_bytes = pto2_sm_layout::ring_segment_offsets(*eff_task_window_size).end; + const uint64_t sm_bytes = sm_layout::ring_segment_offsets(*eff_task_window_size).end; if (sm_bytes > static_cast(INT32_MAX)) { LOG_ERROR( "ring_task_window=%" PRIu64 " needs a %" PRIu64 "-byte shared memory image, past the %d-byte limit " @@ -307,7 +307,7 @@ static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t return true; } -static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, PTO2SharedMemoryHeader *host_header) { +static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { if (runtime == nullptr || api == nullptr || host_header == nullptr) { return 0; } @@ -317,7 +317,7 @@ static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, PT return 0; } - int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(PTO2SharedMemoryHeader)); + int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(SharedMemoryHeader)); if (hdr_rc != 0) { LOG_WARN("Failed to copy PTO2 header from device"); return 0; @@ -554,7 +554,7 @@ int32_t run_host_orchestration( // each written per task at submit and read only for [0, total_tasks). Zero // only the fixed-size header here; the per-slot segments are initialized in // orch::prepare_task and shipped bounded to total_tasks below. - const pto2_sm_layout::PTO2RingSegmentOffsets sm_segs = pto2_sm_layout::ring_segment_offsets(eff_task_window_size); + const sm_layout::PTO2RingSegmentOffsets sm_segs = sm_layout::ring_segment_offsets(eff_task_window_size); // Over-allocated and rounded up: every segment offset is a multiple of // PTO2_ALIGN_SIZE and ChipTaskSlotState is alignas(64), which a plain // new uint8_t[] does not guarantee. @@ -585,7 +585,7 @@ int32_t run_host_orchestration( return PTO_RUNTIME_ERR_INTERNAL; } - PTO2SharedMemoryHandle host_sm_handle; + SharedMemoryHandle host_sm_handle; if (!host_sm_handle.init(host_sm, sm_size, eff_task_window_size)) { LOG_ERROR("host-orch: host SM init failed"); return PTO_RUNTIME_ERR_INTERNAL; @@ -671,7 +671,7 @@ int32_t run_host_orchestration( // described — a heap or tensormap exhaustion drops tasks, a fanin overflow drops // edges. Uploading it would launch the device on an incomplete graph and surface // the cause as whatever the device notices second, usually a scheduler timeout. - const int32_t orch_error = pto2_sm_layout::orch_error_code_addr(host_sm)->load(std::memory_order_acquire); + const int32_t orch_error = sm_layout::orch_error_code_addr(host_sm)->load(std::memory_order_acquire); if (orch_error != SIMPLER_ERROR_NONE || orchestrator.fatal) { // The latched code is the diagnosis, so it is what the caller sees — through the // same mapping the run path uses, since a caller cannot tell which of the two @@ -687,7 +687,7 @@ int32_t run_host_orchestration( return status; } - const int32_t total_tasks = pto2_sm_layout::ring_current_task_index_addr(host_sm)->load(std::memory_order_acquire); + const int32_t total_tasks = sm_layout::ring_current_task_index_addr(host_sm)->load(std::memory_order_acquire); { char attrs[160]; snprintf( @@ -745,13 +745,13 @@ int32_t run_host_orchestration( // exact populated extent of each one — no scan of the mirror is needed, and the // image ships that much rather than the worst case the mirror is dimensioned for. const PTO2OrchestratorState &orch_state = orchestrator; - const pto2_sm_layout::BindUsage bind_usage{ + const sm_layout::BindUsage bind_usage{ nt, static_cast(orch_state.fanin_pool_cursor), static_cast(orch_state.tensor_pool_cursor), static_cast(orch_state.scalar_pool_cursor), }; - const uint64_t image_bytes = pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents(bind_usage)).end; + const uint64_t image_bytes = sm_layout::ring_segment_offsets(sm_layout::image_extents(bind_usage)).end; runtime->sm_image_bytes = image_bytes; // Only now are both sizes known, so this is where the two device regions are @@ -832,7 +832,7 @@ int32_t run_host_orchestration( "a Graph node's storage alignment must be covered by the heap region's base alignment" ); always_assert(reinterpret_cast(gm_heap) % DeviceArena::kDefaultBaseAlign == 0); - const pto2_sm_layout::HeapRebase heap_rebase{reinterpret_cast(gm_heap), heap_bytes}; + const sm_layout::HeapRebase heap_rebase{reinterpret_cast(gm_heap), heap_bytes}; // One host source for one copy: the copied zone and shared-memory image at // exactly the offsets they occupy on the device. @@ -852,7 +852,7 @@ int32_t run_host_orchestration( // the pointer goes early rather than at the guard's scope exit. rt->orchestrator = nullptr; std::memcpy(upload_base, static_cast(host_arena.base()) + layout.off_copied_begin, copied_bytes); - const uint64_t compacted = pto2_sm_layout::compact_live_image( + const uint64_t compacted = sm_layout::compact_live_image( static_cast(host_sm), eff_task_window_size, bind_usage, heap_rebase, upload_base + copied_bytes ); always_assert(compacted == image_bytes); @@ -1157,7 +1157,7 @@ extern "C" int bind_callable_to_runtime_impl( // runs — do NOT record in tensor_pairs_; the free is deferred to // DeviceRunner::finalize(). The runtime-arena size is determined by replaying // the reserve sequence on a host-side arena. - uint64_t sm_size = PTO2SharedMemoryHandle::calculate_size(eff_task_window_size); + uint64_t sm_size = SharedMemoryHandle::calculate_size(eff_task_window_size); const int64_t t_arena_build_ns = bind_now_ns(); DeviceArena host_arena; @@ -1288,7 +1288,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e bool skip_tensor_copy_back = execution_rc != 0; int32_t runtime_status = 0; - PTO2SharedMemoryHeader host_header; + SharedMemoryHeader host_header; memset(&host_header, 0, sizeof(host_header)); if (execution_rc != 0) { diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h index a7ecb43fac..48ff9fe880 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h @@ -58,7 +58,7 @@ struct GraphHostState; */ struct PTO2OrchestratorState { // === SHARED MEMORY ACCESS === - PTO2SharedMemoryHeader *sm_header; + SharedMemoryHeader *sm_header; // === TASK / HEAP ALLOCATION === // hbg is single-ring, so one allocator covers the whole graph. diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index a028e1fda3..55e08f0154 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1507,7 +1507,7 @@ static TaskOutputTensors submit_task_common( dep_gen_host_graph_add_explicit_edge(dep_task_id.raw); } uint8_t dep_ring_id = dep_task_id.ring(); - PTO2SharedMemoryRingHeader &dep_ring = orch->sm_header->ring; + SharedMemoryRingHeader &dep_ring = orch->sm_header->ring; int32_t dep_local_task_id = static_cast(dep_task_id.local()); int32_t dep_slot = dep_ring.get_slot_by_task_id(dep_local_task_id); ChipTaskSlotState *producer_slot_state = &dep_ring.get_slot_state_by_slot(dep_slot); @@ -1524,7 +1524,7 @@ static TaskOutputTensors submit_task_common( auto runtime_emit = [&](TaskId producer_task_id) -> bool { uint8_t prod_ring = producer_task_id.ring(); - PTO2SharedMemoryRingHeader &producer_ring = orch->sm_header->ring; + SharedMemoryRingHeader &producer_ring = orch->sm_header->ring; int32_t prod_slot = producer_ring.get_slot_by_task_id(static_cast(producer_task_id.local())); ChipTaskSlotState *prod_state = &producer_ring.get_slot_state_by_slot(prod_slot); return append_fanin_or_fail(orch, prod_ring, prod_slot, prod_state, producer_task_id, &fanin_builder); @@ -1796,7 +1796,7 @@ bool graph_submit_outer( return false; } const TaskId task_id = TaskId::make(0, static_cast(allocation.task_id)); - PTO2SharedMemoryRingHeader &ring = orch->sm_header->ring; + SharedMemoryRingHeader &ring = orch->sm_header->ring; TaskDescriptor &task = ring.task_descriptors[allocation.slot]; TaskPayload &payload = ring.task_payloads[allocation.slot]; ChipTaskSlotState &slot = ring.get_slot_state_by_slot(allocation.slot); @@ -2751,7 +2751,7 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) // every consumer register_wakes on a producer that never runs on device and // the run hangs. (The device watermark walk transparently steps past this // pre-set flag when a later on-device task completes.) - PTO2SharedMemoryRingHeader &done_ring = orch->sm_header->ring; + SharedMemoryRingHeader &done_ring = orch->sm_header->ring; int32_t done_local = static_cast(prepared.task_id.local()); done_ring.set_completion_flag(done_local); } diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 415c6e92eb..4dac7bd3cd 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -247,7 +247,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo // completed_watermark reaches the producer's highest consumer id (set at // submit in append_fanin_or_fail). Replaces the fanout_refcount == // fanout_count wiring check, which polling removes. - PTO2SharedMemoryRingHeader &cons_ring = orch.sm_header->ring; + SharedMemoryRingHeader &cons_ring = orch.sm_header->ring; while (cons_ring.completed_watermark.load(std::memory_order_acquire) < slot.last_consumer_local_id) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h index 4a65e43d1d..77fe42c062 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h @@ -170,7 +170,7 @@ struct RuntimeContext { PTO2ScopeMode pending_scope_mode; // Components - PTO2SharedMemoryHandle *sm_handle; + SharedMemoryHandle *sm_handle; // Host-only, and by pointer so that this header stays trivially copyable: // the orchestrator runs on the host and owns non-trivial scratch. Null on the // device — bind drops it before the copied zone is uploaded, so no device code diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h index 5ff913e8f9..8655213cc0 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -472,12 +472,12 @@ struct PTO2SchedulerLayout { */ struct PTO2SchedulerState { // Shared memory access - PTO2SharedMemoryHeader *sm_header; + SharedMemoryHeader *sm_header; // Per-ring state struct alignas(64) RingSchedState { // --- Cache Line 0: ring pointer (read-only) + hot path (read-write) --- - PTO2SharedMemoryRingHeader *ring; + SharedMemoryRingHeader *ring; std::atomic advance_lock; // multi-thread CAS // Polling: no per-ring dep_pool. Readiness is derived from the SM ring's @@ -586,7 +586,7 @@ struct PTO2SchedulerState { // completion re-scans its waiters via on_mixed_task_complete's wake drain. int classify_fanin_state(const ChipTaskSlotState *s) const { const TaskPayload &p = *s->payload; - const PTO2SharedMemoryRingHeader &ring = *ring_sched_state.ring; + const SharedMemoryRingHeader &ring = *ring_sched_state.ring; const int32_t *fanin = p.fanin_data(); for (int32_t i = p.fanin_count - 1; i >= 0; i--) { if (!ring.is_completion_flag_set(fanin[i])) return i; @@ -599,7 +599,7 @@ struct PTO2SchedulerState { // ready only when every fanin is met, else re-target the next unmet producer // and retry. Monotonic completion_flags guarantee termination. void register_wake(ChipTaskSlotState *producer, ChipTaskSlotState *consumer) { - PTO2SharedMemoryRingHeader &ring = *ring_sched_state.ring; + SharedMemoryRingHeader &ring = *ring_sched_state.ring; while (true) { ChipTaskSlotState *expected = producer->wake_list_head.load(std::memory_order_relaxed); while (expected != WAKE_LIST_SENTINEL) { @@ -627,7 +627,7 @@ struct PTO2SchedulerState { // has no device slot reclaim, so no advance_ring_pointers here. void on_mixed_task_complete(ChipTaskSlotState &slot_state) { const int32_t task_id = static_cast(slot_state.task->task_id.local()); - PTO2SharedMemoryRingHeader &ring = *ring_sched_state.ring; + SharedMemoryRingHeader &ring = *ring_sched_state.ring; slot_state.mark_completed(); // host-visible mirror (task_state = COMPLETED) ring.set_completion_flag(task_id); diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index 3356c9258e..3070d33c4c 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -31,7 +31,7 @@ // Cold-path helpers for the main dispatch loop (noinline to reduce hot-loop icache) // ============================================================================= -static void latch_scheduler_error(PTO2SharedMemoryHeader *header, int32_t thread_idx, int32_t error_code) { +static void latch_scheduler_error(SharedMemoryHeader *header, int32_t thread_idx, int32_t error_code) { if (header == nullptr || error_code == SIMPLER_ERROR_NONE) { return; } @@ -53,7 +53,7 @@ void SchedulerContext::fail_scheduler(Runtime *runtime, int32_t thread_idx, int3 } LoopAction SchedulerContext::handle_orchestrator_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count ) { if (completed_.load(std::memory_order_acquire)) { return LoopAction::BREAK_LOOP; @@ -91,8 +91,7 @@ LoopAction SchedulerContext::handle_orchestrator_exit( return LoopAction::NONE; } -LoopAction -SchedulerContext::check_idle_fatal_error(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime) { +LoopAction SchedulerContext::check_idle_fatal_error(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime) { if (completed_.load(std::memory_order_acquire)) { return LoopAction::BREAK_LOOP; } @@ -232,7 +231,7 @@ void SchedulerContext::log_stall_diagnostics( // produce identical TASK lines once per scheduler thread. if (thread_idx == 0) { int32_t cnt_ready = 0, cnt_waiting = 0, cnt_running = 0, submitted_in_ring = 0; - PTO2SharedMemoryRingHeader &ring = *sched_->ring_sched_state.ring; + SharedMemoryRingHeader &ring = *sched_->ring_sched_state.ring; int32_t ring_task_count = ring.fc.current_task_index.load(std::memory_order_relaxed); submitted_in_ring += ring_task_count; for (int32_t si = 0; si < ring_task_count; si++) { @@ -367,7 +366,7 @@ void SchedulerContext::log_shutdown_stall_snapshot( } int32_t SchedulerContext::handle_timeout_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, int32_t last_progress_count #if SIMPLER_DFX , @@ -908,7 +907,7 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // Initialize task counters. Task count comes from PTO2 shared memory. if (runtime->get_gm_sm_ptr()) { - auto *header = static_cast(runtime->get_gm_sm_ptr()); + auto *header = static_cast(runtime->get_gm_sm_ptr()); // Read at one-time boot init, before the SM is reset for the run, so a ring // not yet written holds uninitialized memory (0xbe... under ASAN's // malloc-fill). Only a plausible count is taken — (0, @@ -1125,7 +1124,7 @@ void SchedulerContext::classify_partition(int32_t thread_idx, int32_t nthreads) if (completed_.load(std::memory_order_acquire) || sched_->ring_sched_state.ring == nullptr) { return; } - PTO2SharedMemoryRingHeader &ring = *sched_->ring_sched_state.ring; + SharedMemoryRingHeader &ring = *sched_->ring_sched_state.ring; const int32_t submitted = ring.fc.current_task_index.load(std::memory_order_acquire); // Disjoint contiguous slices covering [0, submitted): thread t owns // [submitted*t/nthreads, submitted*(t+1)/nthreads). int64 math avoids overflow. diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index 249c3223cd..5ade11d27a 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -521,10 +521,10 @@ class SchedulerContext { // ========================================================================= __attribute__((noinline, cold)) LoopAction - handle_orchestrator_exit(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count); + handle_orchestrator_exit(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count); __attribute__((noinline, cold)) LoopAction - check_idle_fatal_error(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime); + check_idle_fatal_error(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime); __attribute__((noinline, cold)) void log_stall_diagnostics(int32_t thread_idx, int32_t task_count, int32_t idle_iterations, int32_t last_progress_count); @@ -554,7 +554,7 @@ class SchedulerContext { bool no_thread_owns_running_task() const; __attribute__((noinline, cold)) int32_t handle_timeout_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, int32_t last_progress_count #if SIMPLER_DFX , diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index e6fcdcfdbf..38f8b6947d 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -891,7 +891,7 @@ int32_t SchedulerContext::try_early_dispatch( // advanced) — the host's wait_for_consumers never observes a stranded prefix. int32_t SchedulerContext::run_resolution_thread(Runtime *runtime, int32_t thread_idx) { always_assert(sched_ != nullptr); - PTO2SharedMemoryHeader *header = sched_->sm_header; + SharedMemoryHeader *header = sched_->sm_header; if (!header) { LOG_ERROR("PTO2 resolution: header is null"); return -1; @@ -1064,7 +1064,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ always_assert(sched_ != nullptr); CoreTracker &tracker = core_trackers_[thread_idx]; - PTO2SharedMemoryHeader *header = sched_->sm_header; + SharedMemoryHeader *header = sched_->sm_header; if (!header) { LOG_ERROR("PTO2 dispatch: header is null"); return -1; diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 71d205a338..29b9a4195f 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -66,7 +66,7 @@ void ready_queue_destroy(PTO2ReadyQueue *queue) { bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base) { // ring stores the device address of the SM ring header — pure offset // arithmetic, no SM load. - ring = pto2_sm_layout::ring_header_addr(sm_dev_base); + ring = sm_layout::ring_header_addr(sm_dev_base); advance_lock.store(0, std::memory_order_relaxed); // Per-slot SM-side initialization (reset_for_reuse + active_mask, and clearing @@ -107,7 +107,7 @@ bool PTO2SchedulerState::init_data_from_layout( const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base ) { PTO2SchedulerState *sched = this; - sched->sm_header = reinterpret_cast(sm_dev_base); + sched->sm_header = reinterpret_cast(sm_dev_base); #if SIMPLER_SCHED_PROFILING sched->tasks_completed.store(0, std::memory_order_relaxed); sched->tasks_consumed.store(0, std::memory_order_relaxed); @@ -208,12 +208,12 @@ bool PTO2OrchestratorState::init( // A power-of-two window lets pto2_task_slot() mask instead of dividing. always_assert(task_window_size > 0 && (task_window_size & (task_window_size - 1)) == 0); - orch->sm_header = reinterpret_cast(sm_base); + orch->sm_header = reinterpret_cast(sm_base); orch->fatal = false; orch->scheduler = scheduler_arg; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_base); - auto *cur_idx_dev = pto2_sm_layout::ring_current_task_index_addr(sm_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_base); + auto *cur_idx_dev = sm_layout::ring_current_task_index_addr(sm_base); orch->task_allocator.init(static_cast(task_window_size), cur_idx_dev, gm_heap, heap_size, orch_err); @@ -221,7 +221,7 @@ bool PTO2OrchestratorState::init( // so it holds for whichever SM this orchestrator was pointed at. The cursors // reset with the rest of the state above. auto *sm_bytes = static_cast(sm_base); - const auto pools = pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::mirror_extents(task_window_size)); + const auto pools = sm_layout::ring_segment_offsets(sm_layout::mirror_extents(task_window_size)); orch->fanin_pool = reinterpret_cast(sm_bytes + pools.fanin_pool); orch->tensor_pool = reinterpret_cast(sm_bytes + pools.tensor_pool); orch->scalar_pool = reinterpret_cast(sm_bytes + pools.scalar_pool); @@ -263,7 +263,7 @@ RuntimeArenaLayout runtime_reserve_layout(DeviceArena &arena, uint64_t task_wind // // The copied zone comes last of the two so the device's shared-memory tail can // begin where it ends, making the two adjacent and the upload one copy. - layout.off_sm_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + layout.off_sm_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); layout.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); layout.off_scheduler = arena.reserve(sizeof(PTO2SchedulerState), alignof(PTO2SchedulerState)); layout.sched = PTO2SchedulerState::reserve_layout(arena); @@ -317,7 +317,7 @@ RuntimeContext *runtime_init_data_from_layout( } void runtime_wire_arena_pointers(DeviceArena &arena, const RuntimeArenaLayout &layout, RuntimeContext *rt) { - rt->sm_handle = static_cast(arena.region_ptr(layout.off_sm_handle)); + rt->sm_handle = static_cast(arena.region_ptr(layout.off_sm_handle)); rt->aicore_mailbox = static_cast(arena.region_ptr(layout.off_mailbox)); rt->scheduler = static_cast(arena.region_ptr(layout.off_scheduler)); rt->scheduler->wire_arena_pointers(layout.sched, arena); diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 8e098d78f2..91b187b733 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -27,29 +27,29 @@ // Size Calculation // ============================================================================= -uint64_t PTO2SharedMemoryHandle::calculate_size(uint64_t task_window_size) { +uint64_t SharedMemoryHandle::calculate_size(uint64_t task_window_size) { // Total SM size = offset just past the ring's slot_states, from the single - // source of truth for the layout (pto2_sm_layout::ring_segment_offsets). - return pto2_sm_layout::ring_segment_offsets(task_window_size).end; + // source of truth for the layout (sm_layout::ring_segment_offsets). + return sm_layout::ring_segment_offsets(task_window_size).end; } // ============================================================================= // Creation and Destruction // ============================================================================= -void PTO2SharedMemoryHandle::setup_pointers(uint64_t pitch) { +void SharedMemoryHandle::setup_pointers(uint64_t pitch) { char *base = (char *)sm_base; - header = (PTO2SharedMemoryHeader *)base; + header = (SharedMemoryHeader *)base; // Per-ring descriptors / payloads / slot_states — offsets from the single source - // of truth (pto2_sm_layout::ring_segment_offsets), so this setup and the + // of truth (sm_layout::ring_segment_offsets), so this setup and the // device-address helpers cannot drift. // // Only the four slot-pitched segments, and no argument pool: the pool extents // differ between the mirror and the image, while these four depend on the pitch // alone. The orchestrator holds the mirror's pool bases; the device resolves an // argument region through the payload's delta and needs no base at all. - auto off = pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents({pitch, 0, 0, 0})); + auto off = sm_layout::ring_segment_offsets(sm_layout::image_extents({pitch, 0, 0, 0})); auto &ring = header->ring; ring.task_descriptors = (TaskDescriptor *)(base + off.descriptors); ring.task_payloads = (TaskPayload *)(base + off.payloads); @@ -57,7 +57,7 @@ void PTO2SharedMemoryHandle::setup_pointers(uint64_t pitch) { ring.completion_flags = (std::atomic *)(base + off.completion_flags); } -bool PTO2SharedMemoryHandle::init(void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size) { +bool SharedMemoryHandle::init(void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size) { if (!sm_base_arg || sm_size_arg == 0) return false; const uint64_t mirror_bytes = calculate_size(task_window_size); // The mirror has to stay inside an int32 delta's reach: a payload names its @@ -78,7 +78,7 @@ bool PTO2SharedMemoryHandle::init(void *sm_base_arg, uint64_t sm_size_arg, uint6 return true; } -bool PTO2SharedMemoryHandle::attach_populated( +bool SharedMemoryHandle::attach_populated( void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size, uint64_t live_slots, uint64_t image_bytes ) { if (!sm_base_arg || sm_size_arg == 0) return false; @@ -91,7 +91,7 @@ bool PTO2SharedMemoryHandle::attach_populated( // that is its argument pools, whose extents are the bind's cursors and are not // recomputable here. The first pool's offset is exactly where those four end. const uint64_t slots_end = - pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents({live_slots, 0, 0, 0})).fanin_pool; + sm_layout::ring_segment_offsets(sm_layout::image_extents({live_slots, 0, 0, 0})).fanin_pool; if (image_bytes < slots_end) return false; // The region holds the compacted image, so that is what has to fit — the ring // capacity is no longer its size. @@ -110,13 +110,13 @@ bool PTO2SharedMemoryHandle::attach_populated( return true; } -PTO2SharedMemoryHandle *PTO2SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { +SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); - const size_t off_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(buffer_size), PTO2_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; - auto *handle = static_cast(arena.region_ptr(off_handle)); + auto *handle = static_cast(arena.region_ptr(off_handle)); memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); memset(buffer, 0, static_cast(buffer_size)); @@ -124,7 +124,7 @@ PTO2SharedMemoryHandle *PTO2SharedMemoryHandle::create_and_init_default(DeviceAr return handle; } -void PTO2SharedMemoryHandle::destroy() { +void SharedMemoryHandle::destroy() { // Arena-owned wrappers (is_owner == false) are reclaimed by arena.release(); // calling destroy on them is a no-op so existing callers stay safe. if (is_owner && sm_base) { @@ -138,7 +138,7 @@ void PTO2SharedMemoryHandle::destroy() { // ============================================================================= // // no need init data in pool, init pool data when used -void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size) { +void SharedMemoryHandle::init_header(uint64_t task_window_size) { // Flow control (starts at 0) header->ring.fc.init(); @@ -149,7 +149,7 @@ void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size) { header->orchestrator_done.store(0, std::memory_order_relaxed); // Ring layout info - uint64_t offset = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t offset = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); header->ring.task_window_size = task_window_size; header->ring.task_window_mask = static_cast(task_window_size - 1); header->ring.task_descriptors_offset = offset; @@ -176,10 +176,10 @@ void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size) { // Debug Utilities // ============================================================================= -void PTO2SharedMemoryHandle::print_layout() { +void SharedMemoryHandle::print_layout() { if (!header) return; - PTO2SharedMemoryHeader *h = header; + SharedMemoryHeader *h = header; LOG_DEBUG("=== PTO2 Shared Memory Layout ==="); LOG_DEBUG("Base address: %p", sm_base); @@ -200,22 +200,22 @@ void PTO2SharedMemoryHandle::print_layout() { LOG_DEBUG("================================"); } -bool PTO2SharedMemoryHandle::validate() { +bool SharedMemoryHandle::validate() { if (!sm_base) return false; if (!header) return false; - PTO2SharedMemoryHeader *h = header; + SharedMemoryHeader *h = header; if (!h->ring.fc.validate(this)) return false; return true; } -bool PTO2RingFlowControl::validate(PTO2SharedMemoryHandle *handle) const { +bool PTO2RingFlowControl::validate(SharedMemoryHandle *handle) const { if (!handle) return false; if (!handle->header) return false; - const PTO2SharedMemoryHeader *h = handle->header; + const SharedMemoryHeader *h = handle->header; // Check that offsets are within bounds if (h->ring.task_descriptors_offset >= h->total_size) return false; diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h index 32582ad0aa..3a76a415fa 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h @@ -19,7 +19,7 @@ * +---------------------------+ * | TaskDescriptor[] | * | TaskPayload[] | - * | TaskSlotState[] | + * | ChipTaskSlotState[] | * +---------------------------+ * * Design principles: @@ -44,7 +44,7 @@ // Shared Memory Header // ============================================================================= -struct PTO2SharedMemoryHandle; +struct SharedMemoryHandle; /** * Per-ring flow control state in shared memory. @@ -64,7 +64,7 @@ struct alignas(64) PTO2RingFlowControl { // submit IDs will be off by the divergence. void init() { current_task_index.store(0, std::memory_order_relaxed); } - bool validate(PTO2SharedMemoryHandle *handle) const; + bool validate(SharedMemoryHandle *handle) const; }; static_assert(sizeof(PTO2RingFlowControl) == 64, "PTO2RingFlowControl must be exactly one cache line (64B)"); @@ -75,7 +75,7 @@ static_assert(sizeof(PTO2RingFlowControl) == 64, "PTO2RingFlowControl must be ex * Groups flow-control, layout info, and per-ring data pointers for a single ring. * Pointers are host-side only (set by setup_pointers, invalid on device). */ -struct alignas(64) PTO2SharedMemoryRingHeader { +struct alignas(64) SharedMemoryRingHeader { PTO2RingFlowControl fc; // Highest task_id such that every task with id in [0, completed_watermark] @@ -154,10 +154,10 @@ struct alignas(64) PTO2SharedMemoryRingHeader { } }; -static_assert(sizeof(PTO2SharedMemoryRingHeader) == 192, "PTO2SharedMemoryRingHeader layout drift"); +static_assert(sizeof(SharedMemoryRingHeader) == 192, "SharedMemoryRingHeader layout drift"); static_assert( - offsetof(PTO2SharedMemoryRingHeader, task_descriptors_offset) == 144, - "PTO2SharedMemoryRingHeader task_descriptors_offset layout drift" + offsetof(SharedMemoryRingHeader, task_descriptors_offset) == 144, + "SharedMemoryRingHeader task_descriptors_offset layout drift" ); /** @@ -165,9 +165,9 @@ static_assert( * * Contains per-ring flow control and global layout information. */ -struct alignas(PTO2_ALIGN_SIZE) PTO2SharedMemoryHeader { +struct alignas(PTO2_ALIGN_SIZE) SharedMemoryHeader { // === RING FLOW CONTROL + LAYOUT INFO (single ring, set once at init) === - PTO2SharedMemoryRingHeader ring; + SharedMemoryRingHeader ring; // === GLOBAL FIELDS === std::atomic orchestrator_done; // Flag: orchestration complete @@ -188,11 +188,9 @@ struct alignas(PTO2_ALIGN_SIZE) PTO2SharedMemoryHeader { std::atomic sched_error_thread; // Thread index of last error writer }; -static_assert(sizeof(PTO2SharedMemoryHeader) == 256, "PTO2SharedMemoryHeader layout drift"); -static_assert(offsetof(PTO2SharedMemoryHeader, total_size) == 200, "PTO2SharedMemoryHeader total_size layout drift"); -static_assert( - offsetof(PTO2SharedMemoryHeader, orch_error_code) == 208, "PTO2SharedMemoryHeader orch_error_code layout drift" -); +static_assert(sizeof(SharedMemoryHeader) == 256, "SharedMemoryHeader layout drift"); +static_assert(offsetof(SharedMemoryHeader, total_size) == 200, "SharedMemoryHeader total_size layout drift"); +static_assert(offsetof(SharedMemoryHeader, orch_error_code) == 208, "SharedMemoryHeader orch_error_code layout drift"); // ============================================================================= // Shared Memory Handle @@ -200,13 +198,13 @@ static_assert( /** * Handle for shared memory lifecycle management (create/destroy). - * Runtime components (orchestrator, scheduler) use PTO2SharedMemoryHeader* directly. + * Runtime components (orchestrator, scheduler) use SharedMemoryHeader* directly. */ -struct PTO2SharedMemoryHandle { +struct SharedMemoryHandle { void *sm_base; // Base address of shared memory uint64_t sm_size; // Total size of shared memory - PTO2SharedMemoryHeader *header; + SharedMemoryHeader *header; // Ownership flag bool is_owner; // True if this handle allocated the memory @@ -219,7 +217,7 @@ struct PTO2SharedMemoryHandle { // default PTO2_TASK_WINDOW_SIZE. Only valid when the arena is otherwise empty // (the call performs the single commit). All memory is owned by the arena — // caller must not call destroy(). - static PTO2SharedMemoryHandle *create_and_init_default(DeviceArena &arena); + static SharedMemoryHandle *create_and_init_default(DeviceArena &arena); // === Instance methods === @@ -276,25 +274,25 @@ struct PTO2SharedMemoryHandle { // // These helpers compute those addresses by offset arithmetic on the SM // device base. Pure pointer math, no loads/stores; safe to call from host. -// The same arithmetic happens on AICPU too (via PTO2SharedMemoryHandle's +// The same arithmetic happens on AICPU too (via SharedMemoryHandle's // own setup_pointers), so values are guaranteed consistent across sides. -namespace pto2_sm_layout { +namespace sm_layout { inline std::atomic *orch_error_code_addr(void *sm_dev_base) noexcept { return reinterpret_cast *>( - static_cast(sm_dev_base) + offsetof(PTO2SharedMemoryHeader, orch_error_code) + static_cast(sm_dev_base) + offsetof(SharedMemoryHeader, orch_error_code) ); } -inline PTO2SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base) noexcept { - return reinterpret_cast( - static_cast(sm_dev_base) + offsetof(PTO2SharedMemoryHeader, ring) +inline SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base) noexcept { + return reinterpret_cast( + static_cast(sm_dev_base) + offsetof(SharedMemoryHeader, ring) ); } inline std::atomic *ring_current_task_index_addr(void *sm_dev_base) noexcept { return reinterpret_cast *>( - reinterpret_cast(ring_header_addr(sm_dev_base)) + offsetof(PTO2SharedMemoryRingHeader, fc) + + reinterpret_cast(ring_header_addr(sm_dev_base)) + offsetof(SharedMemoryRingHeader, fc) + offsetof(PTO2RingFlowControl, current_task_index) ); } @@ -345,7 +343,7 @@ inline RingImageExtents mirror_extents(uint64_t task_window_size) noexcept { // or reordering a segment is a one-line edit here; every consumer follows // automatically, so the layout walk can never silently disagree across call sites. inline PTO2RingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) noexcept { - uint64_t off = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); PTO2RingSegmentOffsets o{}; o.descriptors = off; off += PTO2_ALIGN_UP(e.slots * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); @@ -481,7 +479,7 @@ inline uint64_t compact_live_image( // The header and the descriptors offset are pitch-independent, so the header // lands where it already was. std::memcpy(out_base, mirror_base, to.descriptors); - auto &out_ring = reinterpret_cast(out_base)->ring; + auto &out_ring = reinterpret_cast(out_base)->ring; out_ring.task_descriptors = nullptr; out_ring.task_payloads = nullptr; out_ring.slot_states = nullptr; @@ -563,4 +561,4 @@ inline uint64_t compact_live_image( return to.end; } -} // namespace pto2_sm_layout +} // namespace sm_layout diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index 81be57acde..b9eb0cc4c8 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -89,7 +89,7 @@ static int32_t read_runtime_status(Runtime *runtime) { return 0; } - auto *header = static_cast(sm); + auto *header = static_cast(sm); int32_t orch_error_code = header->orch_error_code.load(std::memory_order_acquire); int32_t sched_error_code = header->sched_error_code.load(std::memory_order_acquire); return runtime_status_from_error_codes(orch_error_code, sched_error_code); @@ -611,7 +611,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { // Wire every arena-internal pointer field (host wrote host-mirror // addresses; we overwrite them with device addresses). runtime_wire_arena_pointers(runtime_arena_, rt->prebuilt_layout, rt); - sm_size = PTO2SharedMemoryHandle::calculate_size_per_ring(rt->prebuilt_layout.sizing.task_window_sizes); + sm_size = SharedMemoryHandle::calculate_size_per_ring(rt->prebuilt_layout.sizing.task_window_sizes); } // Reset SM state. setup_pointers + init_header_per_ring restore diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 55d6d0675e..0895420d8e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -72,7 +72,7 @@ PTO2RingSet rings[CHIP_MAX_RING_DEPTH]; Ring selection: `current_ring_id() = min(scope_stack_top, CHIP_MAX_RING_DEPTH - 1)`. -### 4.3 PTO2SharedMemoryHeader (modified) +### 4.3 SharedMemoryHeader (modified) Per-ring flow control and per-ring layout info are grouped together: @@ -84,7 +84,7 @@ struct PTO2RingFlowControl { std::atomic heap_tail; // heap reclaim pointer }; -struct alignas(64) PTO2SharedMemoryRingHeader { +struct alignas(64) SharedMemoryRingHeader { PTO2RingFlowControl fc; // Layout metadata (set once at init) @@ -108,20 +108,20 @@ struct alignas(64) PTO2SharedMemoryRingHeader { }; // In header: -PTO2SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; +SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; ``` -Per-ring try-locks in the scheduler state prevent concurrent scheduler threads from interleaving watermark writes within the same ring. `FaninPool`/`DepListPool` `reclaim`/`ensure_space` take `PTO2SharedMemoryRingHeader&` directly (no `ring_id` or `fc` parameters). +Per-ring try-locks in the scheduler state prevent concurrent scheduler threads from interleaving watermark writes within the same ring. `FaninPool`/`DepListPool` `reclaim`/`ensure_space` take `SharedMemoryRingHeader&` directly (no `ring_id` or `fc` parameters). -### 4.4 PTO2SharedMemoryHandle (lifecycle-only) +### 4.4 SharedMemoryHandle (lifecycle-only) -Slimmed to lifecycle management only. Per-ring data pointers now live in `PTO2SharedMemoryRingHeader` (§4.3). Runtime components (orchestrator, scheduler) store `PTO2SharedMemoryHeader*` directly, eliminating one indirection on every per-ring access. +Slimmed to lifecycle management only. Per-ring data pointers now live in `SharedMemoryRingHeader` (§4.3). Runtime components (orchestrator, scheduler) store `SharedMemoryHeader*` directly, eliminating one indirection on every per-ring access. ```cpp -struct PTO2SharedMemoryHandle { +struct SharedMemoryHandle { void *sm_base; uint64_t sm_size; - PTO2SharedMemoryHeader *header; + SharedMemoryHeader *header; bool is_owner; }; ``` @@ -131,7 +131,7 @@ struct PTO2SharedMemoryHandle { ```cpp struct RingSchedState { // Cache Line 0: ring pointer (read-only) + hot path (read-write) - PTO2SharedMemoryRingHeader *ring; // direct pointer, no indirection + SharedMemoryRingHeader *ring; // direct pointer, no indirection int32_t last_task_alive; std::atomic advance_lock; // multi-thread CAS diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index a2fcd1610e..b5bb017121 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -145,7 +145,7 @@ The orchestrator and schedulers communicate through a contiguous shared memory r ```text ┌─────────────────────────────┐ offset 0 -│ PTO2SharedMemoryHeader │ (per-ring flow control + layout, global flags) +│ SharedMemoryHeader │ (per-ring flow control + layout, global flags) ├─────────────────────────────┤ aligned │ Per-ring regions ×4: │ │ TaskDescriptor[N] │ N = task_window_size per ring @@ -163,8 +163,8 @@ The orchestrator and schedulers communicate through a contiguous shared memory r | `heap_top` | Orchestrator | Scheduler | Heap ring allocation pointer | | `heap_tail` | Scheduler | Orchestrator | Heap ring reclamation pointer | | `orchestrator_done` | Orchestrator | Scheduler | Signals orchestration completion | -| `task_window_size` | Init | Both | Number of task slots (per-ring, in `PTO2SharedMemoryRingHeader`) | -| `heap_size` | Init | Both | Heap total size (per-ring, in `PTO2SharedMemoryRingHeader`) | +| `task_window_size` | Init | Both | Number of task slots (per-ring, in `SharedMemoryRingHeader`) | +| `heap_size` | Init | Both | Heap total size (per-ring, in `SharedMemoryRingHeader`) | | `task_descriptors_offset` | Init | Both | Offset to TaskDescriptor array in SM (per-ring) | | `total_size` | Init | Both | Total shared memory size | @@ -174,7 +174,7 @@ The orchestrator and schedulers communicate through a contiguous shared memory r total = ALIGN(Header) + Σ_ring [ ALIGN(window_size * sizeof(TaskDescriptor)) + ALIGN(window_size * sizeof(TaskPayload)) - + ALIGN(window_size * sizeof(TaskSlotState)) ] + + ALIGN(window_size * sizeof(ChipTaskSlotState)) ] ``` Alignment is 64 bytes (`PTO2_ALIGN_SIZE`). diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index a808dc0484..f260f66392 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -284,7 +284,7 @@ static bool resolve_ring_config( return true; } -static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, PTO2SharedMemoryHeader *host_header) { +static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { if (runtime == nullptr || host_header == nullptr) { return 0; } @@ -294,7 +294,7 @@ static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, PT return 0; } - int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(PTO2SharedMemoryHeader)); + int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(SharedMemoryHeader)); if (hdr_rc != 0) { LOG_WARN("Failed to copy PTO2 header from device"); return 0; @@ -570,7 +570,7 @@ static bool derive_arena_static_sizes(const ArenaSizingConfig &sizing, ArenaStat } out->total_heap += sizing.heap_sizes[r]; } - out->sm_size = PTO2SharedMemoryHandle::calculate_size_per_ring(sizing.task_window_sizes); + out->sm_size = SharedMemoryHandle::calculate_size_per_ring(sizing.task_window_sizes); return true; } @@ -1017,7 +1017,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e bool skip_tensor_copy_back = execution_rc != 0; int32_t runtime_status = 0; - PTO2SharedMemoryHeader host_header; + SharedMemoryHeader host_header; memset(&host_header, 0, sizeof(host_header)); if (execution_rc != 0) { diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index f7870d23e3..7d7da9adcc 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -965,7 +965,7 @@ static TaskOutputTensors submit_task_common( return result; } uint8_t dep_ring_id = dep_task_id.ring(); - PTO2SharedMemoryRingHeader &dep_ring = orch->sm_header->rings[dep_ring_id]; + SharedMemoryRingHeader &dep_ring = orch->sm_header->rings[dep_ring_id]; int32_t dep_local_task_id = static_cast(dep_task_id.local()); int32_t dep_last_task_alive = dep_ring.fc.last_task_alive.load(std::memory_order_acquire); if (dep_local_task_id < dep_last_task_alive) { @@ -989,7 +989,7 @@ static TaskOutputTensors submit_task_common( auto runtime_emit = [&](TaskId producer_task_id, DepFlags kind) -> bool { uint8_t prod_ring = producer_task_id.ring(); - PTO2SharedMemoryRingHeader &producer_ring = orch->sm_header->rings[prod_ring]; + SharedMemoryRingHeader &producer_ring = orch->sm_header->rings[prod_ring]; int32_t prod_slot = producer_ring.get_slot_by_task_id(static_cast(producer_task_id.local())); ChipTaskSlotState *prod_state = &producer_ring.get_slot_state_by_slot(prod_slot); return append_fanin_or_fail( diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 50409074a5..4cc060dcd1 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -64,7 +64,7 @@ struct PTO2OrchestratorLayout { */ struct PTO2OrchestratorState { // === SHARED MEMORY ACCESS === - PTO2SharedMemoryHeader *sm_header; + SharedMemoryHeader *sm_header; // === PER-RING RESOURCES === PTO2RingSet rings[CHIP_MAX_RING_DEPTH]; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index 6a5115e34d..d6e1ae427a 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -34,7 +34,7 @@ static void latch_pool_error(std::atomic *error_code_ptr, int32_t error // ============================================================================= // Fanin Spill Pool Implementation // ============================================================================= -void PTO2FaninPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { +void PTO2FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { if (sm_last_task_alive <= reclaim_task_cursor) return; int32_t scan_end = sm_last_task_alive; @@ -53,7 +53,7 @@ void PTO2FaninPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_ta reclaim_task_cursor = scan_end; } -bool PTO2FaninPool::ensure_space(PTO2SharedMemoryRingHeader &ring, int32_t needed) { +bool PTO2FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { if (available() >= needed) return true; int spin_count = 0; @@ -124,7 +124,7 @@ bool PTO2FaninPool::ensure_space(PTO2SharedMemoryRingHeader &ring, int32_t neede // ============================================================================= // Dependency List Pool Implementation // ============================================================================= -void PTO2DepListPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { +void PTO2DepListPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { if (sm_last_task_alive >= last_reclaimed + PTO2_DEP_POOL_CLEANUP_INTERVAL && sm_last_task_alive > 0) { int32_t mark = ring.get_slot_state_by_task_id(sm_last_task_alive - 1).dep_pool_mark; if (mark > 0) { @@ -135,7 +135,7 @@ void PTO2DepListPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_ } void PTO2DepListPool::report_deadlock( - PTO2SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated + SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated ) { int32_t current = ring.fc.current_task_index.load(std::memory_order_acquire); LOG_ERROR("========================================"); @@ -176,9 +176,7 @@ void PTO2DepListPool::report_deadlock( LOG_ERROR("========================================"); } -bool PTO2DepListPool::ensure_space( - PTO2SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task -) { +bool PTO2DepListPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task) { if (available() >= needed) return true; int spin_count = 0; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 8652006c7f..b2fac8e344 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -533,9 +533,9 @@ struct PTO2FaninPool { error_code_ptr = in_error_code_ptr; } - void reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_task_alive); + void reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive); - bool ensure_space(PTO2SharedMemoryRingHeader &ring, int32_t needed); + bool ensure_space(SharedMemoryRingHeader &ring, int32_t needed); PTO2FaninSpillEntry *alloc() { int32_t used = top - tail; @@ -719,7 +719,7 @@ struct PTO2DepListPool { * @param ring Ring header (for reading slot dep_pool_mark) * @param sm_last_task_alive Current last_task_alive from shared memory */ - void reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_task_alive); + void reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive); /** * Ensure dep pool for a specific ring has at least `needed` entries available. @@ -728,14 +728,14 @@ struct PTO2DepListPool { * reclaim watermark the same way PTO2TaskAllocator::alloc does: a structural * head-of-line check plus a wall-clock backstop, each emitting report_deadlock. */ - bool ensure_space(PTO2SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task = nullptr); + bool ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task = nullptr); /** * Structured dep-pool deadlock report, mirroring PTO2TaskAllocator::report_deadlock. * scope_gated marks the provable head-of-line case where the head is pinned * by an open scope on this ring, as opposed to the wall-clock backstop. */ - void report_deadlock(PTO2SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated); + void report_deadlock(SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated); /** * Allocate a single entry from the pool (single-thread per pool instance) diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index 8b5d20e828..fcf6251129 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -156,7 +156,7 @@ struct RuntimeContext { PTO2ScopeMode pending_scope_mode; // Components - PTO2SharedMemoryHandle *sm_handle; + SharedMemoryHandle *sm_handle; PTO2OrchestratorState orchestrator; PTO2SchedulerState scheduler; AICoreCompletionMailbox *aicore_mailbox; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 47ccf0e13a..b68f78e046 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -443,12 +443,12 @@ struct PTO2SchedulerLayout { */ struct PTO2SchedulerState { // Shared memory access - PTO2SharedMemoryHeader *sm_header; + SharedMemoryHeader *sm_header; // Per-ring state struct alignas(64) RingSchedState { // --- Cache Line 0: ring pointer (read-only) + hot path (read-write) --- - PTO2SharedMemoryRingHeader *ring; + SharedMemoryRingHeader *ring; int32_t last_task_alive; std::atomic advance_lock; // multi-thread CAS diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index d02801a3db..142a0496b4 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -38,7 +38,7 @@ // Returns true iff this call won the first-writer CAS for sched_error_code — the // caller may then write companion fields (e.g. the stall detail) knowing they // describe the same observation that owns the latched code. -static bool latch_scheduler_error(PTO2SharedMemoryHeader *header, int32_t thread_idx, int32_t error_code) { +static bool latch_scheduler_error(SharedMemoryHeader *header, int32_t thread_idx, int32_t error_code) { if (header == nullptr || error_code == SIMPLER_ERROR_NONE) { return false; } @@ -55,7 +55,7 @@ static bool latch_scheduler_error(PTO2SharedMemoryHeader *header, int32_t thread } LoopAction SchedulerContext::handle_orchestrator_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count ) { if (completed_.load(std::memory_order_acquire)) { return LoopAction::BREAK_LOOP; @@ -92,8 +92,7 @@ LoopAction SchedulerContext::handle_orchestrator_exit( return LoopAction::NONE; } -LoopAction -SchedulerContext::check_idle_fatal_error(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime) { +LoopAction SchedulerContext::check_idle_fatal_error(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime) { if (completed_.load(std::memory_order_acquire)) { return LoopAction::BREAK_LOOP; } @@ -230,7 +229,7 @@ void SchedulerContext::log_stall_diagnostics( if (thread_idx == 0) { int32_t cnt_ready = 0, cnt_waiting = 0, cnt_running = 0, submitted_in_ring = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - PTO2SharedMemoryRingHeader &ring = *sched_->ring_sched_states[r].ring; + SharedMemoryRingHeader &ring = *sched_->ring_sched_states[r].ring; int32_t ring_task_count = ring.fc.current_task_index.load(std::memory_order_relaxed); submitted_in_ring += ring_task_count; // Scan only live task_ids [last_task_alive, current_task_index): slots @@ -366,7 +365,7 @@ SchedulerContext::StallClassification SchedulerContext::classify_stall_reason() cls.stuck_core = -1; int32_t cnt_running = 0, cnt_ready = 0, cnt_waiting = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - PTO2SharedMemoryRingHeader &ring = *sched_->ring_sched_states[r].ring; + SharedMemoryRingHeader &ring = *sched_->ring_sched_states[r].ring; int32_t ring_task_count = ring.fc.current_task_index.load(std::memory_order_relaxed); // Active task_ids live in [last_task_alive, current_task_index); slots wrap // (slot = task_id % window), so scanning from 0 re-reads each live slot once @@ -417,7 +416,7 @@ SchedulerContext::StallClassification SchedulerContext::classify_stall_reason() } int32_t SchedulerContext::handle_timeout_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, int32_t last_progress_count #if SIMPLER_DFX , @@ -1180,7 +1179,7 @@ int32_t SchedulerContext::pre_handshake_init( // hs_setup_done_, which zeroes these ring counters, so the read completes here // (on the leader, before any thread is released) rather than post-handshake. if (runtime->get_gm_sm_ptr()) { - auto *header = static_cast(runtime->get_gm_sm_ptr()); + auto *header = static_cast(runtime->get_gm_sm_ptr()); int64_t pto2_count = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { int32_t ring_tasks = header->rings[r].fc.current_task_index.load(std::memory_order_acquire); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 9d4f0ab8f3..bca0e7079a 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -474,10 +474,10 @@ class SchedulerContext { // ========================================================================= __attribute__((noinline, cold)) LoopAction - handle_orchestrator_exit(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count); + handle_orchestrator_exit(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count); __attribute__((noinline, cold)) LoopAction - check_idle_fatal_error(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime); + check_idle_fatal_error(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime); __attribute__((noinline, cold)) void log_stall_diagnostics(int32_t thread_idx, int32_t task_count, int32_t idle_iterations, int32_t last_progress_count); @@ -528,7 +528,7 @@ class SchedulerContext { __attribute__((noinline, cold)) StallClassification classify_stall_reason() const; __attribute__((noinline, cold)) int32_t handle_timeout_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, int32_t last_progress_count #if SIMPLER_DFX , diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 88feeaa8bf..a935608aee 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -893,7 +893,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ always_assert(sched_ != nullptr); CoreTracker &tracker = core_trackers_[thread_idx]; - PTO2SharedMemoryHeader *header = sched_->sm_header; + SharedMemoryHeader *header = sched_->sm_header; if (!header) { LOG_ERROR("PTO2 dispatch: header is null"); return -1; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 5bf019b2f5..302da4e1c2 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -88,7 +88,7 @@ void ready_queue_destroy(PTO2ReadyQueue *queue) { bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base, int32_t ring_id) { // ring stores the device address of the SM ring header — pure offset // arithmetic, no SM load. - ring = pto2_sm_layout::ring_header_addr(sm_dev_base, ring_id); + ring = sm_layout::ring_header_addr(sm_dev_base, ring_id); last_task_alive = 0; advance_lock.store(0, std::memory_order_relaxed); #if SIMPLER_DFX @@ -97,7 +97,7 @@ bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base #endif // Per-slot SM-side initialization (bind_ring + reset_for_reuse + - // fanin_count/active_mask zero) lives in PTO2SharedMemoryHandle:: + // fanin_count/active_mask zero) lives in SharedMemoryHandle:: // init_header_per_ring so the AICPU performs it during SM reset; host // prebuilt-arena init skips SM access here. @@ -107,7 +107,7 @@ bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base void PTO2SchedulerState::RingSchedState::reset_for_reuse( void *sm_dev_base, int32_t ring_id, std::atomic *orch_err ) { - ring = pto2_sm_layout::ring_header_addr(sm_dev_base, ring_id); + ring = sm_layout::ring_header_addr(sm_dev_base, ring_id); last_task_alive = 0; advance_lock.store(0, std::memory_order_relaxed); dep_pool.reset_for_reuse(orch_err); @@ -159,7 +159,7 @@ bool PTO2SchedulerState::init_data_from_layout( const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base ) { PTO2SchedulerState *sched = this; - sched->sm_header = reinterpret_cast(sm_dev_base); + sched->sm_header = reinterpret_cast(sm_dev_base); sched->advance_pending_mask.store(0, std::memory_order_relaxed); #if SIMPLER_SCHED_PROFILING sched->tasks_completed.store(0, std::memory_order_relaxed); @@ -206,7 +206,7 @@ bool PTO2SchedulerState::init_data_from_layout( return false; } - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_dev_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { auto *dep_entries = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); memset(dep_entries, 0, static_cast(layout.dep_pool_capacities[r]) * sizeof(PTO2DepListEntry)); @@ -218,14 +218,14 @@ bool PTO2SchedulerState::init_data_from_layout( void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void *sm_dev_base) { PTO2SchedulerState *sched = this; - sched->sm_header = reinterpret_cast(sm_dev_base); + sched->sm_header = reinterpret_cast(sm_dev_base); sched->advance_pending_mask.store(0, std::memory_order_relaxed); #if SIMPLER_SCHED_PROFILING sched->tasks_completed.store(0, std::memory_order_relaxed); sched->tasks_consumed.store(0, std::memory_order_relaxed); #endif - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_dev_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { sched->ring_sched_states[r].reset_for_reuse(sm_dev_base, r, orch_err); } @@ -364,7 +364,7 @@ bool PTO2OrchestratorState::init_data_from_layout( auto *orch = this; *orch = PTO2OrchestratorState{}; - orch->sm_header = reinterpret_cast(sm_dev_base); + orch->sm_header = reinterpret_cast(sm_dev_base); orch->gm_heap_base = gm_heap; uint64_t total_heap_size = 0; if (!sum_ring_heap_sizes(heap_sizes, &total_heap_size)) { @@ -373,14 +373,14 @@ bool PTO2OrchestratorState::init_data_from_layout( orch->gm_heap_size = total_heap_size; orch->fatal = false; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_dev_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); uint64_t heap_offset = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { void *ring_heap_base = reinterpret_cast(gm_heap) + heap_offset; - auto *task_descs_dev = pto2_sm_layout::ring_task_descriptors_addr(sm_dev_base, task_window_sizes, r); - auto *slot_states_dev = pto2_sm_layout::ring_slot_states_addr(sm_dev_base, task_window_sizes, r); - auto *cur_idx_dev = pto2_sm_layout::ring_current_task_index_addr(sm_dev_base, r); - auto *last_alive_dev = pto2_sm_layout::ring_last_task_alive_addr(sm_dev_base, r); + auto *task_descs_dev = sm_layout::ring_task_descriptors_addr(sm_dev_base, task_window_sizes, r); + auto *slot_states_dev = sm_layout::ring_slot_states_addr(sm_dev_base, task_window_sizes, r); + auto *cur_idx_dev = sm_layout::ring_current_task_index_addr(sm_dev_base, r); + auto *last_alive_dev = sm_layout::ring_last_task_alive_addr(sm_dev_base, r); orch->rings[r].task_allocator.init( task_descs_dev, static_cast(task_window_sizes[r]), cur_idx_dev, last_alive_dev, ring_heap_base, @@ -421,7 +421,7 @@ bool PTO2OrchestratorState::reset_for_reuse( const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ) { auto *orch = this; - orch->sm_header = reinterpret_cast(sm_dev_base); + orch->sm_header = reinterpret_cast(sm_dev_base); orch->gm_heap_base = gm_heap; uint64_t total_heap_size = 0; if (!sum_ring_heap_sizes(heap_sizes, &total_heap_size)) { @@ -443,14 +443,14 @@ bool PTO2OrchestratorState::reset_for_reuse( } orch->fanin_seen_current_epoch = next_epoch; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_dev_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); uint64_t heap_offset = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { void *ring_heap_base = reinterpret_cast(gm_heap) + heap_offset; - auto *task_descs_dev = pto2_sm_layout::ring_task_descriptors_addr(sm_dev_base, task_window_sizes, r); - auto *slot_states_dev = pto2_sm_layout::ring_slot_states_addr(sm_dev_base, task_window_sizes, r); - auto *cur_idx_dev = pto2_sm_layout::ring_current_task_index_addr(sm_dev_base, r); - auto *last_alive_dev = pto2_sm_layout::ring_last_task_alive_addr(sm_dev_base, r); + auto *task_descs_dev = sm_layout::ring_task_descriptors_addr(sm_dev_base, task_window_sizes, r); + auto *slot_states_dev = sm_layout::ring_slot_states_addr(sm_dev_base, task_window_sizes, r); + auto *cur_idx_dev = sm_layout::ring_current_task_index_addr(sm_dev_base, r); + auto *last_alive_dev = sm_layout::ring_last_task_alive_addr(sm_dev_base, r); orch->rings[r].task_allocator.init( task_descs_dev, static_cast(task_window_sizes[r]), cur_idx_dev, last_alive_dev, ring_heap_base, @@ -531,7 +531,7 @@ RuntimeArenaLayout runtime_reserve_layout( layout.sizing.dep_pool_capacities[r] = dep_pool_capacities[r]; } - layout.offsets.off_sm_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + layout.offsets.off_sm_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); int32_t task_window_sizes_i32[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { task_window_sizes_i32[r] = static_cast(task_window_sizes[r]); @@ -563,7 +563,7 @@ RuntimeContext *runtime_init_data_from_layout( RuntimeContext *rt = static_cast(arena.region_ptr(layout.offsets.off_runtime)); memset(rt, 0, sizeof(*rt)); - auto *sm_wrap = static_cast(arena.region_ptr(layout.offsets.off_sm_handle)); + auto *sm_wrap = static_cast(arena.region_ptr(layout.offsets.off_sm_handle)); memset(sm_wrap, 0, sizeof(*sm_wrap)); // rt->ops is filled by the AICPU at boot. @@ -593,7 +593,7 @@ RuntimeContext *runtime_init_data_from_layout( } void runtime_wire_arena_pointers(DeviceArena &arena, const RuntimeArenaLayout &layout, RuntimeContext *rt) { - rt->sm_handle = static_cast(arena.region_ptr(layout.offsets.off_sm_handle)); + rt->sm_handle = static_cast(arena.region_ptr(layout.offsets.off_sm_handle)); rt->aicore_mailbox = static_cast(arena.region_ptr(layout.offsets.off_mailbox)); rt->orchestrator.wire_arena_pointers(layout.offsets.orch, arena, &rt->scheduler); rt->scheduler.wire_arena_pointers(layout.offsets.sched, arena); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index 7bb8da7fd8..0912b8aa53 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -27,7 +27,7 @@ // Size Calculation // ============================================================================= -uint64_t PTO2SharedMemoryHandle::calculate_size(uint64_t task_window_size) { +uint64_t SharedMemoryHandle::calculate_size(uint64_t task_window_size) { uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { task_window_sizes[r] = task_window_size; @@ -35,25 +35,25 @@ uint64_t PTO2SharedMemoryHandle::calculate_size(uint64_t task_window_size) { return calculate_size_per_ring(task_window_sizes); } -uint64_t PTO2SharedMemoryHandle::calculate_size_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]) { +uint64_t SharedMemoryHandle::calculate_size_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]) { // Total SM size = offset just past the last ring, from the single source of - // truth for the layout (pto2_sm_layout::ring_segment_offsets). - return pto2_sm_layout::ring_segment_offsets(task_window_sizes, CHIP_MAX_RING_DEPTH - 1).end; + // truth for the layout (sm_layout::ring_segment_offsets). + return sm_layout::ring_segment_offsets(task_window_sizes, CHIP_MAX_RING_DEPTH - 1).end; } // ============================================================================= // Creation and Destruction // ============================================================================= -void PTO2SharedMemoryHandle::setup_pointers_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]) { +void SharedMemoryHandle::setup_pointers_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]) { char *base = (char *)sm_base; - header = (PTO2SharedMemoryHeader *)base; + header = (SharedMemoryHeader *)base; // Per-ring descriptors / payloads / slot_states — offsets from the single - // source of truth (pto2_sm_layout::ring_segment_offsets), so this setup and + // source of truth (sm_layout::ring_segment_offsets), so this setup and // the device-address helpers cannot drift. for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - auto off = pto2_sm_layout::ring_segment_offsets(task_window_sizes, r); + auto off = sm_layout::ring_segment_offsets(task_window_sizes, r); auto &ring = header->rings[r]; ring.task_descriptors = (TaskDescriptor *)(base + off.descriptors); ring.task_payloads = (TaskPayload *)(base + off.payloads); @@ -61,7 +61,7 @@ void PTO2SharedMemoryHandle::setup_pointers_per_ring(const uint64_t task_window_ } } -void PTO2SharedMemoryHandle::setup_pointers(uint64_t task_window_size) { +void SharedMemoryHandle::setup_pointers(uint64_t task_window_size) { uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { task_window_sizes[r] = task_window_size; @@ -69,9 +69,7 @@ void PTO2SharedMemoryHandle::setup_pointers(uint64_t task_window_size) { setup_pointers_per_ring(task_window_sizes); } -bool PTO2SharedMemoryHandle::init( - void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size, uint64_t heap_size -) { +bool SharedMemoryHandle::init(void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size, uint64_t heap_size) { uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]; uint64_t heap_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -81,7 +79,7 @@ bool PTO2SharedMemoryHandle::init( return init_per_ring(sm_base_arg, sm_size_arg, task_window_sizes, heap_sizes); } -bool PTO2SharedMemoryHandle::init_per_ring( +bool SharedMemoryHandle::init_per_ring( void *sm_base_arg, uint64_t sm_size_arg, const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH] ) { @@ -96,13 +94,13 @@ bool PTO2SharedMemoryHandle::init_per_ring( return true; } -PTO2SharedMemoryHandle *PTO2SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { +SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); - const size_t off_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(buffer_size), PTO2_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; - auto *handle = static_cast(arena.region_ptr(off_handle)); + auto *handle = static_cast(arena.region_ptr(off_handle)); memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); memset(buffer, 0, static_cast(buffer_size)); @@ -110,7 +108,7 @@ PTO2SharedMemoryHandle *PTO2SharedMemoryHandle::create_and_init_default(DeviceAr return handle; } -void PTO2SharedMemoryHandle::destroy() { +void SharedMemoryHandle::destroy() { // Arena-owned wrappers (is_owner == false) are reclaimed by arena.release(); // calling destroy on them is a no-op so existing callers stay safe. if (is_owner && sm_base) { @@ -124,7 +122,7 @@ void PTO2SharedMemoryHandle::destroy() { // ============================================================================= // // no need init data in pool, init pool data when used -void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size, uint64_t heap_size) { +void SharedMemoryHandle::init_header(uint64_t task_window_size, uint64_t heap_size) { uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]; uint64_t heap_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -134,7 +132,7 @@ void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size, uint64_t hea init_header_per_ring(task_window_sizes, heap_sizes); } -void PTO2SharedMemoryHandle::init_header_per_ring( +void SharedMemoryHandle::init_header_per_ring( const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH] ) { // Per-ring flow control (start at 0) @@ -145,7 +143,7 @@ void PTO2SharedMemoryHandle::init_header_per_ring( header->orchestrator_done.store(0, std::memory_order_relaxed); // Per-ring layout info - uint64_t offset = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t offset = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { header->rings[r].task_window_size = task_window_sizes[r]; header->rings[r].task_window_mask = static_cast(task_window_sizes[r] - 1); @@ -183,10 +181,10 @@ void PTO2SharedMemoryHandle::init_header_per_ring( // Debug Utilities // ============================================================================= -void PTO2SharedMemoryHandle::print_layout() { +void SharedMemoryHandle::print_layout() { if (!header) return; - PTO2SharedMemoryHeader *h = header; + SharedMemoryHeader *h = header; LOG_DEBUG("=== PTO2 Shared Memory Layout ==="); LOG_DEBUG("Base address: %p", sm_base); @@ -212,11 +210,11 @@ void PTO2SharedMemoryHandle::print_layout() { LOG_DEBUG("================================"); } -bool PTO2SharedMemoryHandle::validate() { +bool SharedMemoryHandle::validate() { if (!sm_base) return false; if (!header) return false; - PTO2SharedMemoryHeader *h = header; + SharedMemoryHeader *h = header; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { if (!h->rings[r].fc.validate(this, r)) return false; @@ -225,12 +223,12 @@ bool PTO2SharedMemoryHandle::validate() { return true; } -bool PTO2RingFlowControl::validate(PTO2SharedMemoryHandle *handle, int32_t ring_id) const { +bool PTO2RingFlowControl::validate(SharedMemoryHandle *handle, int32_t ring_id) const { if (!handle) return false; if (!handle->header) return false; if (ring_id < 0 || ring_id >= CHIP_MAX_RING_DEPTH) return false; - const PTO2SharedMemoryHeader *h = handle->header; + const SharedMemoryHeader *h = handle->header; // Check that offsets are within bounds if (h->rings[ring_id].task_descriptors_offset >= h->total_size) return false; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 421db9da1a..815bc09875 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -14,19 +14,19 @@ * Defines the shared memory structure for Orchestrator-Scheduler communication. * * Memory Layout (per-ring sections repeat for each ring 0..CHIP_MAX_RING_DEPTH-1): - * +---------------------------+ - * | SharedMemoryHeader | (per-ring flow control + sync) - * +---------------------------+ - * | Ring 0: TaskDescriptor[] | - * | Ring 0: TaskPayload[] | - * | Ring 0: TaskSlotState[] | - * +---------------------------+ - * | Ring 1: TaskDescriptor[] | - * | Ring 1: TaskPayload[] | - * | Ring 1: TaskSlotState[] | - * +---------------------------+ - * | ... | - * +---------------------------+ + * +-------------------------------+ + * | SharedMemoryHeader | (per-ring flow control + sync) + * +-------------------------------+ + * | Ring 0: TaskDescriptor[] | + * | Ring 0: TaskPayload[] | + * | Ring 0: ChipTaskSlotState[] | + * +-------------------------------+ + * | Ring 1: TaskDescriptor[] | + * | Ring 1: TaskPayload[] | + * | Ring 1: ChipTaskSlotState[] | + * +-------------------------------+ + * | ... | + * +-------------------------------+ * * Design principles: * - Only data needed for Orchestrator<->Scheduler communication is here @@ -47,7 +47,7 @@ // Shared Memory Header // ============================================================================= -struct PTO2SharedMemoryHandle; +struct SharedMemoryHandle; /** * Per-ring flow control state in shared memory. @@ -72,7 +72,7 @@ struct alignas(64) PTO2RingFlowControl { last_task_alive.store(0, std::memory_order_relaxed); } - bool validate(PTO2SharedMemoryHandle *handle, int32_t ring_id) const; + bool validate(SharedMemoryHandle *handle, int32_t ring_id) const; }; static_assert(sizeof(PTO2RingFlowControl) == 128, "PTO2RingFlowControl must be exactly 2 cache lines (128B)"); @@ -83,7 +83,7 @@ static_assert(sizeof(PTO2RingFlowControl) == 128, "PTO2RingFlowControl must be e * Groups flow-control, layout info, and per-ring data pointers for a single ring. * Pointers are host-side only (set by setup_pointers, invalid on device). */ -struct alignas(64) PTO2SharedMemoryRingHeader { +struct alignas(64) SharedMemoryRingHeader { PTO2RingFlowControl fc; // Layout metadata (set once at init) @@ -114,10 +114,10 @@ struct alignas(64) PTO2SharedMemoryRingHeader { } }; -static_assert(sizeof(PTO2SharedMemoryRingHeader) == 192, "PTO2SharedMemoryRingHeader layout drift"); +static_assert(sizeof(SharedMemoryRingHeader) == 192, "SharedMemoryRingHeader layout drift"); static_assert( - offsetof(PTO2SharedMemoryRingHeader, task_descriptors_offset) == 152, - "PTO2SharedMemoryRingHeader task_descriptors_offset layout drift" + offsetof(SharedMemoryRingHeader, task_descriptors_offset) == 152, + "SharedMemoryRingHeader task_descriptors_offset layout drift" ); /** @@ -125,9 +125,9 @@ static_assert( * * Contains per-ring flow control and global layout information. */ -struct alignas(PTO2_ALIGN_SIZE) PTO2SharedMemoryHeader { +struct alignas(PTO2_ALIGN_SIZE) SharedMemoryHeader { // === PER-RING FLOW CONTROL + LAYOUT INFO (set once at init) === - PTO2SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; + SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; // === GLOBAL FIELDS === std::atomic orchestrator_done; // Flag: orchestration complete @@ -163,14 +163,11 @@ struct alignas(PTO2_ALIGN_SIZE) PTO2SharedMemoryHeader { std::atomic sched_stall_core; // S1: stuck core id (-1 if N/A) }; -static_assert(sizeof(PTO2SharedMemoryHeader) == 896, "PTO2SharedMemoryHeader layout drift"); -static_assert(offsetof(PTO2SharedMemoryHeader, total_size) == 776, "PTO2SharedMemoryHeader total_size layout drift"); +static_assert(sizeof(SharedMemoryHeader) == 896, "SharedMemoryHeader layout drift"); +static_assert(offsetof(SharedMemoryHeader, total_size) == 776, "SharedMemoryHeader total_size layout drift"); +static_assert(offsetof(SharedMemoryHeader, orch_error_code) == 784, "SharedMemoryHeader orch_error_code layout drift"); static_assert( - offsetof(PTO2SharedMemoryHeader, orch_error_code) == 784, "PTO2SharedMemoryHeader orch_error_code layout drift" -); -static_assert( - offsetof(PTO2SharedMemoryHeader, sched_stall_task_id) == 832, - "PTO2SharedMemoryHeader sched_stall_task_id layout drift" + offsetof(SharedMemoryHeader, sched_stall_task_id) == 832, "SharedMemoryHeader sched_stall_task_id layout drift" ); // ============================================================================= @@ -179,13 +176,13 @@ static_assert( /** * Handle for shared memory lifecycle management (create/destroy). - * Runtime components (orchestrator, scheduler) use PTO2SharedMemoryHeader* directly. + * Runtime components (orchestrator, scheduler) use SharedMemoryHeader* directly. */ -struct PTO2SharedMemoryHandle { +struct SharedMemoryHandle { void *sm_base; // Base address of shared memory uint64_t sm_size; // Total size of shared memory - PTO2SharedMemoryHeader *header; + SharedMemoryHeader *header; // Ownership flag bool is_owner; // True if this handle allocated the memory @@ -199,7 +196,7 @@ struct PTO2SharedMemoryHandle { // using default PTO2_TASK_WINDOW_SIZE / PTO2_HEAP_SIZE. Only valid when the // arena is otherwise empty (the call performs the single commit). All // memory is owned by the arena — caller must not call destroy(). - static PTO2SharedMemoryHandle *create_and_init_default(DeviceArena &arena); + static SharedMemoryHandle *create_and_init_default(DeviceArena &arena); // === Instance methods === @@ -238,33 +235,33 @@ struct PTO2SharedMemoryHandle { // // These helpers compute those addresses by offset arithmetic on the SM // device base. Pure pointer math, no loads/stores; safe to call from host. -// The same arithmetic happens on AICPU too (via PTO2SharedMemoryHandle's +// The same arithmetic happens on AICPU too (via SharedMemoryHandle's // own setup_pointers), so values are guaranteed consistent across sides. -namespace pto2_sm_layout { +namespace sm_layout { inline std::atomic *orch_error_code_addr(void *sm_dev_base) noexcept { return reinterpret_cast *>( - static_cast(sm_dev_base) + offsetof(PTO2SharedMemoryHeader, orch_error_code) + static_cast(sm_dev_base) + offsetof(SharedMemoryHeader, orch_error_code) ); } -inline PTO2SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base, int ring_id) noexcept { - return reinterpret_cast( - static_cast(sm_dev_base) + offsetof(PTO2SharedMemoryHeader, rings) + - static_cast(ring_id) * sizeof(PTO2SharedMemoryRingHeader) +inline SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base, int ring_id) noexcept { + return reinterpret_cast( + static_cast(sm_dev_base) + offsetof(SharedMemoryHeader, rings) + + static_cast(ring_id) * sizeof(SharedMemoryRingHeader) ); } inline std::atomic *ring_current_task_index_addr(void *sm_dev_base, int ring_id) noexcept { return reinterpret_cast *>( - reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(PTO2SharedMemoryRingHeader, fc) + + reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(SharedMemoryRingHeader, fc) + offsetof(PTO2RingFlowControl, current_task_index) ); } inline std::atomic *ring_last_task_alive_addr(void *sm_dev_base, int ring_id) noexcept { return reinterpret_cast *>( - reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(PTO2SharedMemoryRingHeader, fc) + + reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(SharedMemoryRingHeader, fc) + offsetof(PTO2RingFlowControl, last_task_alive) ); } @@ -287,8 +284,8 @@ struct PTO2RingSegmentOffsets { // layout walk can never silently disagree across call sites. inline PTO2RingSegmentOffsets ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int ring_id) noexcept { - assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "pto2_sm_layout: ring_id out of range"); - uint64_t off = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "sm_layout: ring_id out of range"); + uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); for (int r = 0; r < ring_id; r++) { off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); @@ -323,4 +320,4 @@ ring_slot_states_addr(void *sm_dev_base, const uint64_t task_window_sizes[CHIP_M ); } -} // namespace pto2_sm_layout +} // namespace sm_layout diff --git a/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp b/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp index 028d9af2b4..6685adcb0f 100644 --- a/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp +++ b/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp @@ -70,7 +70,7 @@ static int32_t read_pto2_runtime_status(Runtime *runtime) { return 0; } - auto *header = static_cast(sm); + auto *header = static_cast(sm); int32_t orch_error_code = header->orch_error_code.load(std::memory_order_acquire); int32_t sched_error_code = header->sched_error_code.load(std::memory_order_acquire); return runtime_status_from_error_codes(orch_error_code, sched_error_code); @@ -267,8 +267,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { // not to the ring capacity, and the device region holds exactly that // image — so its size comes from the same pitch. attach_populated // rejects a pitch outside (0, capacity] and a region too small for it. - const uint64_t live_slots = - pto2_sm_layout::live_slot_pitch(static_cast(runtime->host_total_tasks)); + const uint64_t live_slots = sm_layout::live_slot_pitch(static_cast(runtime->host_total_tasks)); const uint64_t sm_size = runtime->sm_image_bytes; // sm_handle and the scheduler state are the device-only zone: their // bytes never travel, so they start as whatever the pooled arena last diff --git a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp index 79d55a8e0a..e25a729b46 100644 --- a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp @@ -294,7 +294,7 @@ static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t // A slot state reaches its payload and descriptor through a 32-bit // self-relative delta, so every pair of addresses in the shared-memory // image must be within INT32_MAX of each other. - const uint64_t sm_bytes = pto2_sm_layout::ring_segment_offsets(*eff_task_window_size).end; + const uint64_t sm_bytes = sm_layout::ring_segment_offsets(*eff_task_window_size).end; if (sm_bytes > static_cast(INT32_MAX)) { LOG_ERROR( "ring_task_window=%" PRIu64 " needs a %" PRIu64 "-byte shared memory image, past the %d-byte limit " @@ -307,7 +307,7 @@ static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t return true; } -static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, PTO2SharedMemoryHeader *host_header) { +static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { if (runtime == nullptr || api == nullptr || host_header == nullptr) { return 0; } @@ -317,7 +317,7 @@ static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, PT return 0; } - int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(PTO2SharedMemoryHeader)); + int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(SharedMemoryHeader)); if (hdr_rc != 0) { LOG_WARN("Failed to copy PTO2 header from device"); return 0; @@ -560,7 +560,7 @@ int32_t run_host_orchestration( // each written per task at submit and read only for [0, total_tasks). Zero // only the fixed-size header here; the per-slot segments are initialized in // orch::prepare_task and shipped bounded to total_tasks below. - const pto2_sm_layout::PTO2RingSegmentOffsets sm_segs = pto2_sm_layout::ring_segment_offsets(eff_task_window_size); + const sm_layout::PTO2RingSegmentOffsets sm_segs = sm_layout::ring_segment_offsets(eff_task_window_size); // Over-allocated and rounded up: every segment offset is a multiple of // PTO2_ALIGN_SIZE and ChipTaskSlotState is alignas(64), which a plain // new uint8_t[] does not guarantee. @@ -592,7 +592,7 @@ int32_t run_host_orchestration( } // Initialize the host SM header (ring flow control) so submit_task can run. - PTO2SharedMemoryHandle host_sm_handle; + SharedMemoryHandle host_sm_handle; if (!host_sm_handle.init(host_sm, sm_size, eff_task_window_size)) { LOG_ERROR("host-orch: host SM init failed"); return PTO_RUNTIME_ERR_INTERNAL; @@ -687,7 +687,7 @@ int32_t run_host_orchestration( // described — a heap or tensormap exhaustion drops tasks, a fanin overflow drops // edges. Uploading it would launch the device on an incomplete graph and surface // the cause as whatever the device notices second, usually a scheduler timeout. - const int32_t orch_error = pto2_sm_layout::orch_error_code_addr(host_sm)->load(std::memory_order_acquire); + const int32_t orch_error = sm_layout::orch_error_code_addr(host_sm)->load(std::memory_order_acquire); if (orch_error != SIMPLER_ERROR_NONE || orchestrator.fatal) { // The latched code is the diagnosis, so it is what the caller sees — through the // same mapping the run path uses, since a caller cannot tell which of the two @@ -703,7 +703,7 @@ int32_t run_host_orchestration( return status; } - const int32_t total_tasks = pto2_sm_layout::ring_current_task_index_addr(host_sm)->load(std::memory_order_acquire); + const int32_t total_tasks = sm_layout::ring_current_task_index_addr(host_sm)->load(std::memory_order_acquire); { char attrs[160]; snprintf( @@ -761,13 +761,13 @@ int32_t run_host_orchestration( // exact populated extent of each one — no scan of the mirror is needed, and the // image ships that much rather than the worst case the mirror is dimensioned for. const PTO2OrchestratorState &orch_state = orchestrator; - const pto2_sm_layout::BindUsage bind_usage{ + const sm_layout::BindUsage bind_usage{ nt, static_cast(orch_state.fanin_pool_cursor), static_cast(orch_state.tensor_pool_cursor), static_cast(orch_state.scalar_pool_cursor), }; - const uint64_t image_bytes = pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents(bind_usage)).end; + const uint64_t image_bytes = sm_layout::ring_segment_offsets(sm_layout::image_extents(bind_usage)).end; runtime->sm_image_bytes = image_bytes; // Only now are both sizes known, so this is where the two device regions are @@ -848,7 +848,7 @@ int32_t run_host_orchestration( "a Graph node's storage alignment must be covered by the heap region's base alignment" ); always_assert(reinterpret_cast(gm_heap) % DeviceArena::kDefaultBaseAlign == 0); - const pto2_sm_layout::HeapRebase heap_rebase{reinterpret_cast(gm_heap), heap_bytes}; + const sm_layout::HeapRebase heap_rebase{reinterpret_cast(gm_heap), heap_bytes}; // One host source for one copy: the copied zone and shared-memory image at // exactly the offsets they occupy on the device. @@ -868,7 +868,7 @@ int32_t run_host_orchestration( // the pointer goes early rather than at the guard's scope exit. rt->orchestrator = nullptr; std::memcpy(upload_base, static_cast(host_arena.base()) + layout.off_copied_begin, copied_bytes); - const uint64_t compacted = pto2_sm_layout::compact_live_image( + const uint64_t compacted = sm_layout::compact_live_image( static_cast(host_sm), eff_task_window_size, bind_usage, heap_rebase, upload_base + copied_bytes ); always_assert(compacted == image_bytes); @@ -1173,7 +1173,7 @@ extern "C" int bind_callable_to_runtime_impl( // runs — do NOT record in tensor_pairs_; the free is deferred to // DeviceRunner::finalize(). The runtime-arena size is determined by replaying // the reserve sequence on a host-side arena. - uint64_t sm_size = PTO2SharedMemoryHandle::calculate_size(eff_task_window_size); + uint64_t sm_size = SharedMemoryHandle::calculate_size(eff_task_window_size); const int64_t t_arena_build_ns = bind_now_ns(); DeviceArena host_arena; @@ -1311,7 +1311,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e bool skip_tensor_copy_back = execution_rc != 0; int32_t runtime_status = 0; - PTO2SharedMemoryHeader host_header; + SharedMemoryHeader host_header; memset(&host_header, 0, sizeof(host_header)); if (execution_rc != 0) { diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator.h b/src/a5/runtime/host_build_graph/runtime/orchestrator.h index a3d0e89bf9..816345b169 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator.h @@ -58,7 +58,7 @@ struct GraphHostState; */ struct PTO2OrchestratorState { // === SHARED MEMORY ACCESS === - PTO2SharedMemoryHeader *sm_header; + SharedMemoryHeader *sm_header; // === TASK / HEAP ALLOCATION === // hbg is single-ring, so one allocator covers the whole graph. diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index a028e1fda3..55e08f0154 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1507,7 +1507,7 @@ static TaskOutputTensors submit_task_common( dep_gen_host_graph_add_explicit_edge(dep_task_id.raw); } uint8_t dep_ring_id = dep_task_id.ring(); - PTO2SharedMemoryRingHeader &dep_ring = orch->sm_header->ring; + SharedMemoryRingHeader &dep_ring = orch->sm_header->ring; int32_t dep_local_task_id = static_cast(dep_task_id.local()); int32_t dep_slot = dep_ring.get_slot_by_task_id(dep_local_task_id); ChipTaskSlotState *producer_slot_state = &dep_ring.get_slot_state_by_slot(dep_slot); @@ -1524,7 +1524,7 @@ static TaskOutputTensors submit_task_common( auto runtime_emit = [&](TaskId producer_task_id) -> bool { uint8_t prod_ring = producer_task_id.ring(); - PTO2SharedMemoryRingHeader &producer_ring = orch->sm_header->ring; + SharedMemoryRingHeader &producer_ring = orch->sm_header->ring; int32_t prod_slot = producer_ring.get_slot_by_task_id(static_cast(producer_task_id.local())); ChipTaskSlotState *prod_state = &producer_ring.get_slot_state_by_slot(prod_slot); return append_fanin_or_fail(orch, prod_ring, prod_slot, prod_state, producer_task_id, &fanin_builder); @@ -1796,7 +1796,7 @@ bool graph_submit_outer( return false; } const TaskId task_id = TaskId::make(0, static_cast(allocation.task_id)); - PTO2SharedMemoryRingHeader &ring = orch->sm_header->ring; + SharedMemoryRingHeader &ring = orch->sm_header->ring; TaskDescriptor &task = ring.task_descriptors[allocation.slot]; TaskPayload &payload = ring.task_payloads[allocation.slot]; ChipTaskSlotState &slot = ring.get_slot_state_by_slot(allocation.slot); @@ -2751,7 +2751,7 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) // every consumer register_wakes on a producer that never runs on device and // the run hangs. (The device watermark walk transparently steps past this // pre-set flag when a later on-device task completes.) - PTO2SharedMemoryRingHeader &done_ring = orch->sm_header->ring; + SharedMemoryRingHeader &done_ring = orch->sm_header->ring; int32_t done_local = static_cast(prepared.task_id.local()); done_ring.set_completion_flag(done_local); } diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 415c6e92eb..4dac7bd3cd 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -247,7 +247,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo // completed_watermark reaches the producer's highest consumer id (set at // submit in append_fanin_or_fail). Replaces the fanout_refcount == // fanout_count wiring check, which polling removes. - PTO2SharedMemoryRingHeader &cons_ring = orch.sm_header->ring; + SharedMemoryRingHeader &cons_ring = orch.sm_header->ring; while (cons_ring.completed_watermark.load(std::memory_order_acquire) < slot.last_consumer_local_id) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_core.h b/src/a5/runtime/host_build_graph/runtime/runtime_core.h index 8a12ea0a15..f47a43161c 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_core.h @@ -171,7 +171,7 @@ struct RuntimeContext { PTO2ScopeMode pending_scope_mode; // Components - PTO2SharedMemoryHandle *sm_handle; + SharedMemoryHandle *sm_handle; // Host-only, and by pointer so that this header stays trivially copyable: // the orchestrator runs on the host and owns non-trivial scratch. Null on the // device — bind drops it before the copied zone is uploaded, so no device code diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h index 5ff913e8f9..8655213cc0 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -472,12 +472,12 @@ struct PTO2SchedulerLayout { */ struct PTO2SchedulerState { // Shared memory access - PTO2SharedMemoryHeader *sm_header; + SharedMemoryHeader *sm_header; // Per-ring state struct alignas(64) RingSchedState { // --- Cache Line 0: ring pointer (read-only) + hot path (read-write) --- - PTO2SharedMemoryRingHeader *ring; + SharedMemoryRingHeader *ring; std::atomic advance_lock; // multi-thread CAS // Polling: no per-ring dep_pool. Readiness is derived from the SM ring's @@ -586,7 +586,7 @@ struct PTO2SchedulerState { // completion re-scans its waiters via on_mixed_task_complete's wake drain. int classify_fanin_state(const ChipTaskSlotState *s) const { const TaskPayload &p = *s->payload; - const PTO2SharedMemoryRingHeader &ring = *ring_sched_state.ring; + const SharedMemoryRingHeader &ring = *ring_sched_state.ring; const int32_t *fanin = p.fanin_data(); for (int32_t i = p.fanin_count - 1; i >= 0; i--) { if (!ring.is_completion_flag_set(fanin[i])) return i; @@ -599,7 +599,7 @@ struct PTO2SchedulerState { // ready only when every fanin is met, else re-target the next unmet producer // and retry. Monotonic completion_flags guarantee termination. void register_wake(ChipTaskSlotState *producer, ChipTaskSlotState *consumer) { - PTO2SharedMemoryRingHeader &ring = *ring_sched_state.ring; + SharedMemoryRingHeader &ring = *ring_sched_state.ring; while (true) { ChipTaskSlotState *expected = producer->wake_list_head.load(std::memory_order_relaxed); while (expected != WAKE_LIST_SENTINEL) { @@ -627,7 +627,7 @@ struct PTO2SchedulerState { // has no device slot reclaim, so no advance_ring_pointers here. void on_mixed_task_complete(ChipTaskSlotState &slot_state) { const int32_t task_id = static_cast(slot_state.task->task_id.local()); - PTO2SharedMemoryRingHeader &ring = *ring_sched_state.ring; + SharedMemoryRingHeader &ring = *ring_sched_state.ring; slot_state.mark_completed(); // host-visible mirror (task_state = COMPLETED) ring.set_completion_flag(task_id); diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index 3356c9258e..3070d33c4c 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -31,7 +31,7 @@ // Cold-path helpers for the main dispatch loop (noinline to reduce hot-loop icache) // ============================================================================= -static void latch_scheduler_error(PTO2SharedMemoryHeader *header, int32_t thread_idx, int32_t error_code) { +static void latch_scheduler_error(SharedMemoryHeader *header, int32_t thread_idx, int32_t error_code) { if (header == nullptr || error_code == SIMPLER_ERROR_NONE) { return; } @@ -53,7 +53,7 @@ void SchedulerContext::fail_scheduler(Runtime *runtime, int32_t thread_idx, int3 } LoopAction SchedulerContext::handle_orchestrator_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count ) { if (completed_.load(std::memory_order_acquire)) { return LoopAction::BREAK_LOOP; @@ -91,8 +91,7 @@ LoopAction SchedulerContext::handle_orchestrator_exit( return LoopAction::NONE; } -LoopAction -SchedulerContext::check_idle_fatal_error(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime) { +LoopAction SchedulerContext::check_idle_fatal_error(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime) { if (completed_.load(std::memory_order_acquire)) { return LoopAction::BREAK_LOOP; } @@ -232,7 +231,7 @@ void SchedulerContext::log_stall_diagnostics( // produce identical TASK lines once per scheduler thread. if (thread_idx == 0) { int32_t cnt_ready = 0, cnt_waiting = 0, cnt_running = 0, submitted_in_ring = 0; - PTO2SharedMemoryRingHeader &ring = *sched_->ring_sched_state.ring; + SharedMemoryRingHeader &ring = *sched_->ring_sched_state.ring; int32_t ring_task_count = ring.fc.current_task_index.load(std::memory_order_relaxed); submitted_in_ring += ring_task_count; for (int32_t si = 0; si < ring_task_count; si++) { @@ -367,7 +366,7 @@ void SchedulerContext::log_shutdown_stall_snapshot( } int32_t SchedulerContext::handle_timeout_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, int32_t last_progress_count #if SIMPLER_DFX , @@ -908,7 +907,7 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // Initialize task counters. Task count comes from PTO2 shared memory. if (runtime->get_gm_sm_ptr()) { - auto *header = static_cast(runtime->get_gm_sm_ptr()); + auto *header = static_cast(runtime->get_gm_sm_ptr()); // Read at one-time boot init, before the SM is reset for the run, so a ring // not yet written holds uninitialized memory (0xbe... under ASAN's // malloc-fill). Only a plausible count is taken — (0, @@ -1125,7 +1124,7 @@ void SchedulerContext::classify_partition(int32_t thread_idx, int32_t nthreads) if (completed_.load(std::memory_order_acquire) || sched_->ring_sched_state.ring == nullptr) { return; } - PTO2SharedMemoryRingHeader &ring = *sched_->ring_sched_state.ring; + SharedMemoryRingHeader &ring = *sched_->ring_sched_state.ring; const int32_t submitted = ring.fc.current_task_index.load(std::memory_order_acquire); // Disjoint contiguous slices covering [0, submitted): thread t owns // [submitted*t/nthreads, submitted*(t+1)/nthreads). int64 math avoids overflow. diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index 5cfd7c1c83..727aa205a7 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -521,10 +521,10 @@ class SchedulerContext { // ========================================================================= __attribute__((noinline, cold)) LoopAction - handle_orchestrator_exit(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count); + handle_orchestrator_exit(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count); __attribute__((noinline, cold)) LoopAction - check_idle_fatal_error(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime); + check_idle_fatal_error(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime); __attribute__((noinline, cold)) void log_stall_diagnostics(int32_t thread_idx, int32_t task_count, int32_t idle_iterations, int32_t last_progress_count); @@ -554,7 +554,7 @@ class SchedulerContext { bool no_thread_owns_running_task() const; __attribute__((noinline, cold)) int32_t handle_timeout_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, int32_t last_progress_count #if SIMPLER_DFX , diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index e6fcdcfdbf..38f8b6947d 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -891,7 +891,7 @@ int32_t SchedulerContext::try_early_dispatch( // advanced) — the host's wait_for_consumers never observes a stranded prefix. int32_t SchedulerContext::run_resolution_thread(Runtime *runtime, int32_t thread_idx) { always_assert(sched_ != nullptr); - PTO2SharedMemoryHeader *header = sched_->sm_header; + SharedMemoryHeader *header = sched_->sm_header; if (!header) { LOG_ERROR("PTO2 resolution: header is null"); return -1; @@ -1064,7 +1064,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ always_assert(sched_ != nullptr); CoreTracker &tracker = core_trackers_[thread_idx]; - PTO2SharedMemoryHeader *header = sched_->sm_header; + SharedMemoryHeader *header = sched_->sm_header; if (!header) { LOG_ERROR("PTO2 dispatch: header is null"); return -1; diff --git a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 71d205a338..29b9a4195f 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -66,7 +66,7 @@ void ready_queue_destroy(PTO2ReadyQueue *queue) { bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base) { // ring stores the device address of the SM ring header — pure offset // arithmetic, no SM load. - ring = pto2_sm_layout::ring_header_addr(sm_dev_base); + ring = sm_layout::ring_header_addr(sm_dev_base); advance_lock.store(0, std::memory_order_relaxed); // Per-slot SM-side initialization (reset_for_reuse + active_mask, and clearing @@ -107,7 +107,7 @@ bool PTO2SchedulerState::init_data_from_layout( const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base ) { PTO2SchedulerState *sched = this; - sched->sm_header = reinterpret_cast(sm_dev_base); + sched->sm_header = reinterpret_cast(sm_dev_base); #if SIMPLER_SCHED_PROFILING sched->tasks_completed.store(0, std::memory_order_relaxed); sched->tasks_consumed.store(0, std::memory_order_relaxed); @@ -208,12 +208,12 @@ bool PTO2OrchestratorState::init( // A power-of-two window lets pto2_task_slot() mask instead of dividing. always_assert(task_window_size > 0 && (task_window_size & (task_window_size - 1)) == 0); - orch->sm_header = reinterpret_cast(sm_base); + orch->sm_header = reinterpret_cast(sm_base); orch->fatal = false; orch->scheduler = scheduler_arg; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_base); - auto *cur_idx_dev = pto2_sm_layout::ring_current_task_index_addr(sm_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_base); + auto *cur_idx_dev = sm_layout::ring_current_task_index_addr(sm_base); orch->task_allocator.init(static_cast(task_window_size), cur_idx_dev, gm_heap, heap_size, orch_err); @@ -221,7 +221,7 @@ bool PTO2OrchestratorState::init( // so it holds for whichever SM this orchestrator was pointed at. The cursors // reset with the rest of the state above. auto *sm_bytes = static_cast(sm_base); - const auto pools = pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::mirror_extents(task_window_size)); + const auto pools = sm_layout::ring_segment_offsets(sm_layout::mirror_extents(task_window_size)); orch->fanin_pool = reinterpret_cast(sm_bytes + pools.fanin_pool); orch->tensor_pool = reinterpret_cast(sm_bytes + pools.tensor_pool); orch->scalar_pool = reinterpret_cast(sm_bytes + pools.scalar_pool); @@ -263,7 +263,7 @@ RuntimeArenaLayout runtime_reserve_layout(DeviceArena &arena, uint64_t task_wind // // The copied zone comes last of the two so the device's shared-memory tail can // begin where it ends, making the two adjacent and the upload one copy. - layout.off_sm_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + layout.off_sm_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); layout.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); layout.off_scheduler = arena.reserve(sizeof(PTO2SchedulerState), alignof(PTO2SchedulerState)); layout.sched = PTO2SchedulerState::reserve_layout(arena); @@ -317,7 +317,7 @@ RuntimeContext *runtime_init_data_from_layout( } void runtime_wire_arena_pointers(DeviceArena &arena, const RuntimeArenaLayout &layout, RuntimeContext *rt) { - rt->sm_handle = static_cast(arena.region_ptr(layout.off_sm_handle)); + rt->sm_handle = static_cast(arena.region_ptr(layout.off_sm_handle)); rt->aicore_mailbox = static_cast(arena.region_ptr(layout.off_mailbox)); rt->scheduler = static_cast(arena.region_ptr(layout.off_scheduler)); rt->scheduler->wire_arena_pointers(layout.sched, arena); diff --git a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 8e098d78f2..91b187b733 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -27,29 +27,29 @@ // Size Calculation // ============================================================================= -uint64_t PTO2SharedMemoryHandle::calculate_size(uint64_t task_window_size) { +uint64_t SharedMemoryHandle::calculate_size(uint64_t task_window_size) { // Total SM size = offset just past the ring's slot_states, from the single - // source of truth for the layout (pto2_sm_layout::ring_segment_offsets). - return pto2_sm_layout::ring_segment_offsets(task_window_size).end; + // source of truth for the layout (sm_layout::ring_segment_offsets). + return sm_layout::ring_segment_offsets(task_window_size).end; } // ============================================================================= // Creation and Destruction // ============================================================================= -void PTO2SharedMemoryHandle::setup_pointers(uint64_t pitch) { +void SharedMemoryHandle::setup_pointers(uint64_t pitch) { char *base = (char *)sm_base; - header = (PTO2SharedMemoryHeader *)base; + header = (SharedMemoryHeader *)base; // Per-ring descriptors / payloads / slot_states — offsets from the single source - // of truth (pto2_sm_layout::ring_segment_offsets), so this setup and the + // of truth (sm_layout::ring_segment_offsets), so this setup and the // device-address helpers cannot drift. // // Only the four slot-pitched segments, and no argument pool: the pool extents // differ between the mirror and the image, while these four depend on the pitch // alone. The orchestrator holds the mirror's pool bases; the device resolves an // argument region through the payload's delta and needs no base at all. - auto off = pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents({pitch, 0, 0, 0})); + auto off = sm_layout::ring_segment_offsets(sm_layout::image_extents({pitch, 0, 0, 0})); auto &ring = header->ring; ring.task_descriptors = (TaskDescriptor *)(base + off.descriptors); ring.task_payloads = (TaskPayload *)(base + off.payloads); @@ -57,7 +57,7 @@ void PTO2SharedMemoryHandle::setup_pointers(uint64_t pitch) { ring.completion_flags = (std::atomic *)(base + off.completion_flags); } -bool PTO2SharedMemoryHandle::init(void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size) { +bool SharedMemoryHandle::init(void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size) { if (!sm_base_arg || sm_size_arg == 0) return false; const uint64_t mirror_bytes = calculate_size(task_window_size); // The mirror has to stay inside an int32 delta's reach: a payload names its @@ -78,7 +78,7 @@ bool PTO2SharedMemoryHandle::init(void *sm_base_arg, uint64_t sm_size_arg, uint6 return true; } -bool PTO2SharedMemoryHandle::attach_populated( +bool SharedMemoryHandle::attach_populated( void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size, uint64_t live_slots, uint64_t image_bytes ) { if (!sm_base_arg || sm_size_arg == 0) return false; @@ -91,7 +91,7 @@ bool PTO2SharedMemoryHandle::attach_populated( // that is its argument pools, whose extents are the bind's cursors and are not // recomputable here. The first pool's offset is exactly where those four end. const uint64_t slots_end = - pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents({live_slots, 0, 0, 0})).fanin_pool; + sm_layout::ring_segment_offsets(sm_layout::image_extents({live_slots, 0, 0, 0})).fanin_pool; if (image_bytes < slots_end) return false; // The region holds the compacted image, so that is what has to fit — the ring // capacity is no longer its size. @@ -110,13 +110,13 @@ bool PTO2SharedMemoryHandle::attach_populated( return true; } -PTO2SharedMemoryHandle *PTO2SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { +SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); - const size_t off_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(buffer_size), PTO2_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; - auto *handle = static_cast(arena.region_ptr(off_handle)); + auto *handle = static_cast(arena.region_ptr(off_handle)); memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); memset(buffer, 0, static_cast(buffer_size)); @@ -124,7 +124,7 @@ PTO2SharedMemoryHandle *PTO2SharedMemoryHandle::create_and_init_default(DeviceAr return handle; } -void PTO2SharedMemoryHandle::destroy() { +void SharedMemoryHandle::destroy() { // Arena-owned wrappers (is_owner == false) are reclaimed by arena.release(); // calling destroy on them is a no-op so existing callers stay safe. if (is_owner && sm_base) { @@ -138,7 +138,7 @@ void PTO2SharedMemoryHandle::destroy() { // ============================================================================= // // no need init data in pool, init pool data when used -void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size) { +void SharedMemoryHandle::init_header(uint64_t task_window_size) { // Flow control (starts at 0) header->ring.fc.init(); @@ -149,7 +149,7 @@ void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size) { header->orchestrator_done.store(0, std::memory_order_relaxed); // Ring layout info - uint64_t offset = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t offset = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); header->ring.task_window_size = task_window_size; header->ring.task_window_mask = static_cast(task_window_size - 1); header->ring.task_descriptors_offset = offset; @@ -176,10 +176,10 @@ void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size) { // Debug Utilities // ============================================================================= -void PTO2SharedMemoryHandle::print_layout() { +void SharedMemoryHandle::print_layout() { if (!header) return; - PTO2SharedMemoryHeader *h = header; + SharedMemoryHeader *h = header; LOG_DEBUG("=== PTO2 Shared Memory Layout ==="); LOG_DEBUG("Base address: %p", sm_base); @@ -200,22 +200,22 @@ void PTO2SharedMemoryHandle::print_layout() { LOG_DEBUG("================================"); } -bool PTO2SharedMemoryHandle::validate() { +bool SharedMemoryHandle::validate() { if (!sm_base) return false; if (!header) return false; - PTO2SharedMemoryHeader *h = header; + SharedMemoryHeader *h = header; if (!h->ring.fc.validate(this)) return false; return true; } -bool PTO2RingFlowControl::validate(PTO2SharedMemoryHandle *handle) const { +bool PTO2RingFlowControl::validate(SharedMemoryHandle *handle) const { if (!handle) return false; if (!handle->header) return false; - const PTO2SharedMemoryHeader *h = handle->header; + const SharedMemoryHeader *h = handle->header; // Check that offsets are within bounds if (h->ring.task_descriptors_offset >= h->total_size) return false; diff --git a/src/a5/runtime/host_build_graph/runtime/shared_memory.h b/src/a5/runtime/host_build_graph/runtime/shared_memory.h index 32582ad0aa..3a76a415fa 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a5/runtime/host_build_graph/runtime/shared_memory.h @@ -19,7 +19,7 @@ * +---------------------------+ * | TaskDescriptor[] | * | TaskPayload[] | - * | TaskSlotState[] | + * | ChipTaskSlotState[] | * +---------------------------+ * * Design principles: @@ -44,7 +44,7 @@ // Shared Memory Header // ============================================================================= -struct PTO2SharedMemoryHandle; +struct SharedMemoryHandle; /** * Per-ring flow control state in shared memory. @@ -64,7 +64,7 @@ struct alignas(64) PTO2RingFlowControl { // submit IDs will be off by the divergence. void init() { current_task_index.store(0, std::memory_order_relaxed); } - bool validate(PTO2SharedMemoryHandle *handle) const; + bool validate(SharedMemoryHandle *handle) const; }; static_assert(sizeof(PTO2RingFlowControl) == 64, "PTO2RingFlowControl must be exactly one cache line (64B)"); @@ -75,7 +75,7 @@ static_assert(sizeof(PTO2RingFlowControl) == 64, "PTO2RingFlowControl must be ex * Groups flow-control, layout info, and per-ring data pointers for a single ring. * Pointers are host-side only (set by setup_pointers, invalid on device). */ -struct alignas(64) PTO2SharedMemoryRingHeader { +struct alignas(64) SharedMemoryRingHeader { PTO2RingFlowControl fc; // Highest task_id such that every task with id in [0, completed_watermark] @@ -154,10 +154,10 @@ struct alignas(64) PTO2SharedMemoryRingHeader { } }; -static_assert(sizeof(PTO2SharedMemoryRingHeader) == 192, "PTO2SharedMemoryRingHeader layout drift"); +static_assert(sizeof(SharedMemoryRingHeader) == 192, "SharedMemoryRingHeader layout drift"); static_assert( - offsetof(PTO2SharedMemoryRingHeader, task_descriptors_offset) == 144, - "PTO2SharedMemoryRingHeader task_descriptors_offset layout drift" + offsetof(SharedMemoryRingHeader, task_descriptors_offset) == 144, + "SharedMemoryRingHeader task_descriptors_offset layout drift" ); /** @@ -165,9 +165,9 @@ static_assert( * * Contains per-ring flow control and global layout information. */ -struct alignas(PTO2_ALIGN_SIZE) PTO2SharedMemoryHeader { +struct alignas(PTO2_ALIGN_SIZE) SharedMemoryHeader { // === RING FLOW CONTROL + LAYOUT INFO (single ring, set once at init) === - PTO2SharedMemoryRingHeader ring; + SharedMemoryRingHeader ring; // === GLOBAL FIELDS === std::atomic orchestrator_done; // Flag: orchestration complete @@ -188,11 +188,9 @@ struct alignas(PTO2_ALIGN_SIZE) PTO2SharedMemoryHeader { std::atomic sched_error_thread; // Thread index of last error writer }; -static_assert(sizeof(PTO2SharedMemoryHeader) == 256, "PTO2SharedMemoryHeader layout drift"); -static_assert(offsetof(PTO2SharedMemoryHeader, total_size) == 200, "PTO2SharedMemoryHeader total_size layout drift"); -static_assert( - offsetof(PTO2SharedMemoryHeader, orch_error_code) == 208, "PTO2SharedMemoryHeader orch_error_code layout drift" -); +static_assert(sizeof(SharedMemoryHeader) == 256, "SharedMemoryHeader layout drift"); +static_assert(offsetof(SharedMemoryHeader, total_size) == 200, "SharedMemoryHeader total_size layout drift"); +static_assert(offsetof(SharedMemoryHeader, orch_error_code) == 208, "SharedMemoryHeader orch_error_code layout drift"); // ============================================================================= // Shared Memory Handle @@ -200,13 +198,13 @@ static_assert( /** * Handle for shared memory lifecycle management (create/destroy). - * Runtime components (orchestrator, scheduler) use PTO2SharedMemoryHeader* directly. + * Runtime components (orchestrator, scheduler) use SharedMemoryHeader* directly. */ -struct PTO2SharedMemoryHandle { +struct SharedMemoryHandle { void *sm_base; // Base address of shared memory uint64_t sm_size; // Total size of shared memory - PTO2SharedMemoryHeader *header; + SharedMemoryHeader *header; // Ownership flag bool is_owner; // True if this handle allocated the memory @@ -219,7 +217,7 @@ struct PTO2SharedMemoryHandle { // default PTO2_TASK_WINDOW_SIZE. Only valid when the arena is otherwise empty // (the call performs the single commit). All memory is owned by the arena — // caller must not call destroy(). - static PTO2SharedMemoryHandle *create_and_init_default(DeviceArena &arena); + static SharedMemoryHandle *create_and_init_default(DeviceArena &arena); // === Instance methods === @@ -276,25 +274,25 @@ struct PTO2SharedMemoryHandle { // // These helpers compute those addresses by offset arithmetic on the SM // device base. Pure pointer math, no loads/stores; safe to call from host. -// The same arithmetic happens on AICPU too (via PTO2SharedMemoryHandle's +// The same arithmetic happens on AICPU too (via SharedMemoryHandle's // own setup_pointers), so values are guaranteed consistent across sides. -namespace pto2_sm_layout { +namespace sm_layout { inline std::atomic *orch_error_code_addr(void *sm_dev_base) noexcept { return reinterpret_cast *>( - static_cast(sm_dev_base) + offsetof(PTO2SharedMemoryHeader, orch_error_code) + static_cast(sm_dev_base) + offsetof(SharedMemoryHeader, orch_error_code) ); } -inline PTO2SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base) noexcept { - return reinterpret_cast( - static_cast(sm_dev_base) + offsetof(PTO2SharedMemoryHeader, ring) +inline SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base) noexcept { + return reinterpret_cast( + static_cast(sm_dev_base) + offsetof(SharedMemoryHeader, ring) ); } inline std::atomic *ring_current_task_index_addr(void *sm_dev_base) noexcept { return reinterpret_cast *>( - reinterpret_cast(ring_header_addr(sm_dev_base)) + offsetof(PTO2SharedMemoryRingHeader, fc) + + reinterpret_cast(ring_header_addr(sm_dev_base)) + offsetof(SharedMemoryRingHeader, fc) + offsetof(PTO2RingFlowControl, current_task_index) ); } @@ -345,7 +343,7 @@ inline RingImageExtents mirror_extents(uint64_t task_window_size) noexcept { // or reordering a segment is a one-line edit here; every consumer follows // automatically, so the layout walk can never silently disagree across call sites. inline PTO2RingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) noexcept { - uint64_t off = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); PTO2RingSegmentOffsets o{}; o.descriptors = off; off += PTO2_ALIGN_UP(e.slots * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); @@ -481,7 +479,7 @@ inline uint64_t compact_live_image( // The header and the descriptors offset are pitch-independent, so the header // lands where it already was. std::memcpy(out_base, mirror_base, to.descriptors); - auto &out_ring = reinterpret_cast(out_base)->ring; + auto &out_ring = reinterpret_cast(out_base)->ring; out_ring.task_descriptors = nullptr; out_ring.task_payloads = nullptr; out_ring.slot_states = nullptr; @@ -563,4 +561,4 @@ inline uint64_t compact_live_image( return to.end; } -} // namespace pto2_sm_layout +} // namespace sm_layout diff --git a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index 853012d96e..aa04f53b1f 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -89,7 +89,7 @@ static int32_t read_runtime_status(Runtime *runtime) { return 0; } - auto *header = static_cast(sm); + auto *header = static_cast(sm); int32_t orch_error_code = header->orch_error_code.load(std::memory_order_acquire); int32_t sched_error_code = header->sched_error_code.load(std::memory_order_acquire); return runtime_status_from_error_codes(orch_error_code, sched_error_code); @@ -615,7 +615,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { // Wire every arena-internal pointer field (host wrote host-mirror // addresses; we overwrite them with device addresses). runtime_wire_arena_pointers(runtime_arena_, rt->prebuilt_layout, rt); - sm_size = PTO2SharedMemoryHandle::calculate_size_per_ring(rt->prebuilt_layout.sizing.task_window_sizes); + sm_size = SharedMemoryHandle::calculate_size_per_ring(rt->prebuilt_layout.sizing.task_window_sizes); } // Reset SM state. setup_pointers + init_header_per_ring restore diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 24b185d855..dc363705df 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -72,7 +72,7 @@ PTO2RingSet rings[CHIP_MAX_RING_DEPTH]; Ring selection: `current_ring_id() = min(scope_stack_top, CHIP_MAX_RING_DEPTH - 1)`. -### 4.3 PTO2SharedMemoryHeader (modified) +### 4.3 SharedMemoryHeader (modified) Per-ring flow control and per-ring layout info are grouped together: @@ -84,7 +84,7 @@ struct PTO2RingFlowControl { std::atomic heap_tail; // heap reclaim pointer }; -struct alignas(64) PTO2SharedMemoryRingHeader { +struct alignas(64) SharedMemoryRingHeader { PTO2RingFlowControl fc; // Layout metadata (set once at init) @@ -108,20 +108,20 @@ struct alignas(64) PTO2SharedMemoryRingHeader { }; // In header: -PTO2SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; +SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; ``` -Per-ring try-locks in the scheduler state prevent concurrent scheduler threads from interleaving watermark writes within the same ring. `FaninPool`/`DepListPool` `reclaim`/`ensure_space` take `PTO2SharedMemoryRingHeader&` directly (no `ring_id` or `fc` parameters). +Per-ring try-locks in the scheduler state prevent concurrent scheduler threads from interleaving watermark writes within the same ring. `FaninPool`/`DepListPool` `reclaim`/`ensure_space` take `SharedMemoryRingHeader&` directly (no `ring_id` or `fc` parameters). -### 4.4 PTO2SharedMemoryHandle (lifecycle-only) +### 4.4 SharedMemoryHandle (lifecycle-only) -Slimmed to lifecycle management only. Per-ring data pointers now live in `PTO2SharedMemoryRingHeader` (§4.3). Runtime components (orchestrator, scheduler) store `PTO2SharedMemoryHeader*` directly, eliminating one indirection on every per-ring access. +Slimmed to lifecycle management only. Per-ring data pointers now live in `SharedMemoryRingHeader` (§4.3). Runtime components (orchestrator, scheduler) store `SharedMemoryHeader*` directly, eliminating one indirection on every per-ring access. ```cpp -struct PTO2SharedMemoryHandle { +struct SharedMemoryHandle { void *sm_base; uint64_t sm_size; - PTO2SharedMemoryHeader *header; + SharedMemoryHeader *header; bool is_owner; }; ``` @@ -131,7 +131,7 @@ struct PTO2SharedMemoryHandle { ```cpp struct RingSchedState { // Cache Line 0: ring pointer (read-only) + hot path (read-write) - PTO2SharedMemoryRingHeader *ring; // direct pointer, no indirection + SharedMemoryRingHeader *ring; // direct pointer, no indirection int32_t last_task_alive; std::atomic advance_lock; // multi-thread CAS diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index bfdfa8e73f..e542cc12bb 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -146,7 +146,7 @@ The orchestrator and schedulers communicate through a contiguous shared memory r ```text ┌─────────────────────────────┐ offset 0 -│ PTO2SharedMemoryHeader │ (per-ring flow control + layout, global flags) +│ SharedMemoryHeader │ (per-ring flow control + layout, global flags) ├─────────────────────────────┤ aligned │ Per-ring regions ×4: │ │ TaskDescriptor[N] │ N = task_window_size per ring @@ -164,8 +164,8 @@ The orchestrator and schedulers communicate through a contiguous shared memory r | `heap_top` | Orchestrator | Scheduler | Heap ring allocation pointer | | `heap_tail` | Scheduler | Orchestrator | Heap ring reclamation pointer | | `orchestrator_done` | Orchestrator | Scheduler | Signals orchestration completion | -| `task_window_size` | Init | Both | Number of task slots (per-ring, in `PTO2SharedMemoryRingHeader`) | -| `heap_size` | Init | Both | Heap total size (per-ring, in `PTO2SharedMemoryRingHeader`) | +| `task_window_size` | Init | Both | Number of task slots (per-ring, in `SharedMemoryRingHeader`) | +| `heap_size` | Init | Both | Heap total size (per-ring, in `SharedMemoryRingHeader`) | | `task_descriptors_offset` | Init | Both | Offset to TaskDescriptor array in SM (per-ring) | | `total_size` | Init | Both | Total shared memory size | @@ -175,7 +175,7 @@ The orchestrator and schedulers communicate through a contiguous shared memory r total = ALIGN(Header) + Σ_ring [ ALIGN(window_size * sizeof(TaskDescriptor)) + ALIGN(window_size * sizeof(TaskPayload)) - + ALIGN(window_size * sizeof(TaskSlotState)) ] + + ALIGN(window_size * sizeof(ChipTaskSlotState)) ] ``` Alignment is 64 bytes (`PTO2_ALIGN_SIZE`). diff --git a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index 7158e4d405..9e321bdb10 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -284,7 +284,7 @@ static bool resolve_ring_config( return true; } -static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, PTO2SharedMemoryHeader *host_header) { +static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { if (runtime == nullptr || host_header == nullptr) { return 0; } @@ -294,7 +294,7 @@ static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, PT return 0; } - int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(PTO2SharedMemoryHeader)); + int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(SharedMemoryHeader)); if (hdr_rc != 0) { LOG_WARN("Failed to copy PTO2 header from device"); return 0; @@ -570,7 +570,7 @@ static bool derive_arena_static_sizes(const ArenaSizingConfig &sizing, ArenaStat } out->total_heap += sizing.heap_sizes[r]; } - out->sm_size = PTO2SharedMemoryHandle::calculate_size_per_ring(sizing.task_window_sizes); + out->sm_size = SharedMemoryHandle::calculate_size_per_ring(sizing.task_window_sizes); return true; } @@ -1017,7 +1017,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e bool skip_tensor_copy_back = execution_rc != 0; int32_t runtime_status = 0; - PTO2SharedMemoryHeader host_header; + SharedMemoryHeader host_header; memset(&host_header, 0, sizeof(host_header)); if (execution_rc != 0) { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index c47fcd1d99..cc13d7ba3d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -967,7 +967,7 @@ static TaskOutputTensors submit_task_common( return result; } uint8_t dep_ring_id = dep_task_id.ring(); - PTO2SharedMemoryRingHeader &dep_ring = orch->sm_header->rings[dep_ring_id]; + SharedMemoryRingHeader &dep_ring = orch->sm_header->rings[dep_ring_id]; int32_t dep_local_task_id = static_cast(dep_task_id.local()); int32_t dep_last_task_alive = dep_ring.fc.last_task_alive.load(std::memory_order_acquire); if (dep_local_task_id < dep_last_task_alive) { @@ -991,7 +991,7 @@ static TaskOutputTensors submit_task_common( auto runtime_emit = [&](TaskId producer_task_id, DepFlags kind) -> bool { uint8_t prod_ring = producer_task_id.ring(); - PTO2SharedMemoryRingHeader &producer_ring = orch->sm_header->rings[prod_ring]; + SharedMemoryRingHeader &producer_ring = orch->sm_header->rings[prod_ring]; int32_t prod_slot = producer_ring.get_slot_by_task_id(static_cast(producer_task_id.local())); ChipTaskSlotState *prod_state = &producer_ring.get_slot_state_by_slot(prod_slot); return append_fanin_or_fail( diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 50409074a5..4cc060dcd1 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -64,7 +64,7 @@ struct PTO2OrchestratorLayout { */ struct PTO2OrchestratorState { // === SHARED MEMORY ACCESS === - PTO2SharedMemoryHeader *sm_header; + SharedMemoryHeader *sm_header; // === PER-RING RESOURCES === PTO2RingSet rings[CHIP_MAX_RING_DEPTH]; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index d731d53da4..60d35b05c4 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -34,7 +34,7 @@ static void latch_pool_error(std::atomic *error_code_ptr, int32_t error // ============================================================================= // Fanin Spill Pool Implementation // ============================================================================= -void PTO2FaninPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { +void PTO2FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { if (sm_last_task_alive <= reclaim_task_cursor) return; int32_t scan_end = sm_last_task_alive; @@ -53,7 +53,7 @@ void PTO2FaninPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_ta reclaim_task_cursor = scan_end; } -bool PTO2FaninPool::ensure_space(PTO2SharedMemoryRingHeader &ring, int32_t needed) { +bool PTO2FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { if (available() >= needed) return true; int spin_count = 0; @@ -137,7 +137,7 @@ bool PTO2FaninPool::ensure_space(PTO2SharedMemoryRingHeader &ring, int32_t neede // ============================================================================= // Dependency List Pool Implementation // ============================================================================= -void PTO2DepListPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { +void PTO2DepListPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { if (sm_last_task_alive >= last_reclaimed + PTO2_DEP_POOL_CLEANUP_INTERVAL && sm_last_task_alive > 0) { int32_t mark = ring.get_slot_state_by_task_id(sm_last_task_alive - 1).dep_pool_mark; if (mark > 0) { @@ -148,7 +148,7 @@ void PTO2DepListPool::reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_ } void PTO2DepListPool::report_deadlock( - PTO2SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated + SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated ) { int32_t current = ring.fc.current_task_index.load(std::memory_order_acquire); LOG_ERROR("========================================"); @@ -189,9 +189,7 @@ void PTO2DepListPool::report_deadlock( LOG_ERROR("========================================"); } -bool PTO2DepListPool::ensure_space( - PTO2SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task -) { +bool PTO2DepListPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task) { if (available() >= needed) return true; int spin_count = 0; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index ec6a0ed111..59bd39d743 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -626,9 +626,9 @@ struct PTO2FaninPool { error_code_ptr = in_error_code_ptr; } - void reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_task_alive); + void reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive); - bool ensure_space(PTO2SharedMemoryRingHeader &ring, int32_t needed); + bool ensure_space(SharedMemoryRingHeader &ring, int32_t needed); PTO2FaninSpillEntry *alloc() { int32_t used = top - tail; @@ -832,7 +832,7 @@ struct PTO2DepListPool { * @param ring Ring header (for reading slot dep_pool_mark) * @param sm_last_task_alive Current last_task_alive from shared memory */ - void reclaim(PTO2SharedMemoryRingHeader &ring, int32_t sm_last_task_alive); + void reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive); /** * Ensure dep pool for a specific ring has at least `needed` entries available. @@ -842,14 +842,14 @@ struct PTO2DepListPool { * exact publication, then use a structural head-of-line check plus a * wall-clock backstop, each emitting report_deadlock. */ - bool ensure_space(PTO2SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task = nullptr); + bool ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task = nullptr); /** * Structured dep-pool deadlock report, mirroring PTO2TaskAllocator::report_deadlock. * scope_gated marks the provable head-of-line case where the head is pinned * by an open scope on this ring, as opposed to the wall-clock backstop. */ - void report_deadlock(PTO2SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated); + void report_deadlock(SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated); /** * Allocate a single entry from the pool (single-thread per pool instance) diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index 8b5d20e828..fcf6251129 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -156,7 +156,7 @@ struct RuntimeContext { PTO2ScopeMode pending_scope_mode; // Components - PTO2SharedMemoryHandle *sm_handle; + SharedMemoryHandle *sm_handle; PTO2OrchestratorState orchestrator; PTO2SchedulerState scheduler; AICoreCompletionMailbox *aicore_mailbox; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index b74aee70c6..a7cb9ce1d9 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -444,14 +444,14 @@ struct PTO2SchedulerLayout { */ struct PTO2SchedulerState { // Shared memory access - PTO2SharedMemoryHeader *sm_header; + SharedMemoryHeader *sm_header; // Per-ring state struct alignas(64) RingSchedState { static constexpr int32_t PUBLISH_INTERVAL_K = 16; // --- Cache Line 0: ring pointer (read-only) + hot path (read-write) --- - PTO2SharedMemoryRingHeader *ring; + SharedMemoryRingHeader *ring; int32_t last_task_alive; // Publication may trail local reclamation by PUBLISH_INTERVAL_K - 1 // tasks only while no reclaim consumer is blocked on the watermark. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 2fcc440b05..0f2f554882 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -36,7 +36,7 @@ // Returns true iff this call won the first-writer CAS for sched_error_code — the // caller may then write companion fields (e.g. the stall detail) knowing they // describe the same observation that owns the latched code. -static bool latch_scheduler_error(PTO2SharedMemoryHeader *header, int32_t thread_idx, int32_t error_code) { +static bool latch_scheduler_error(SharedMemoryHeader *header, int32_t thread_idx, int32_t error_code) { if (header == nullptr || error_code == SIMPLER_ERROR_NONE) { return false; } @@ -53,7 +53,7 @@ static bool latch_scheduler_error(PTO2SharedMemoryHeader *header, int32_t thread } LoopAction SchedulerContext::handle_orchestrator_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count ) { if (completed_.load(std::memory_order_acquire)) { return LoopAction::BREAK_LOOP; @@ -94,8 +94,7 @@ LoopAction SchedulerContext::handle_orchestrator_exit( return LoopAction::NONE; } -LoopAction -SchedulerContext::check_idle_fatal_error(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime) { +LoopAction SchedulerContext::check_idle_fatal_error(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime) { if (completed_.load(std::memory_order_acquire)) { return LoopAction::BREAK_LOOP; } @@ -236,7 +235,7 @@ void SchedulerContext::log_stall_diagnostics( if (thread_idx == 0) { int32_t cnt_ready = 0, cnt_waiting = 0, cnt_running = 0, submitted_in_ring = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - PTO2SharedMemoryRingHeader &ring = *sched_->ring_sched_states[r].ring; + SharedMemoryRingHeader &ring = *sched_->ring_sched_states[r].ring; int32_t ring_task_count = ring.fc.current_task_index.load(std::memory_order_relaxed); submitted_in_ring += ring_task_count; // Scan only live task_ids [last_task_alive, current_task_index): slots @@ -372,7 +371,7 @@ SchedulerContext::StallClassification SchedulerContext::classify_stall_reason() cls.stuck_core = -1; int32_t cnt_running = 0, cnt_ready = 0, cnt_waiting = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - PTO2SharedMemoryRingHeader &ring = *sched_->ring_sched_states[r].ring; + SharedMemoryRingHeader &ring = *sched_->ring_sched_states[r].ring; int32_t ring_task_count = ring.fc.current_task_index.load(std::memory_order_relaxed); // Active task_ids live in [last_task_alive, current_task_index); slots wrap // (slot = task_id % window), so scanning from 0 re-reads each live slot once @@ -423,7 +422,7 @@ SchedulerContext::StallClassification SchedulerContext::classify_stall_reason() } int32_t SchedulerContext::handle_timeout_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, int32_t last_progress_count #if SIMPLER_DFX , @@ -1161,7 +1160,7 @@ int32_t SchedulerContext::pre_handshake_init( // hs_setup_done_, which zeroes these ring counters, so the read completes here // (on the leader, before any thread is released) rather than post-handshake. if (runtime->get_gm_sm_ptr()) { - auto *header = static_cast(runtime->get_gm_sm_ptr()); + auto *header = static_cast(runtime->get_gm_sm_ptr()); int64_t task_count = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { int32_t ring_tasks = header->rings[r].fc.current_task_index.load(std::memory_order_acquire); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index fec3a2ef64..a5f80f8b55 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -408,10 +408,10 @@ class SchedulerContext { // ========================================================================= __attribute__((noinline, cold)) LoopAction - handle_orchestrator_exit(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count); + handle_orchestrator_exit(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t &task_count); __attribute__((noinline, cold)) LoopAction - check_idle_fatal_error(int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime); + check_idle_fatal_error(int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime); __attribute__((noinline, cold)) void log_stall_diagnostics(int32_t thread_idx, int32_t task_count, int32_t idle_iterations, int32_t last_progress_count); @@ -462,7 +462,7 @@ class SchedulerContext { __attribute__((noinline, cold)) StallClassification classify_stall_reason() const; __attribute__((noinline, cold)) int32_t handle_timeout_exit( - int32_t thread_idx, PTO2SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, + int32_t thread_idx, SharedMemoryHeader *header, Runtime *runtime, int32_t idle_iterations, int32_t last_progress_count #if SIMPLER_DFX , diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 6e798ccd04..6962605b6c 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -843,7 +843,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ always_assert(sched_ != nullptr); CoreTracker &tracker = core_trackers_[thread_idx]; - PTO2SharedMemoryHeader *header = sched_->sm_header; + SharedMemoryHeader *header = sched_->sm_header; if (!header) { LOG_ERROR("PTO2 dispatch: header is null"); return -1; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 11c7fa190d..cacdf0b61d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -88,7 +88,7 @@ void ready_queue_destroy(PTO2ReadyQueue *queue) { bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base, int32_t ring_id) { // ring stores the device address of the SM ring header — pure offset // arithmetic, no SM load. - ring = pto2_sm_layout::ring_header_addr(sm_dev_base, ring_id); + ring = sm_layout::ring_header_addr(sm_dev_base, ring_id); last_task_alive = 0; last_published_to_sm = 0; publication_batching_enabled = false; @@ -99,7 +99,7 @@ bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base #endif // Per-slot SM-side initialization (bind_ring + reset_for_reuse + - // fanin_count/active_mask zero) lives in PTO2SharedMemoryHandle:: + // fanin_count/active_mask zero) lives in SharedMemoryHandle:: // init_header_per_ring so the AICPU performs it during SM reset; host // prebuilt-arena init skips SM access here. @@ -109,7 +109,7 @@ bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base void PTO2SchedulerState::RingSchedState::reset_for_reuse( void *sm_dev_base, int32_t ring_id, std::atomic *orch_err ) { - ring = pto2_sm_layout::ring_header_addr(sm_dev_base, ring_id); + ring = sm_layout::ring_header_addr(sm_dev_base, ring_id); last_task_alive = 0; last_published_to_sm = 0; publication_batching_enabled = false; @@ -166,7 +166,7 @@ bool PTO2SchedulerState::init_data_from_layout( const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base ) { PTO2SchedulerState *sched = this; - sched->sm_header = reinterpret_cast(sm_dev_base); + sched->sm_header = reinterpret_cast(sm_dev_base); sched->advance_pending_mask.store(0, std::memory_order_relaxed); sched->publication_request_mask.store(0, std::memory_order_relaxed); sched->publication_ack_mask.store(0, std::memory_order_relaxed); @@ -215,7 +215,7 @@ bool PTO2SchedulerState::init_data_from_layout( return false; } - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_dev_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { auto *dep_entries = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); memset(dep_entries, 0, static_cast(layout.dep_pool_capacities[r]) * sizeof(PTO2DepListEntry)); @@ -230,7 +230,7 @@ bool PTO2SchedulerState::init_data_from_layout( void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void *sm_dev_base) { PTO2SchedulerState *sched = this; - sched->sm_header = reinterpret_cast(sm_dev_base); + sched->sm_header = reinterpret_cast(sm_dev_base); sched->advance_pending_mask.store(0, std::memory_order_relaxed); sched->publication_request_mask.store(0, std::memory_order_relaxed); sched->publication_ack_mask.store(0, std::memory_order_relaxed); @@ -239,7 +239,7 @@ void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void sched->tasks_consumed.store(0, std::memory_order_relaxed); #endif - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_dev_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { sched->ring_sched_states[r].reset_for_reuse(sm_dev_base, r, orch_err); } @@ -381,7 +381,7 @@ bool PTO2OrchestratorState::init_data_from_layout( auto *orch = this; *orch = PTO2OrchestratorState{}; - orch->sm_header = reinterpret_cast(sm_dev_base); + orch->sm_header = reinterpret_cast(sm_dev_base); orch->gm_heap_base = gm_heap; uint64_t total_heap_size = 0; if (!sum_ring_heap_sizes(heap_sizes, &total_heap_size)) { @@ -390,14 +390,14 @@ bool PTO2OrchestratorState::init_data_from_layout( orch->gm_heap_size = total_heap_size; orch->fatal = false; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_dev_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); uint64_t heap_offset = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { void *ring_heap_base = reinterpret_cast(gm_heap) + heap_offset; - auto *task_descs_dev = pto2_sm_layout::ring_task_descriptors_addr(sm_dev_base, task_window_sizes, r); - auto *slot_states_dev = pto2_sm_layout::ring_slot_states_addr(sm_dev_base, task_window_sizes, r); - auto *cur_idx_dev = pto2_sm_layout::ring_current_task_index_addr(sm_dev_base, r); - auto *last_alive_dev = pto2_sm_layout::ring_last_task_alive_addr(sm_dev_base, r); + auto *task_descs_dev = sm_layout::ring_task_descriptors_addr(sm_dev_base, task_window_sizes, r); + auto *slot_states_dev = sm_layout::ring_slot_states_addr(sm_dev_base, task_window_sizes, r); + auto *cur_idx_dev = sm_layout::ring_current_task_index_addr(sm_dev_base, r); + auto *last_alive_dev = sm_layout::ring_last_task_alive_addr(sm_dev_base, r); orch->rings[r].task_allocator.init( task_descs_dev, static_cast(task_window_sizes[r]), cur_idx_dev, last_alive_dev, ring_heap_base, @@ -438,7 +438,7 @@ bool PTO2OrchestratorState::reset_for_reuse( const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ) { auto *orch = this; - orch->sm_header = reinterpret_cast(sm_dev_base); + orch->sm_header = reinterpret_cast(sm_dev_base); orch->gm_heap_base = gm_heap; uint64_t total_heap_size = 0; if (!sum_ring_heap_sizes(heap_sizes, &total_heap_size)) { @@ -460,14 +460,14 @@ bool PTO2OrchestratorState::reset_for_reuse( } orch->fanin_seen_current_epoch = next_epoch; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_dev_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); uint64_t heap_offset = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { void *ring_heap_base = reinterpret_cast(gm_heap) + heap_offset; - auto *task_descs_dev = pto2_sm_layout::ring_task_descriptors_addr(sm_dev_base, task_window_sizes, r); - auto *slot_states_dev = pto2_sm_layout::ring_slot_states_addr(sm_dev_base, task_window_sizes, r); - auto *cur_idx_dev = pto2_sm_layout::ring_current_task_index_addr(sm_dev_base, r); - auto *last_alive_dev = pto2_sm_layout::ring_last_task_alive_addr(sm_dev_base, r); + auto *task_descs_dev = sm_layout::ring_task_descriptors_addr(sm_dev_base, task_window_sizes, r); + auto *slot_states_dev = sm_layout::ring_slot_states_addr(sm_dev_base, task_window_sizes, r); + auto *cur_idx_dev = sm_layout::ring_current_task_index_addr(sm_dev_base, r); + auto *last_alive_dev = sm_layout::ring_last_task_alive_addr(sm_dev_base, r); orch->rings[r].task_allocator.init( task_descs_dev, static_cast(task_window_sizes[r]), cur_idx_dev, last_alive_dev, ring_heap_base, @@ -564,7 +564,7 @@ RuntimeArenaLayout runtime_reserve_layout( layout.sizing.dep_pool_capacities[r] = dep_pool_capacities[r]; } - layout.offsets.off_sm_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + layout.offsets.off_sm_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); int32_t task_window_sizes_i32[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { task_window_sizes_i32[r] = static_cast(task_window_sizes[r]); @@ -596,7 +596,7 @@ RuntimeContext *runtime_init_data_from_layout( RuntimeContext *rt = static_cast(arena.region_ptr(layout.offsets.off_runtime)); memset(rt, 0, sizeof(*rt)); - auto *sm_wrap = static_cast(arena.region_ptr(layout.offsets.off_sm_handle)); + auto *sm_wrap = static_cast(arena.region_ptr(layout.offsets.off_sm_handle)); memset(sm_wrap, 0, sizeof(*sm_wrap)); // rt->ops is filled by the AICPU at boot. @@ -651,7 +651,7 @@ static void enable_publication_batching_if_safe(RuntimeContext *rt) { void runtime_wire_arena_pointers(DeviceArena &arena, const RuntimeArenaLayout &layout, RuntimeContext *rt) { rt->scheduler.set_publication_batching_enabled(false); - rt->sm_handle = static_cast(arena.region_ptr(layout.offsets.off_sm_handle)); + rt->sm_handle = static_cast(arena.region_ptr(layout.offsets.off_sm_handle)); rt->aicore_mailbox = static_cast(arena.region_ptr(layout.offsets.off_mailbox)); rt->orchestrator.wire_arena_pointers(layout.offsets.orch, arena, &rt->scheduler); rt->scheduler.wire_arena_pointers(layout.offsets.sched, arena); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index 7bb8da7fd8..0912b8aa53 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -27,7 +27,7 @@ // Size Calculation // ============================================================================= -uint64_t PTO2SharedMemoryHandle::calculate_size(uint64_t task_window_size) { +uint64_t SharedMemoryHandle::calculate_size(uint64_t task_window_size) { uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { task_window_sizes[r] = task_window_size; @@ -35,25 +35,25 @@ uint64_t PTO2SharedMemoryHandle::calculate_size(uint64_t task_window_size) { return calculate_size_per_ring(task_window_sizes); } -uint64_t PTO2SharedMemoryHandle::calculate_size_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]) { +uint64_t SharedMemoryHandle::calculate_size_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]) { // Total SM size = offset just past the last ring, from the single source of - // truth for the layout (pto2_sm_layout::ring_segment_offsets). - return pto2_sm_layout::ring_segment_offsets(task_window_sizes, CHIP_MAX_RING_DEPTH - 1).end; + // truth for the layout (sm_layout::ring_segment_offsets). + return sm_layout::ring_segment_offsets(task_window_sizes, CHIP_MAX_RING_DEPTH - 1).end; } // ============================================================================= // Creation and Destruction // ============================================================================= -void PTO2SharedMemoryHandle::setup_pointers_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]) { +void SharedMemoryHandle::setup_pointers_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]) { char *base = (char *)sm_base; - header = (PTO2SharedMemoryHeader *)base; + header = (SharedMemoryHeader *)base; // Per-ring descriptors / payloads / slot_states — offsets from the single - // source of truth (pto2_sm_layout::ring_segment_offsets), so this setup and + // source of truth (sm_layout::ring_segment_offsets), so this setup and // the device-address helpers cannot drift. for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - auto off = pto2_sm_layout::ring_segment_offsets(task_window_sizes, r); + auto off = sm_layout::ring_segment_offsets(task_window_sizes, r); auto &ring = header->rings[r]; ring.task_descriptors = (TaskDescriptor *)(base + off.descriptors); ring.task_payloads = (TaskPayload *)(base + off.payloads); @@ -61,7 +61,7 @@ void PTO2SharedMemoryHandle::setup_pointers_per_ring(const uint64_t task_window_ } } -void PTO2SharedMemoryHandle::setup_pointers(uint64_t task_window_size) { +void SharedMemoryHandle::setup_pointers(uint64_t task_window_size) { uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { task_window_sizes[r] = task_window_size; @@ -69,9 +69,7 @@ void PTO2SharedMemoryHandle::setup_pointers(uint64_t task_window_size) { setup_pointers_per_ring(task_window_sizes); } -bool PTO2SharedMemoryHandle::init( - void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size, uint64_t heap_size -) { +bool SharedMemoryHandle::init(void *sm_base_arg, uint64_t sm_size_arg, uint64_t task_window_size, uint64_t heap_size) { uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]; uint64_t heap_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -81,7 +79,7 @@ bool PTO2SharedMemoryHandle::init( return init_per_ring(sm_base_arg, sm_size_arg, task_window_sizes, heap_sizes); } -bool PTO2SharedMemoryHandle::init_per_ring( +bool SharedMemoryHandle::init_per_ring( void *sm_base_arg, uint64_t sm_size_arg, const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH] ) { @@ -96,13 +94,13 @@ bool PTO2SharedMemoryHandle::init_per_ring( return true; } -PTO2SharedMemoryHandle *PTO2SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { +SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); - const size_t off_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(buffer_size), PTO2_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; - auto *handle = static_cast(arena.region_ptr(off_handle)); + auto *handle = static_cast(arena.region_ptr(off_handle)); memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); memset(buffer, 0, static_cast(buffer_size)); @@ -110,7 +108,7 @@ PTO2SharedMemoryHandle *PTO2SharedMemoryHandle::create_and_init_default(DeviceAr return handle; } -void PTO2SharedMemoryHandle::destroy() { +void SharedMemoryHandle::destroy() { // Arena-owned wrappers (is_owner == false) are reclaimed by arena.release(); // calling destroy on them is a no-op so existing callers stay safe. if (is_owner && sm_base) { @@ -124,7 +122,7 @@ void PTO2SharedMemoryHandle::destroy() { // ============================================================================= // // no need init data in pool, init pool data when used -void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size, uint64_t heap_size) { +void SharedMemoryHandle::init_header(uint64_t task_window_size, uint64_t heap_size) { uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]; uint64_t heap_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -134,7 +132,7 @@ void PTO2SharedMemoryHandle::init_header(uint64_t task_window_size, uint64_t hea init_header_per_ring(task_window_sizes, heap_sizes); } -void PTO2SharedMemoryHandle::init_header_per_ring( +void SharedMemoryHandle::init_header_per_ring( const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH] ) { // Per-ring flow control (start at 0) @@ -145,7 +143,7 @@ void PTO2SharedMemoryHandle::init_header_per_ring( header->orchestrator_done.store(0, std::memory_order_relaxed); // Per-ring layout info - uint64_t offset = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t offset = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { header->rings[r].task_window_size = task_window_sizes[r]; header->rings[r].task_window_mask = static_cast(task_window_sizes[r] - 1); @@ -183,10 +181,10 @@ void PTO2SharedMemoryHandle::init_header_per_ring( // Debug Utilities // ============================================================================= -void PTO2SharedMemoryHandle::print_layout() { +void SharedMemoryHandle::print_layout() { if (!header) return; - PTO2SharedMemoryHeader *h = header; + SharedMemoryHeader *h = header; LOG_DEBUG("=== PTO2 Shared Memory Layout ==="); LOG_DEBUG("Base address: %p", sm_base); @@ -212,11 +210,11 @@ void PTO2SharedMemoryHandle::print_layout() { LOG_DEBUG("================================"); } -bool PTO2SharedMemoryHandle::validate() { +bool SharedMemoryHandle::validate() { if (!sm_base) return false; if (!header) return false; - PTO2SharedMemoryHeader *h = header; + SharedMemoryHeader *h = header; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { if (!h->rings[r].fc.validate(this, r)) return false; @@ -225,12 +223,12 @@ bool PTO2SharedMemoryHandle::validate() { return true; } -bool PTO2RingFlowControl::validate(PTO2SharedMemoryHandle *handle, int32_t ring_id) const { +bool PTO2RingFlowControl::validate(SharedMemoryHandle *handle, int32_t ring_id) const { if (!handle) return false; if (!handle->header) return false; if (ring_id < 0 || ring_id >= CHIP_MAX_RING_DEPTH) return false; - const PTO2SharedMemoryHeader *h = handle->header; + const SharedMemoryHeader *h = handle->header; // Check that offsets are within bounds if (h->rings[ring_id].task_descriptors_offset >= h->total_size) return false; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 421db9da1a..815bc09875 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -14,19 +14,19 @@ * Defines the shared memory structure for Orchestrator-Scheduler communication. * * Memory Layout (per-ring sections repeat for each ring 0..CHIP_MAX_RING_DEPTH-1): - * +---------------------------+ - * | SharedMemoryHeader | (per-ring flow control + sync) - * +---------------------------+ - * | Ring 0: TaskDescriptor[] | - * | Ring 0: TaskPayload[] | - * | Ring 0: TaskSlotState[] | - * +---------------------------+ - * | Ring 1: TaskDescriptor[] | - * | Ring 1: TaskPayload[] | - * | Ring 1: TaskSlotState[] | - * +---------------------------+ - * | ... | - * +---------------------------+ + * +-------------------------------+ + * | SharedMemoryHeader | (per-ring flow control + sync) + * +-------------------------------+ + * | Ring 0: TaskDescriptor[] | + * | Ring 0: TaskPayload[] | + * | Ring 0: ChipTaskSlotState[] | + * +-------------------------------+ + * | Ring 1: TaskDescriptor[] | + * | Ring 1: TaskPayload[] | + * | Ring 1: ChipTaskSlotState[] | + * +-------------------------------+ + * | ... | + * +-------------------------------+ * * Design principles: * - Only data needed for Orchestrator<->Scheduler communication is here @@ -47,7 +47,7 @@ // Shared Memory Header // ============================================================================= -struct PTO2SharedMemoryHandle; +struct SharedMemoryHandle; /** * Per-ring flow control state in shared memory. @@ -72,7 +72,7 @@ struct alignas(64) PTO2RingFlowControl { last_task_alive.store(0, std::memory_order_relaxed); } - bool validate(PTO2SharedMemoryHandle *handle, int32_t ring_id) const; + bool validate(SharedMemoryHandle *handle, int32_t ring_id) const; }; static_assert(sizeof(PTO2RingFlowControl) == 128, "PTO2RingFlowControl must be exactly 2 cache lines (128B)"); @@ -83,7 +83,7 @@ static_assert(sizeof(PTO2RingFlowControl) == 128, "PTO2RingFlowControl must be e * Groups flow-control, layout info, and per-ring data pointers for a single ring. * Pointers are host-side only (set by setup_pointers, invalid on device). */ -struct alignas(64) PTO2SharedMemoryRingHeader { +struct alignas(64) SharedMemoryRingHeader { PTO2RingFlowControl fc; // Layout metadata (set once at init) @@ -114,10 +114,10 @@ struct alignas(64) PTO2SharedMemoryRingHeader { } }; -static_assert(sizeof(PTO2SharedMemoryRingHeader) == 192, "PTO2SharedMemoryRingHeader layout drift"); +static_assert(sizeof(SharedMemoryRingHeader) == 192, "SharedMemoryRingHeader layout drift"); static_assert( - offsetof(PTO2SharedMemoryRingHeader, task_descriptors_offset) == 152, - "PTO2SharedMemoryRingHeader task_descriptors_offset layout drift" + offsetof(SharedMemoryRingHeader, task_descriptors_offset) == 152, + "SharedMemoryRingHeader task_descriptors_offset layout drift" ); /** @@ -125,9 +125,9 @@ static_assert( * * Contains per-ring flow control and global layout information. */ -struct alignas(PTO2_ALIGN_SIZE) PTO2SharedMemoryHeader { +struct alignas(PTO2_ALIGN_SIZE) SharedMemoryHeader { // === PER-RING FLOW CONTROL + LAYOUT INFO (set once at init) === - PTO2SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; + SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; // === GLOBAL FIELDS === std::atomic orchestrator_done; // Flag: orchestration complete @@ -163,14 +163,11 @@ struct alignas(PTO2_ALIGN_SIZE) PTO2SharedMemoryHeader { std::atomic sched_stall_core; // S1: stuck core id (-1 if N/A) }; -static_assert(sizeof(PTO2SharedMemoryHeader) == 896, "PTO2SharedMemoryHeader layout drift"); -static_assert(offsetof(PTO2SharedMemoryHeader, total_size) == 776, "PTO2SharedMemoryHeader total_size layout drift"); +static_assert(sizeof(SharedMemoryHeader) == 896, "SharedMemoryHeader layout drift"); +static_assert(offsetof(SharedMemoryHeader, total_size) == 776, "SharedMemoryHeader total_size layout drift"); +static_assert(offsetof(SharedMemoryHeader, orch_error_code) == 784, "SharedMemoryHeader orch_error_code layout drift"); static_assert( - offsetof(PTO2SharedMemoryHeader, orch_error_code) == 784, "PTO2SharedMemoryHeader orch_error_code layout drift" -); -static_assert( - offsetof(PTO2SharedMemoryHeader, sched_stall_task_id) == 832, - "PTO2SharedMemoryHeader sched_stall_task_id layout drift" + offsetof(SharedMemoryHeader, sched_stall_task_id) == 832, "SharedMemoryHeader sched_stall_task_id layout drift" ); // ============================================================================= @@ -179,13 +176,13 @@ static_assert( /** * Handle for shared memory lifecycle management (create/destroy). - * Runtime components (orchestrator, scheduler) use PTO2SharedMemoryHeader* directly. + * Runtime components (orchestrator, scheduler) use SharedMemoryHeader* directly. */ -struct PTO2SharedMemoryHandle { +struct SharedMemoryHandle { void *sm_base; // Base address of shared memory uint64_t sm_size; // Total size of shared memory - PTO2SharedMemoryHeader *header; + SharedMemoryHeader *header; // Ownership flag bool is_owner; // True if this handle allocated the memory @@ -199,7 +196,7 @@ struct PTO2SharedMemoryHandle { // using default PTO2_TASK_WINDOW_SIZE / PTO2_HEAP_SIZE. Only valid when the // arena is otherwise empty (the call performs the single commit). All // memory is owned by the arena — caller must not call destroy(). - static PTO2SharedMemoryHandle *create_and_init_default(DeviceArena &arena); + static SharedMemoryHandle *create_and_init_default(DeviceArena &arena); // === Instance methods === @@ -238,33 +235,33 @@ struct PTO2SharedMemoryHandle { // // These helpers compute those addresses by offset arithmetic on the SM // device base. Pure pointer math, no loads/stores; safe to call from host. -// The same arithmetic happens on AICPU too (via PTO2SharedMemoryHandle's +// The same arithmetic happens on AICPU too (via SharedMemoryHandle's // own setup_pointers), so values are guaranteed consistent across sides. -namespace pto2_sm_layout { +namespace sm_layout { inline std::atomic *orch_error_code_addr(void *sm_dev_base) noexcept { return reinterpret_cast *>( - static_cast(sm_dev_base) + offsetof(PTO2SharedMemoryHeader, orch_error_code) + static_cast(sm_dev_base) + offsetof(SharedMemoryHeader, orch_error_code) ); } -inline PTO2SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base, int ring_id) noexcept { - return reinterpret_cast( - static_cast(sm_dev_base) + offsetof(PTO2SharedMemoryHeader, rings) + - static_cast(ring_id) * sizeof(PTO2SharedMemoryRingHeader) +inline SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base, int ring_id) noexcept { + return reinterpret_cast( + static_cast(sm_dev_base) + offsetof(SharedMemoryHeader, rings) + + static_cast(ring_id) * sizeof(SharedMemoryRingHeader) ); } inline std::atomic *ring_current_task_index_addr(void *sm_dev_base, int ring_id) noexcept { return reinterpret_cast *>( - reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(PTO2SharedMemoryRingHeader, fc) + + reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(SharedMemoryRingHeader, fc) + offsetof(PTO2RingFlowControl, current_task_index) ); } inline std::atomic *ring_last_task_alive_addr(void *sm_dev_base, int ring_id) noexcept { return reinterpret_cast *>( - reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(PTO2SharedMemoryRingHeader, fc) + + reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(SharedMemoryRingHeader, fc) + offsetof(PTO2RingFlowControl, last_task_alive) ); } @@ -287,8 +284,8 @@ struct PTO2RingSegmentOffsets { // layout walk can never silently disagree across call sites. inline PTO2RingSegmentOffsets ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int ring_id) noexcept { - assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "pto2_sm_layout: ring_id out of range"); - uint64_t off = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "sm_layout: ring_id out of range"); + uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); for (int r = 0; r < ring_id; r++) { off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); @@ -323,4 +320,4 @@ ring_slot_states_addr(void *sm_dev_base, const uint64_t task_window_sizes[CHIP_M ); } -} // namespace pto2_sm_layout +} // namespace sm_layout diff --git a/tests/ut/cpp/a2a3/test_dep_list_pool.cpp b/tests/ut/cpp/a2a3/test_dep_list_pool.cpp index 4ca1a857ef..160f4be764 100644 --- a/tests/ut/cpp/a2a3/test_dep_list_pool.cpp +++ b/tests/ut/cpp/a2a3/test_dep_list_pool.cpp @@ -84,7 +84,7 @@ TEST_F(DepListPoolTest, EnsureSpaceDeadlockReturnsFalseAndLatchesError) { ASSERT_NE(pool.alloc(), nullptr); } - PTO2SharedMemoryRingHeader ring{}; + SharedMemoryRingHeader ring{}; ring.fc.init(); ring.fc.current_task_index.store(POOL_CAP + 1, std::memory_order_release); ring.fc.last_task_alive.store(0, std::memory_order_release); diff --git a/tests/ut/cpp/a2a3/test_fanin_pool.cpp b/tests/ut/cpp/a2a3/test_fanin_pool.cpp index 790814ba54..586539b36e 100644 --- a/tests/ut/cpp/a2a3/test_fanin_pool.cpp +++ b/tests/ut/cpp/a2a3/test_fanin_pool.cpp @@ -91,7 +91,7 @@ TEST_F(FaninPoolTest, EnsureSpaceDeadlockReturnsFalseAndLatchesError) { ASSERT_NE(pool.alloc(), nullptr); } - PTO2SharedMemoryRingHeader ring{}; + SharedMemoryRingHeader ring{}; ring.fc.init(); ring.fc.current_task_index.store(POOL_CAP + 1, std::memory_order_release); ring.fc.last_task_alive.store(0, std::memory_order_release); diff --git a/tests/ut/cpp/a2a3/test_graph_activation.cpp b/tests/ut/cpp/a2a3/test_graph_activation.cpp index c55180a193..6d33bed597 100644 --- a/tests/ut/cpp/a2a3/test_graph_activation.cpp +++ b/tests/ut/cpp/a2a3/test_graph_activation.cpp @@ -33,12 +33,12 @@ class GraphActivationTest : public ::testing::Test { protected: DeviceArena sm_arena; DeviceArena runtime_arena; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; PTO2SchedulerState sched{}; PTO2SchedulerLayout sched_layout{}; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); diff --git a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp index 09bbcf54fd..7039a6b3bc 100644 --- a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp @@ -46,14 +46,14 @@ class HbgSubmitPoisonTest : public ::testing::Test { protected: DeviceArena sm_arena; DeviceArena runtime_arena; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; PTO2SchedulerState sched{}; PTO2SchedulerLayout sched_layout{}; std::vector gm_heap; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); gm_heap.resize(4096); diff --git a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp index 2eb7fff030..085ee024ff 100644 --- a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp @@ -23,7 +23,7 @@ class OrchestratorFaninTest : public ::testing::Test { protected: DeviceArena sm_arena; DeviceArena runtime_arena; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; PTO2SchedulerState sched{}; PTO2OrchestratorLayout orch_layout{}; @@ -31,7 +31,7 @@ class OrchestratorFaninTest : public ::testing::Test { std::vector gm_heap; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); gm_heap.resize(4096 * CHIP_MAX_RING_DEPTH); diff --git a/tests/ut/cpp/a2a3/test_scheduler_state.cpp b/tests/ut/cpp/a2a3/test_scheduler_state.cpp index c4ad461157..357c537f4a 100644 --- a/tests/ut/cpp/a2a3/test_scheduler_state.cpp +++ b/tests/ut/cpp/a2a3/test_scheduler_state.cpp @@ -54,7 +54,7 @@ TEST(SyncStartDrainAttemptTest, LateAckCannotSatisfyNextAttemptBarrier) { class SchedulerStateTest : public ::testing::Test { protected: PTO2SchedulerState sched; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -66,7 +66,7 @@ class SchedulerStateTest : public ::testing::Test { int slot_payload_pool_idx_ = 0; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); auto layout = PTO2SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); @@ -102,8 +102,8 @@ class SchedulerStateTest : public ::testing::Test { } void init_ring_slot( - PTO2SharedMemoryRingHeader &ring, int32_t task_id, PTO2TaskState state, uint8_t ring_id, - uint32_t fanout_count = 1, uint32_t fanout_refcount = 1 + SharedMemoryRingHeader &ring, int32_t task_id, PTO2TaskState state, uint8_t ring_id, uint32_t fanout_count = 1, + uint32_t fanout_refcount = 1 ) { ChipTaskSlotState &slot = ring.get_slot_state_by_task_id(task_id); TaskPayload &payload = ring.get_payload_by_task_id(task_id); @@ -129,7 +129,7 @@ class SchedulerStateTest : public ::testing::Test { } void setup_ring_for_reclaim_race(int32_t ring_id, int32_t current_task_index, int32_t blocked_task_id) { - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ring.fc.current_task_index.store(current_task_index, std::memory_order_release); @@ -145,7 +145,7 @@ class SchedulerStateTest : public ::testing::Test { } void setup_contended_head_case(int32_t ring_id, int32_t head_task_id) { - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ring.fc.current_task_index.store(head_task_id + 2, std::memory_order_release); ring.fc.last_task_alive.store(head_task_id, std::memory_order_release); @@ -184,7 +184,7 @@ TEST_F(SchedulerStateTest, ConsumedHeadAdvancesAfterContendedAdvanceLock) { constexpr int32_t head_task_id = 0; setup_ring_for_reclaim_race(ring_id, /*current_task_index=*/1, head_task_id); - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); @@ -230,7 +230,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadSetsPendingAndIdleDrainAdvances) constexpr int32_t head_task_id = 17; setup_contended_head_case(ring_id, head_task_id); - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); @@ -261,7 +261,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadIdleDrainStress) { SCOPED_TRACE(::testing::Message() << "ring_id=" << ring_id << " head_task_id=" << head_task_id); setup_contended_head_case(ring_id, head_task_id); - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); diff --git a/tests/ut/cpp/a2a3/test_shared_memory.cpp b/tests/ut/cpp/a2a3/test_shared_memory.cpp index aa45f780a0..22b3b73bd2 100644 --- a/tests/ut/cpp/a2a3/test_shared_memory.cpp +++ b/tests/ut/cpp/a2a3/test_shared_memory.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2SharedMemory layout from shared_memory.h + * Unit tests for SharedMemory layout from shared_memory.h * * Tests creation, validation, per-ring independence, alignment, size * calculation, and error handling under the DeviceArena-backed init model: @@ -29,13 +29,13 @@ namespace { // Reserve + commit a fresh handle + sm_base on `arena` and run init. // Returns the wrapper pointer (arena-owned) or nullptr on init failure. -PTO2SharedMemoryHandle *make_handle(DeviceArena &arena, uint64_t task_window_size, uint64_t heap_size) { - const uint64_t sm_size = PTO2SharedMemoryHandle::calculate_size(task_window_size); - const size_t off_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); +SharedMemoryHandle *make_handle(DeviceArena &arena, uint64_t task_window_size, uint64_t heap_size) { + const uint64_t sm_size = SharedMemoryHandle::calculate_size(task_window_size); + const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; - auto *handle = static_cast(arena.region_ptr(off_handle)); + auto *handle = static_cast(arena.region_ptr(off_handle)); std::memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); std::memset(buffer, 0, static_cast(sm_size)); @@ -52,10 +52,10 @@ PTO2SharedMemoryHandle *make_handle(DeviceArena &arena, uint64_t task_window_siz class SharedMemoryTest : public ::testing::Test { protected: DeviceArena arena; - PTO2SharedMemoryHandle *handle = nullptr; + SharedMemoryHandle *handle = nullptr; void SetUp() override { - handle = PTO2SharedMemoryHandle::create_and_init_default(arena); + handle = SharedMemoryHandle::create_and_init_default(arena); ASSERT_NE(handle, nullptr); } @@ -165,7 +165,7 @@ TEST_F(SharedMemoryTest, HeaderAlignment) { // Descriptor and payload regions don't overlap within or across rings. TEST(SharedMemoryLayout, RegionsNonOverlapping) { DeviceArena arena; - PTO2SharedMemoryHandle *h = make_handle(arena, /*ws=*/64, /*heap=*/4096); + SharedMemoryHandle *h = make_handle(arena, /*ws=*/64, /*heap=*/4096); ASSERT_NE(h, nullptr); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -188,37 +188,37 @@ TEST(SharedMemoryLayout, RegionsNonOverlapping) { // ============================================================================= TEST(SharedMemoryCalcSize, NonZero) { - uint64_t size = PTO2SharedMemoryHandle::calculate_size(PTO2_TASK_WINDOW_SIZE); + uint64_t size = SharedMemoryHandle::calculate_size(PTO2_TASK_WINDOW_SIZE); EXPECT_GT(size, 0u); } TEST(SharedMemoryCalcSize, LargerWindowGivesLargerSize) { - uint64_t small_size = PTO2SharedMemoryHandle::calculate_size(64); - uint64_t large_size = PTO2SharedMemoryHandle::calculate_size(256); + uint64_t small_size = SharedMemoryHandle::calculate_size(64); + uint64_t large_size = SharedMemoryHandle::calculate_size(256); EXPECT_GT(large_size, small_size); } -TEST(SharedMemoryCalcSize, HeaderAligned) { EXPECT_EQ(sizeof(PTO2SharedMemoryHeader) % PTO2_ALIGN_SIZE, 0u); } +TEST(SharedMemoryCalcSize, HeaderAligned) { EXPECT_EQ(sizeof(SharedMemoryHeader) % PTO2_ALIGN_SIZE, 0u); } TEST(SharedMemoryCalcSize, PerRingDifferentSizes) { uint64_t ws[CHIP_MAX_RING_DEPTH] = {128, 256, 512, 1024}; - uint64_t size = PTO2SharedMemoryHandle::calculate_size_per_ring(ws); + uint64_t size = SharedMemoryHandle::calculate_size_per_ring(ws); - uint64_t uniform_size = PTO2SharedMemoryHandle::calculate_size(128); + uint64_t uniform_size = SharedMemoryHandle::calculate_size(128); EXPECT_GT(size, uniform_size); } TEST(SharedMemoryLayout, InitPerRingWritesHeaderValues) { uint64_t ws[CHIP_MAX_RING_DEPTH] = {16, 32, 64, 128}; uint64_t heaps[CHIP_MAX_RING_DEPTH] = {10 * 1024, 20 * 1024, 30 * 1024, 40 * 1024}; - const uint64_t sm_size = PTO2SharedMemoryHandle::calculate_size_per_ring(ws); + const uint64_t sm_size = SharedMemoryHandle::calculate_size_per_ring(ws); DeviceArena arena; - const size_t off_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); ASSERT_NE(arena.commit(), nullptr); - auto *handle = static_cast(arena.region_ptr(off_handle)); + auto *handle = static_cast(arena.region_ptr(off_handle)); std::memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); std::memset(buffer, 0, static_cast(sm_size)); @@ -235,7 +235,7 @@ TEST(RuntimeArenaLayout, PerRingConfigInitializesRuntimeComponents) { uint64_t ws[CHIP_MAX_RING_DEPTH] = {16, 32, 64, 128}; uint64_t heaps[CHIP_MAX_RING_DEPTH] = {10 * 1024, 20 * 1024, 30 * 1024, 40 * 1024}; int32_t dep_caps[CHIP_MAX_RING_DEPTH] = {4, 8, 16, 32}; - const uint64_t sm_size = PTO2SharedMemoryHandle::calculate_size_per_ring(ws); + const uint64_t sm_size = SharedMemoryHandle::calculate_size_per_ring(ws); uint64_t total_heap = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { total_heap += heaps[r]; @@ -292,12 +292,12 @@ TEST(RuntimeArenaLayout, RejectsOverflowingPerRingHeapSum) { // Zero window size: all ring descriptor pointers collapse to the same address. TEST(SharedMemoryBoundary, ZeroWindowSize) { - uint64_t size = PTO2SharedMemoryHandle::calculate_size(0); - uint64_t header_size = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t size = SharedMemoryHandle::calculate_size(0); + uint64_t header_size = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); EXPECT_EQ(size, header_size); DeviceArena arena; - PTO2SharedMemoryHandle *h = make_handle(arena, /*ws=*/0, /*heap=*/4096); + SharedMemoryHandle *h = make_handle(arena, /*ws=*/0, /*heap=*/4096); if (h) { for (int r = 0; r < CHIP_MAX_RING_DEPTH - 1; r++) { EXPECT_EQ(h->header->rings[r].task_descriptors, h->header->rings[r + 1].task_descriptors) @@ -308,7 +308,7 @@ TEST(SharedMemoryBoundary, ZeroWindowSize) { TEST(SharedMemoryBoundary, ValidateDetectsCorruption) { DeviceArena arena; - PTO2SharedMemoryHandle *h = make_handle(arena, /*ws=*/256, /*heap=*/4096); + SharedMemoryHandle *h = make_handle(arena, /*ws=*/256, /*heap=*/4096); ASSERT_NE(h, nullptr); EXPECT_TRUE(h->validate()); @@ -318,7 +318,7 @@ TEST(SharedMemoryBoundary, ValidateDetectsCorruption) { TEST(SharedMemoryBoundary, InitRejectsUndersizedBuffer) { // init() must refuse an SM buffer smaller than calculate_size(window_size). - PTO2SharedMemoryHandle handle{}; + SharedMemoryHandle handle{}; char buf[64]{}; EXPECT_FALSE(handle.init(buf, sizeof(buf), /*task_window_size=*/256, /*heap=*/4096)); } diff --git a/tests/ut/cpp/a2a3/test_task_state.cpp b/tests/ut/cpp/a2a3/test_task_state.cpp index 5a60677149..23caa3181b 100644 --- a/tests/ut/cpp/a2a3/test_task_state.cpp +++ b/tests/ut/cpp/a2a3/test_task_state.cpp @@ -34,7 +34,7 @@ class TaskStateTest : public ::testing::Test { protected: PTO2SchedulerState sched; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -46,7 +46,7 @@ class TaskStateTest : public ::testing::Test { int slot_payload_pool_idx_ = 0; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); auto layout = PTO2SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index 17ce61ae8b..0f89a0bc5d 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -45,7 +45,7 @@ class WiringTest : public ::testing::Test { protected: PTO2OrchestratorState orch{}; PTO2SchedulerState sched{}; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -58,7 +58,7 @@ class WiringTest : public ::testing::Test { int slot_payload_pool_idx_ = 0; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); auto layout = PTO2SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); diff --git a/tests/ut/cpp/a5/test_dep_list_pool.cpp b/tests/ut/cpp/a5/test_dep_list_pool.cpp index 4ca1a857ef..160f4be764 100644 --- a/tests/ut/cpp/a5/test_dep_list_pool.cpp +++ b/tests/ut/cpp/a5/test_dep_list_pool.cpp @@ -84,7 +84,7 @@ TEST_F(DepListPoolTest, EnsureSpaceDeadlockReturnsFalseAndLatchesError) { ASSERT_NE(pool.alloc(), nullptr); } - PTO2SharedMemoryRingHeader ring{}; + SharedMemoryRingHeader ring{}; ring.fc.init(); ring.fc.current_task_index.store(POOL_CAP + 1, std::memory_order_release); ring.fc.last_task_alive.store(0, std::memory_order_release); diff --git a/tests/ut/cpp/a5/test_fanin_pool.cpp b/tests/ut/cpp/a5/test_fanin_pool.cpp index 790814ba54..586539b36e 100644 --- a/tests/ut/cpp/a5/test_fanin_pool.cpp +++ b/tests/ut/cpp/a5/test_fanin_pool.cpp @@ -91,7 +91,7 @@ TEST_F(FaninPoolTest, EnsureSpaceDeadlockReturnsFalseAndLatchesError) { ASSERT_NE(pool.alloc(), nullptr); } - PTO2SharedMemoryRingHeader ring{}; + SharedMemoryRingHeader ring{}; ring.fc.init(); ring.fc.current_task_index.store(POOL_CAP + 1, std::memory_order_release); ring.fc.last_task_alive.store(0, std::memory_order_release); diff --git a/tests/ut/cpp/a5/test_graph_activation.cpp b/tests/ut/cpp/a5/test_graph_activation.cpp index c55180a193..6d33bed597 100644 --- a/tests/ut/cpp/a5/test_graph_activation.cpp +++ b/tests/ut/cpp/a5/test_graph_activation.cpp @@ -33,12 +33,12 @@ class GraphActivationTest : public ::testing::Test { protected: DeviceArena sm_arena; DeviceArena runtime_arena; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; PTO2SchedulerState sched{}; PTO2SchedulerLayout sched_layout{}; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); diff --git a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp index 09bbcf54fd..7039a6b3bc 100644 --- a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp @@ -46,14 +46,14 @@ class HbgSubmitPoisonTest : public ::testing::Test { protected: DeviceArena sm_arena; DeviceArena runtime_arena; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; PTO2SchedulerState sched{}; PTO2SchedulerLayout sched_layout{}; std::vector gm_heap; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); gm_heap.resize(4096); diff --git a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp index c703974977..a53a363fac 100644 --- a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp @@ -23,7 +23,7 @@ class OrchestratorFaninTest : public ::testing::Test { protected: DeviceArena sm_arena; DeviceArena runtime_arena; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; PTO2SchedulerState sched{}; PTO2OrchestratorLayout orch_layout{}; @@ -31,7 +31,7 @@ class OrchestratorFaninTest : public ::testing::Test { std::vector gm_heap; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); gm_heap.resize(4096 * CHIP_MAX_RING_DEPTH); diff --git a/tests/ut/cpp/a5/test_scheduler_state.cpp b/tests/ut/cpp/a5/test_scheduler_state.cpp index 1f5f14f48f..a8be096e6d 100644 --- a/tests/ut/cpp/a5/test_scheduler_state.cpp +++ b/tests/ut/cpp/a5/test_scheduler_state.cpp @@ -54,7 +54,7 @@ TEST(SyncStartDrainAttemptTest, LateAckCannotSatisfyNextAttemptBarrier) { class SchedulerStateTest : public ::testing::Test { protected: PTO2SchedulerState sched; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -66,7 +66,7 @@ class SchedulerStateTest : public ::testing::Test { int slot_payload_pool_idx_ = 0; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); auto layout = PTO2SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); @@ -102,8 +102,8 @@ class SchedulerStateTest : public ::testing::Test { } void init_ring_slot( - PTO2SharedMemoryRingHeader &ring, int32_t task_id, PTO2TaskState state, uint8_t ring_id, - uint32_t fanout_count = 1, uint32_t fanout_refcount = 1 + SharedMemoryRingHeader &ring, int32_t task_id, PTO2TaskState state, uint8_t ring_id, uint32_t fanout_count = 1, + uint32_t fanout_refcount = 1 ) { ChipTaskSlotState &slot = ring.get_slot_state_by_task_id(task_id); TaskPayload &payload = ring.get_payload_by_task_id(task_id); @@ -129,7 +129,7 @@ class SchedulerStateTest : public ::testing::Test { } void setup_ring_for_reclaim_race(int32_t ring_id, int32_t current_task_index, int32_t blocked_task_id) { - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ring.fc.current_task_index.store(current_task_index, std::memory_order_release); @@ -145,7 +145,7 @@ class SchedulerStateTest : public ::testing::Test { } void setup_contended_head_case(int32_t ring_id, int32_t head_task_id) { - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ring.fc.current_task_index.store(head_task_id + 2, std::memory_order_release); ring.fc.last_task_alive.store(head_task_id, std::memory_order_release); @@ -163,7 +163,7 @@ class SchedulerStateTest : public ::testing::Test { TEST_F(SchedulerStateTest, UnvalidatedWiringPublishesEveryAdvance) { constexpr int32_t ring_id = 0; - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; EXPECT_FALSE(ring_sched.publication_batching_enabled); @@ -200,7 +200,7 @@ TEST_F(SchedulerStateTest, ConsumedHeadAdvancesAfterContendedAdvanceLock) { constexpr int32_t head_task_id = 0; setup_ring_for_reclaim_race(ring_id, /*current_task_index=*/1, head_task_id); - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); @@ -246,7 +246,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadSetsPendingAndIdleDrainAdvances) constexpr int32_t head_task_id = 17; setup_contended_head_case(ring_id, head_task_id); - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); @@ -274,7 +274,7 @@ TEST_F(SchedulerStateTest, DeferredAdvanceDoesNotAcknowledgePublication) { constexpr int32_t head_task_id = 17; setup_contended_head_case(ring_id, head_task_id); - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); @@ -314,7 +314,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadIdleDrainStress) { SCOPED_TRACE(::testing::Message() << "ring_id=" << ring_id << " head_task_id=" << head_task_id); setup_contended_head_case(ring_id, head_task_id); - PTO2SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; + SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); diff --git a/tests/ut/cpp/a5/test_shared_memory.cpp b/tests/ut/cpp/a5/test_shared_memory.cpp index ba83ffbad7..3c9cb32883 100644 --- a/tests/ut/cpp/a5/test_shared_memory.cpp +++ b/tests/ut/cpp/a5/test_shared_memory.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2SharedMemory layout from shared_memory.h + * Unit tests for SharedMemory layout from shared_memory.h * * Tests creation, validation, per-ring independence, alignment, size * calculation, and error handling under the DeviceArena-backed init model: @@ -29,13 +29,13 @@ namespace { // Reserve + commit a fresh handle + sm_base on `arena` and run init. // Returns the wrapper pointer (arena-owned) or nullptr on init failure. -PTO2SharedMemoryHandle *make_handle(DeviceArena &arena, uint64_t task_window_size, uint64_t heap_size) { - const uint64_t sm_size = PTO2SharedMemoryHandle::calculate_size(task_window_size); - const size_t off_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); +SharedMemoryHandle *make_handle(DeviceArena &arena, uint64_t task_window_size, uint64_t heap_size) { + const uint64_t sm_size = SharedMemoryHandle::calculate_size(task_window_size); + const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; - auto *handle = static_cast(arena.region_ptr(off_handle)); + auto *handle = static_cast(arena.region_ptr(off_handle)); std::memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); std::memset(buffer, 0, static_cast(sm_size)); @@ -52,10 +52,10 @@ PTO2SharedMemoryHandle *make_handle(DeviceArena &arena, uint64_t task_window_siz class SharedMemoryTest : public ::testing::Test { protected: DeviceArena arena; - PTO2SharedMemoryHandle *handle = nullptr; + SharedMemoryHandle *handle = nullptr; void SetUp() override { - handle = PTO2SharedMemoryHandle::create_and_init_default(arena); + handle = SharedMemoryHandle::create_and_init_default(arena); ASSERT_NE(handle, nullptr); } @@ -165,7 +165,7 @@ TEST_F(SharedMemoryTest, HeaderAlignment) { // Descriptor and payload regions don't overlap within or across rings. TEST(SharedMemoryLayout, RegionsNonOverlapping) { DeviceArena arena; - PTO2SharedMemoryHandle *h = make_handle(arena, /*ws=*/64, /*heap=*/4096); + SharedMemoryHandle *h = make_handle(arena, /*ws=*/64, /*heap=*/4096); ASSERT_NE(h, nullptr); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -188,37 +188,37 @@ TEST(SharedMemoryLayout, RegionsNonOverlapping) { // ============================================================================= TEST(SharedMemoryCalcSize, NonZero) { - uint64_t size = PTO2SharedMemoryHandle::calculate_size(PTO2_TASK_WINDOW_SIZE); + uint64_t size = SharedMemoryHandle::calculate_size(PTO2_TASK_WINDOW_SIZE); EXPECT_GT(size, 0u); } TEST(SharedMemoryCalcSize, LargerWindowGivesLargerSize) { - uint64_t small_size = PTO2SharedMemoryHandle::calculate_size(64); - uint64_t large_size = PTO2SharedMemoryHandle::calculate_size(256); + uint64_t small_size = SharedMemoryHandle::calculate_size(64); + uint64_t large_size = SharedMemoryHandle::calculate_size(256); EXPECT_GT(large_size, small_size); } -TEST(SharedMemoryCalcSize, HeaderAligned) { EXPECT_EQ(sizeof(PTO2SharedMemoryHeader) % PTO2_ALIGN_SIZE, 0u); } +TEST(SharedMemoryCalcSize, HeaderAligned) { EXPECT_EQ(sizeof(SharedMemoryHeader) % PTO2_ALIGN_SIZE, 0u); } TEST(SharedMemoryCalcSize, PerRingDifferentSizes) { uint64_t ws[CHIP_MAX_RING_DEPTH] = {128, 256, 512, 1024}; - uint64_t size = PTO2SharedMemoryHandle::calculate_size_per_ring(ws); + uint64_t size = SharedMemoryHandle::calculate_size_per_ring(ws); - uint64_t uniform_size = PTO2SharedMemoryHandle::calculate_size(128); + uint64_t uniform_size = SharedMemoryHandle::calculate_size(128); EXPECT_GT(size, uniform_size); } TEST(SharedMemoryLayout, InitPerRingWritesHeaderValues) { uint64_t ws[CHIP_MAX_RING_DEPTH] = {16, 32, 64, 128}; uint64_t heaps[CHIP_MAX_RING_DEPTH] = {10 * 1024, 20 * 1024, 30 * 1024, 40 * 1024}; - const uint64_t sm_size = PTO2SharedMemoryHandle::calculate_size_per_ring(ws); + const uint64_t sm_size = SharedMemoryHandle::calculate_size_per_ring(ws); DeviceArena arena; - const size_t off_handle = arena.reserve(sizeof(PTO2SharedMemoryHandle), alignof(PTO2SharedMemoryHandle)); + const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); ASSERT_NE(arena.commit(), nullptr); - auto *handle = static_cast(arena.region_ptr(off_handle)); + auto *handle = static_cast(arena.region_ptr(off_handle)); std::memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); std::memset(buffer, 0, static_cast(sm_size)); @@ -235,7 +235,7 @@ TEST(RuntimeArenaLayout, PerRingConfigInitializesRuntimeComponents) { uint64_t ws[CHIP_MAX_RING_DEPTH] = {16, 32, 64, 128}; uint64_t heaps[CHIP_MAX_RING_DEPTH] = {10 * 1024, 20 * 1024, 30 * 1024, 40 * 1024}; int32_t dep_caps[CHIP_MAX_RING_DEPTH] = {4, 8, 16, 32}; - const uint64_t sm_size = PTO2SharedMemoryHandle::calculate_size_per_ring(ws); + const uint64_t sm_size = SharedMemoryHandle::calculate_size_per_ring(ws); uint64_t total_heap = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { total_heap += heaps[r]; @@ -283,7 +283,7 @@ TEST(RuntimeArenaLayout, RewiresReclaimPublicationPointersAfterRelocation) { uint64_t ws[CHIP_MAX_RING_DEPTH] = {16, 32, 64, 128}; uint64_t heaps[CHIP_MAX_RING_DEPTH] = {10 * 1024, 20 * 1024, 30 * 1024, 40 * 1024}; int32_t dep_caps[CHIP_MAX_RING_DEPTH] = {4, 8, 16, 32}; - const uint64_t sm_size = PTO2SharedMemoryHandle::calculate_size_per_ring(ws); + const uint64_t sm_size = SharedMemoryHandle::calculate_size_per_ring(ws); DeviceArena source_arena; RuntimeArenaLayout layout = runtime_reserve_layout(source_arena, ws, heaps, dep_caps); @@ -339,12 +339,12 @@ TEST(RuntimeArenaLayout, RejectsOverflowingPerRingHeapSum) { // Zero window size: all ring descriptor pointers collapse to the same address. TEST(SharedMemoryBoundary, ZeroWindowSize) { - uint64_t size = PTO2SharedMemoryHandle::calculate_size(0); - uint64_t header_size = PTO2_ALIGN_UP(sizeof(PTO2SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t size = SharedMemoryHandle::calculate_size(0); + uint64_t header_size = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); EXPECT_EQ(size, header_size); DeviceArena arena; - PTO2SharedMemoryHandle *h = make_handle(arena, /*ws=*/0, /*heap=*/4096); + SharedMemoryHandle *h = make_handle(arena, /*ws=*/0, /*heap=*/4096); if (h) { for (int r = 0; r < CHIP_MAX_RING_DEPTH - 1; r++) { EXPECT_EQ(h->header->rings[r].task_descriptors, h->header->rings[r + 1].task_descriptors) @@ -355,7 +355,7 @@ TEST(SharedMemoryBoundary, ZeroWindowSize) { TEST(SharedMemoryBoundary, ValidateDetectsCorruption) { DeviceArena arena; - PTO2SharedMemoryHandle *h = make_handle(arena, /*ws=*/256, /*heap=*/4096); + SharedMemoryHandle *h = make_handle(arena, /*ws=*/256, /*heap=*/4096); ASSERT_NE(h, nullptr); EXPECT_TRUE(h->validate()); @@ -365,7 +365,7 @@ TEST(SharedMemoryBoundary, ValidateDetectsCorruption) { TEST(SharedMemoryBoundary, InitRejectsUndersizedBuffer) { // init() must refuse an SM buffer smaller than calculate_size(window_size). - PTO2SharedMemoryHandle handle{}; + SharedMemoryHandle handle{}; char buf[64]{}; EXPECT_FALSE(handle.init(buf, sizeof(buf), /*task_window_size=*/256, /*heap=*/4096)); } diff --git a/tests/ut/cpp/a5/test_task_state.cpp b/tests/ut/cpp/a5/test_task_state.cpp index 5a60677149..23caa3181b 100644 --- a/tests/ut/cpp/a5/test_task_state.cpp +++ b/tests/ut/cpp/a5/test_task_state.cpp @@ -34,7 +34,7 @@ class TaskStateTest : public ::testing::Test { protected: PTO2SchedulerState sched; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -46,7 +46,7 @@ class TaskStateTest : public ::testing::Test { int slot_payload_pool_idx_ = 0; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); auto layout = PTO2SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index c3a3cab4bc..3299b24a46 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -41,7 +41,7 @@ class WiringTest : public ::testing::Test { protected: PTO2OrchestratorState orch{}; PTO2SchedulerState sched{}; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -54,7 +54,7 @@ class WiringTest : public ::testing::Test { int slot_payload_pool_idx_ = 0; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); auto layout = PTO2SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); @@ -750,7 +750,7 @@ TEST_F(WiringTest, TaskWindowOpensOnceWithheldProgressIsPublished) { ASSERT_EQ(ring->fc.last_task_alive.load(), 0); PTO2TaskAllocator allocator; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_handle->sm_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_handle->sm_base); allocator.init( ring->task_descriptors, 4, &ring->fc.current_task_index, &ring->fc.last_task_alive, heap, sizeof(heap), orch_err, ring->slot_states, 3, 0 @@ -780,7 +780,7 @@ TEST_F(WiringTest, HeapReclaimsOnceWithheldProgressIsPublished) { alignas(64) char heap[64] = {}; PTO2TaskAllocator allocator; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_handle->sm_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_handle->sm_base); allocator.init( ring->task_descriptors, 8, &ring->fc.current_task_index, &ring->fc.last_task_alive, heap, sizeof(heap), orch_err, ring->slot_states, 0, 0 @@ -859,7 +859,7 @@ TEST_F(WiringTest, FaninPoolReclaimsOnceWithheldProgressIsPublished) { PTO2FaninSpillEntry entries[4]{}; PTO2FaninPool pool{}; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_handle->sm_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_handle->sm_base); pool.init(entries, 4, orch_err); pool.set_reclaim_publication_request(&sched.publication_request_mask, &sched.publication_ack_mask, 0); for (int i = 0; i < 4; i++) { @@ -889,13 +889,13 @@ TEST_F(WiringTest, RingReuseResetsPublicationShadow) { rss.last_task_alive = 17; rss.last_published_to_sm = 16; - auto *orch_err = pto2_sm_layout::orch_error_code_addr(sm_handle->sm_base); + auto *orch_err = sm_layout::orch_error_code_addr(sm_handle->sm_base); rss.reset_for_reuse(sm_handle->sm_base, 0, orch_err); EXPECT_EQ(rss.last_task_alive, 0); EXPECT_EQ(rss.last_published_to_sm, 0); EXPECT_FALSE(rss.publication_batching_enabled); - EXPECT_EQ(rss.ring, pto2_sm_layout::ring_header_addr(sm_handle->sm_base, 0)); + EXPECT_EQ(rss.ring, sm_layout::ring_header_addr(sm_handle->sm_base, 0)); } TEST_F(WiringTest, AdvanceRingPointersStopsAtNonConsumed) { diff --git a/tests/ut/cpp/common/test_hbg_graph_cache.cpp b/tests/ut/cpp/common/test_hbg_graph_cache.cpp index 0d3c1d1e90..59758e6bda 100644 --- a/tests/ut/cpp/common/test_hbg_graph_cache.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_cache.cpp @@ -635,7 +635,7 @@ TEST(GraphExecutionErrors, ReadyQueueOverflowHasTriageText) { } TEST(GraphExecutionErrors, GraphReadyQueueOverflowIsReported) { - PTO2SharedMemoryHeader header{}; + SharedMemoryHeader header{}; PTO2SchedulerState scheduler{}; scheduler.sm_header = &header; PTO2ReadyQueueSlot queue_slots[2]{}; @@ -659,7 +659,7 @@ TEST(GraphExecutionErrors, GraphReadyQueueOverflowIsReported) { } TEST(GraphExecutionErrors, GraphPrepareQueueOverflowIsReported) { - PTO2SharedMemoryHeader header{}; + SharedMemoryHeader header{}; PTO2SchedulerState scheduler{}; scheduler.sm_header = &header; PTO2ReadyQueueSlot queue_slots[2]{}; diff --git a/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp b/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp index 7083d137de..909d3a3973 100644 --- a/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp @@ -41,7 +41,7 @@ class HbgGraphDefinitionArenaTest : public ::testing::Test { protected: DeviceArena sm_arena; DeviceArena runtime_arena; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; PTO2SchedulerState sched{}; PTO2SchedulerLayout sched_layout{}; @@ -53,7 +53,7 @@ class HbgGraphDefinitionArenaTest : public ::testing::Test { static constexpr size_t HEAP_BYTES = 64 * 1024; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); gm_heap.resize(HEAP_BYTES); diff --git a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp index 2d6b1d1dff..328ca26043 100644 --- a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp @@ -33,7 +33,7 @@ class HbgGraphSubmitFailureTest : public ::testing::Test { protected: DeviceArena sm_arena; DeviceArena runtime_arena; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; PTO2SchedulerState sched{}; PTO2SchedulerLayout sched_layout{}; @@ -62,7 +62,7 @@ class HbgGraphSubmitFailureTest : public ::testing::Test { } void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); gm_heap.resize(HEAP_BYTES); @@ -741,7 +741,7 @@ TEST_F(HbgGraphSubmitFailureTest, AnOrdinaryAllocationInterleavesWithADeferredSh EXPECT_FALSE(orch.fatal); EXPECT_GT(orch.task_allocator.heap_top(), heap_after_ordinary) << "the shell's block sits above the ordinary task's, not before it"; - PTO2SharedMemoryRingHeader &ring = sm_handle->header->ring; + SharedMemoryRingHeader &ring = sm_handle->header->ring; const int32_t shell_slot = ring.get_slot_by_task_id(static_cast(graph.task_id.local())); const TaskDescriptor *shell = ring.slot_states[shell_slot].task.get(); ASSERT_NE(shell, nullptr); diff --git a/tests/ut/cpp/common/test_hbg_slot_claim.cpp b/tests/ut/cpp/common/test_hbg_slot_claim.cpp index 7a98ccbf2a..e2d19d5380 100644 --- a/tests/ut/cpp/common/test_hbg_slot_claim.cpp +++ b/tests/ut/cpp/common/test_hbg_slot_claim.cpp @@ -32,7 +32,7 @@ class HbgSlotClaimTest : public ::testing::Test { protected: DeviceArena sm_arena; DeviceArena runtime_arena; - PTO2SharedMemoryHandle *sm_handle = nullptr; + SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; PTO2SchedulerState sched{}; PTO2SchedulerLayout sched_layout{}; @@ -46,7 +46,7 @@ class HbgSlotClaimTest : public ::testing::Test { static constexpr size_t STAGING_BYTES = 256 * 1024; void SetUp() override { - sm_handle = PTO2SharedMemoryHandle::create_and_init_default(sm_arena); + sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); gm_heap.resize(HEAP_BYTES); diff --git a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp index baea7c9631..cfb4d389ea 100644 --- a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp +++ b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp @@ -69,9 +69,9 @@ class AlignedImage { class Mirror { public: Mirror() : - image_(pto2_sm_layout::ring_segment_offsets(WINDOW).end) { - const auto off = pto2_sm_layout::ring_segment_offsets(WINDOW); - auto *header = reinterpret_cast(image_.base()); + image_(sm_layout::ring_segment_offsets(WINDOW).end) { + const auto off = sm_layout::ring_segment_offsets(WINDOW); + auto *header = reinterpret_cast(image_.base()); auto &ring = header->ring; ring.task_window_size = WINDOW; ring.task_window_mask = static_cast(WINDOW - 1); @@ -134,30 +134,28 @@ class Mirror { const char *base() const { return image_.base(); } TaskDescriptor *descriptors() { - return reinterpret_cast( - image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).descriptors - ); + return reinterpret_cast(image_.base() + sm_layout::ring_segment_offsets(WINDOW).descriptors); } TaskPayload *payloads() { - return reinterpret_cast(image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).payloads); + return reinterpret_cast(image_.base() + sm_layout::ring_segment_offsets(WINDOW).payloads); } ChipTensor *tensor_pool() { - return reinterpret_cast(image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).tensor_pool); + return reinterpret_cast(image_.base() + sm_layout::ring_segment_offsets(WINDOW).tensor_pool); } uint64_t *scalar_pool() { - return reinterpret_cast(image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).scalar_pool); + return reinterpret_cast(image_.base() + sm_layout::ring_segment_offsets(WINDOW).scalar_pool); } int32_t *fanin_pool() { - return reinterpret_cast(image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).fanin_pool); + return reinterpret_cast(image_.base() + sm_layout::ring_segment_offsets(WINDOW).fanin_pool); } ChipTaskSlotState *slot_states() { return reinterpret_cast( - image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).slot_states + image_.base() + sm_layout::ring_segment_offsets(WINDOW).slot_states ); } std::atomic *completion_flags() { return reinterpret_cast *>( - image_.base() + pto2_sm_layout::ring_segment_offsets(WINDOW).completion_flags + image_.base() + sm_layout::ring_segment_offsets(WINDOW).completion_flags ); } @@ -167,8 +165,8 @@ class Mirror { // What a bind of `submitted` tasks put in the pools, given the per-task shape the // Mirror writes. This is the same arithmetic the orchestrator's cursors perform. -inline pto2_sm_layout::BindUsage usage_for(uint64_t submitted) { - return pto2_sm_layout::BindUsage{ +inline sm_layout::BindUsage usage_for(uint64_t submitted) { + return sm_layout::BindUsage{ submitted, submitted * static_cast(FANIN_STRIDE), submitted * static_cast(TENSORS_PER_TASK), @@ -178,24 +176,24 @@ inline pto2_sm_layout::BindUsage usage_for(uint64_t submitted) { // A bind that allocated nothing out of the graph heap, so the restack has no // address to move. Every test that is not about the rebase passes this. -inline constexpr pto2_sm_layout::HeapRebase kNoHeapAddresses{HEAP_VIRTUAL_BASE, 0}; +inline constexpr sm_layout::HeapRebase kNoHeapAddresses{HEAP_VIRTUAL_BASE, 0}; struct Compacted { AlignedImage image; uint64_t bytes; - pto2_sm_layout::BindUsage used; + sm_layout::BindUsage used; explicit Compacted( - Mirror &mirror, uint64_t submitted = SUBMITTED, const pto2_sm_layout::HeapRebase &rebase = kNoHeapAddresses + Mirror &mirror, uint64_t submitted = SUBMITTED, const sm_layout::HeapRebase &rebase = kNoHeapAddresses ) : - image(pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents(usage_for(submitted))).end, 0xAA), + image(sm_layout::ring_segment_offsets(sm_layout::image_extents(usage_for(submitted))).end, 0xAA), bytes(0), used(usage_for(submitted)) { - bytes = pto2_sm_layout::compact_live_image(mirror.base(), WINDOW, used, rebase, image.base()); + bytes = sm_layout::compact_live_image(mirror.base(), WINDOW, used, rebase, image.base()); } - pto2_sm_layout::PTO2RingSegmentOffsets off(uint64_t submitted = SUBMITTED) const { - return pto2_sm_layout::ring_segment_offsets(pto2_sm_layout::image_extents(usage_for(submitted))); + sm_layout::PTO2RingSegmentOffsets off(uint64_t submitted = SUBMITTED) const { + return sm_layout::ring_segment_offsets(sm_layout::image_extents(usage_for(submitted))); } TaskPayload *payload_at(uint64_t i) { return reinterpret_cast(image.base() + off().payloads) + i; } @@ -217,7 +215,7 @@ TEST(HbgSmCompaction, ShipsOnlyTheLivePrefix) { Compacted compacted(mirror); EXPECT_EQ(compacted.bytes, compacted.off().end); - EXPECT_LT(compacted.bytes, pto2_sm_layout::ring_segment_offsets(WINDOW).end); + EXPECT_LT(compacted.bytes, sm_layout::ring_segment_offsets(WINDOW).end); } TEST(HbgSmCompaction, CarriesEveryLiveSlotsContent) { @@ -235,7 +233,7 @@ TEST(HbgSmCompaction, CarriesEveryLiveSlotsContent) { } // The header's pitch-independent fields come across; the mirror slot past the // prefix does not. - auto &ring = reinterpret_cast(compacted.image.base())->ring; + auto &ring = reinterpret_cast(compacted.image.base())->ring; EXPECT_EQ(ring.task_window_size, WINDOW); EXPECT_EQ(ring.fc.current_task_index.load(std::memory_order_relaxed), static_cast(SUBMITTED)); } @@ -282,7 +280,7 @@ TEST(HbgSmCompaction, LeavesNoHostPointerInTheHeader) { Mirror mirror; Compacted compacted(mirror); - auto &ring = reinterpret_cast(compacted.image.base())->ring; + auto &ring = reinterpret_cast(compacted.image.base())->ring; EXPECT_EQ(ring.task_descriptors, nullptr); EXPECT_EQ(ring.task_payloads, nullptr); EXPECT_EQ(ring.slot_states, nullptr); @@ -297,8 +295,8 @@ TEST(HbgSmCompaction, ZeroSubmittedShipsTheHeaderAlone) { Compacted compacted(mirror, /*submitted=*/0); EXPECT_EQ(compacted.bytes, compacted.off(0).end); - EXPECT_LT(compacted.bytes, pto2_sm_layout::ring_segment_offsets(1).end); - auto &ring = reinterpret_cast(compacted.image.base())->ring; + EXPECT_LT(compacted.bytes, sm_layout::ring_segment_offsets(1).end); + auto &ring = reinterpret_cast(compacted.image.base())->ring; EXPECT_EQ(ring.task_window_size, WINDOW); EXPECT_EQ(ring.task_descriptors, nullptr); } @@ -312,7 +310,7 @@ TEST(HbgSmCompaction, PackedPoolsShipFewerBytesAndStillResolve) { // The mirror's pools cover WINDOW tasks at their full caps; the image's cover // SUBMITTED tasks at the shape they actually used. - EXPECT_LT(compacted.bytes, pto2_sm_layout::ring_segment_offsets(WINDOW).end); + EXPECT_LT(compacted.bytes, sm_layout::ring_segment_offsets(WINDOW).end); for (uint64_t i = 0; i < SUBMITTED; ++i) { TaskPayload *shipped = compacted.payload_at(i); @@ -406,16 +404,16 @@ TEST(HbgSmCompaction, LayoutStaysWithinDeltaReach) { EXPECT_LT(compacted.bytes, static_cast(INT32_MAX)); constexpr uint64_t REACH = static_cast(INT32_MAX); - EXPECT_LE(pto2_sm_layout::ring_segment_offsets(PTO2_TASK_WINDOW_SIZE).end, REACH); + EXPECT_LE(sm_layout::ring_segment_offsets(PTO2_TASK_WINDOW_SIZE).end, REACH); // The bound is a property of the layout, not of the capacity alone: there is a // capacity past which the mirror no longer fits, and it is above the default. uint64_t window = PTO2_TASK_WINDOW_SIZE; - while (window <= (UINT64_MAX / 2) && pto2_sm_layout::ring_segment_offsets(window).end <= REACH) { + while (window <= (UINT64_MAX / 2) && sm_layout::ring_segment_offsets(window).end <= REACH) { window *= 2; } EXPECT_GT(window, static_cast(PTO2_TASK_WINDOW_SIZE)); - EXPECT_GT(pto2_sm_layout::ring_segment_offsets(window).end, REACH); + EXPECT_GT(sm_layout::ring_segment_offsets(window).end, REACH); } // The graph heap's device region is committed after orchestration, so the diff --git a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp index 5d99a298fc..ae987ff199 100644 --- a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp +++ b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp @@ -45,7 +45,7 @@ TEST(ScopeDeadlockDetectionTest, DepPoolUsesTimeoutForDifferentScopeHead) { alignas(64) ChipTaskSlotState slot_states[WINDOW_SIZE]{}; TaskDescriptor task_descriptors[WINDOW_SIZE]{}; - PTO2SharedMemoryRingHeader ring{}; + SharedMemoryRingHeader ring{}; ring.fc.init(); ring.task_window_size = WINDOW_SIZE; ring.task_window_mask = WINDOW_SIZE - 1; @@ -77,7 +77,7 @@ TEST(ScopeDeadlockDetectionTest, DepPoolRejectsCurrentScopeHeadStructurally) { alignas(64) ChipTaskSlotState slot_states[WINDOW_SIZE]{}; TaskDescriptor task_descriptors[WINDOW_SIZE]{}; - PTO2SharedMemoryRingHeader ring{}; + SharedMemoryRingHeader ring{}; ring.fc.init(); ring.task_window_size = WINDOW_SIZE; ring.task_window_mask = WINDOW_SIZE - 1; diff --git a/tests/ut/cpp/common/test_trb_runtime_temp_buffer.cpp b/tests/ut/cpp/common/test_trb_runtime_temp_buffer.cpp index 0f6ce790a9..e7af56e0e3 100644 --- a/tests/ut/cpp/common/test_trb_runtime_temp_buffer.cpp +++ b/tests/ut/cpp/common/test_trb_runtime_temp_buffer.cpp @@ -298,7 +298,7 @@ TEST_F(TrbRuntimeTempBufferTest, FailedExecutionCopiesRuntimeStatus) { ArgDirection signature[1] = {ArgDirection::OUT}; ASSERT_EQ(bind_runtime(runtime, api_, args, signature, 1), 0); - auto *header = static_cast(runtime.get_gm_sm_ptr()); + auto *header = static_cast(runtime.get_gm_sm_ptr()); ASSERT_NE(header, nullptr); header->orch_error_code.store(SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, std::memory_order_relaxed); From 9daa24def79c98a1abfc3c2a4a6c8a02fa52d029 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:18:00 -0700 Subject: [PATCH 04/22] Refactor: retire the PTO2 prefix on the scheduler-state structs `PTO2SchedulerState` -> `SchedulerState`, `PTO2SchedulerLayout` -> `SchedulerLayout`, `PTO2SchedProfilingData` -> `SchedProfilingData`, and the two `PTO2_SCHED_CYCLE_*` markers lose the prefix with them. All plain strips: no declaration in the tree, in the CANN headers or in pto-isa holds any of these names. 397 occurrences across 71 files. Co-Authored-By: Claude Opus 5 (1M context) --- .../host_build_graph/docs/RUNTIME_LOGIC.md | 6 +- .../host_build_graph/runtime/async_wait.h | 6 +- .../host_build_graph/runtime/orchestrator.h | 7 +- .../orchestrator_core/orchestrator.cpp | 2 +- .../host_build_graph/runtime/runtime_core.h | 4 +- .../runtime/scheduler/scheduler.cpp | 10 +-- .../runtime/scheduler/scheduler.h | 34 +++++----- .../runtime/scheduler/scheduler_cold_path.cpp | 2 +- .../scheduler/scheduler_completion.cpp | 4 +- .../runtime/scheduler/scheduler_context.h | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 23 ++++--- .../runtime/scheduler/scheduler_types.h | 2 +- .../runtime/shared/runtime_init.cpp | 38 ++++++----- .../host_build_graph/runtime/submit_types.h | 2 +- .../docs/MULTI_RING.md | 2 +- .../runtime/async_wait.h | 6 +- .../runtime/orchestrator.cpp | 8 +-- .../runtime/orchestrator.h | 6 +- .../runtime/runtime_core.h | 4 +- .../runtime/scheduler/scheduler.cpp | 12 ++-- .../runtime/scheduler/scheduler.h | 45 +++++++------ .../runtime/scheduler/scheduler_cold_path.cpp | 2 +- .../scheduler/scheduler_completion.cpp | 4 +- .../runtime/scheduler/scheduler_context.h | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 14 ++-- .../runtime/shared/runtime_init.cpp | 38 ++++++----- .../runtime/submit_types.h | 2 +- .../host_build_graph/docs/RUNTIME_LOGIC.md | 6 +- .../host_build_graph/runtime/async_wait.h | 6 +- .../host_build_graph/runtime/orchestrator.h | 7 +- .../orchestrator_core/orchestrator.cpp | 2 +- .../host_build_graph/runtime/runtime_core.h | 4 +- .../runtime/scheduler/scheduler.cpp | 10 +-- .../runtime/scheduler/scheduler.h | 34 +++++----- .../runtime/scheduler/scheduler_cold_path.cpp | 2 +- .../scheduler/scheduler_completion.cpp | 4 +- .../runtime/scheduler/scheduler_context.h | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 23 ++++--- .../runtime/scheduler/scheduler_types.h | 2 +- .../runtime/shared/runtime_init.cpp | 38 ++++++----- .../host_build_graph/runtime/submit_types.h | 2 +- .../docs/MULTI_RING.md | 2 +- .../runtime/async_wait.h | 6 +- .../runtime/orchestrator.cpp | 10 +-- .../runtime/orchestrator.h | 6 +- .../runtime/runtime_core.h | 4 +- .../runtime/scheduler/scheduler.cpp | 12 ++-- .../runtime/scheduler/scheduler.h | 43 ++++++------- .../runtime/scheduler/scheduler_cold_path.cpp | 2 +- .../scheduler/scheduler_completion.cpp | 4 +- .../runtime/scheduler/scheduler_context.h | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 14 ++-- .../runtime/shared/runtime_init.cpp | 38 ++++++----- .../runtime/submit_types.h | 2 +- tests/ut/cpp/CMakeLists.txt | 2 +- tests/ut/cpp/a2a3/test_graph_activation.cpp | 6 +- tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp | 6 +- tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp | 6 +- tests/ut/cpp/a2a3/test_scheduler_state.cpp | 20 +++--- tests/ut/cpp/a2a3/test_task_state.cpp | 6 +- tests/ut/cpp/a2a3/test_wiring.cpp | 64 +++++++++---------- tests/ut/cpp/a5/test_graph_activation.cpp | 6 +- tests/ut/cpp/a5/test_hbg_submit_poison.cpp | 6 +- tests/ut/cpp/a5/test_orchestrator_fanin.cpp | 6 +- tests/ut/cpp/a5/test_scheduler_state.cpp | 28 ++++---- tests/ut/cpp/a5/test_task_state.cpp | 6 +- tests/ut/cpp/a5/test_wiring.cpp | 16 ++--- tests/ut/cpp/common/test_hbg_graph_cache.cpp | 14 ++-- .../test_hbg_graph_definition_arena.cpp | 6 +- .../common/test_hbg_graph_submit_failure.cpp | 6 +- tests/ut/cpp/common/test_hbg_slot_claim.cpp | 6 +- tests/ut/cpp/stubs/test_stubs.cpp | 2 +- 72 files changed, 386 insertions(+), 400 deletions(-) diff --git a/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md index c8ebfd16d3..724a6763c6 100644 --- a/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md @@ -102,7 +102,7 @@ reserves them in this order: | Zone | Regions | Copied | Allocated on device | Written by | | ---- | ------- | ------ | ------------------- | ---------- | -| device-only | `sm_handle`, the completion mailbox, `PTO2SchedulerState` and its thirteen queue slot arrays | never | yes | AICPU at boot | +| device-only | `sm_handle`, the completion mailbox, `SchedulerState` and its thirteen queue slot arrays | never | yes | AICPU at boot | | copied | `[off_copied_begin, off_copied_end)`: the runtime header | whole zone, one copy | yes | host | The copied zone comes last, so `bind` is a single contiguous `copy_to_device` @@ -122,7 +122,7 @@ putting an owning member back inside it. The one orchestrator value the device-s scheduler reads, the count of tasks completed inline during orchestration, is a scalar `rt_orchestration_done` publishes into the runtime header. -**Why the scheduler state is device-written.** `PTO2SchedulerState` holds no +**Why the scheduler state is device-written.** `SchedulerState` holds no per-run content: `sm_header` and the ring pointer derive from a pooled SM base, queue capacities are compile-time constants, hbg never advances `last_task_alive`, and it has no host-side entry point at all. So the host would @@ -137,7 +137,7 @@ when that slot's `sequence` already equals `pos`, so an empty queue is a match and every later position reads a lower sequence, which is the full-queue signal, so such a queue accepts one push and then reports full. The ramp is mandatory but it is a function of `capacity` alone, so -`PTO2SchedulerState::seed_queue_slots()` writes it on the device rather than `bind` +`SchedulerState::seed_queue_slots()` writes it on the device rather than `bind` shipping 1,775,616 bytes of it. The ready queues are still *not* bounded to `total_tasks`: graph execution expands a GRAPH task into on-device nodes that push past the host task count, so every slot must carry a valid sequence. diff --git a/src/a2a3/runtime/host_build_graph/runtime/async_wait.h b/src/a2a3/runtime/host_build_graph/runtime/async_wait.h index 2fe5e335f6..db86ba05d1 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/async_wait.h +++ b/src/a2a3/runtime/host_build_graph/runtime/async_wait.h @@ -22,7 +22,7 @@ #include "completion_token.h" #include "runtime_types.h" -struct PTO2SchedulerState; +struct SchedulerState; struct CompletionStats; inline constexpr int32_t MAX_ASYNC_WAITS = 64; @@ -172,7 +172,7 @@ struct AsyncWaitList { // NotDeferred tasks inline (without storing a transient entry in // entries[]). struct DrainCompletionSink { - PTO2SchedulerState *sched{nullptr}; + SchedulerState *sched{nullptr}; int32_t inline_completed{0}; int32_t inline_resolved{0}; int32_t error_code{SIMPLER_ERROR_NONE}; @@ -293,7 +293,7 @@ struct AsyncWaitList { template AsyncPollResult poll_and_complete( - AICoreCompletionMailbox *aicore_mailbox, PTO2SchedulerState *sched + AICoreCompletionMailbox *aicore_mailbox, SchedulerState *sched #if SIMPLER_SCHED_PROFILING , int thread_idx diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h index 48ff9fe880..99827fde70 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h @@ -80,7 +80,7 @@ struct PTO2OrchestratorState { // === SCHEDULER REFERENCE === // Note: In simulated mode, orchestrator and scheduler share address space // In real mode, they communicate via shared memory only - PTO2SchedulerState *scheduler; // For simulated mode only + SchedulerState *scheduler; // For simulated mode only // Total core counts set once at executor init; used for submit-time deadlock detection. int32_t total_cluster_count{0}; // AIC cores = MIX clusters @@ -139,10 +139,9 @@ struct PTO2OrchestratorState { // // Returns false when an allocation fails; the caller then has no hazard map // and must not orchestrate. - bool - init(void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, PTO2SchedulerState *scheduler); + bool init(void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, SchedulerState *scheduler); - void set_scheduler(PTO2SchedulerState *scheduler); + void set_scheduler(SchedulerState *scheduler); void report_fatal(int32_t error_code, const char *func, const char *fmt, ...); void begin_scope(PTO2ScopeMode mode = PTO2ScopeMode::AUTO); void end_scope(); diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 55e08f0154..e229f0cd41 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1460,7 +1460,7 @@ static TaskOutputTensors submit_task_common( if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; } - PTO2SchedulerState *sched = orch->scheduler; + SchedulerState *sched = orch->scheduler; TaskId task_id = prepared.task_id; TaskDescriptor &task = *prepared.task; TaskPayload &payload = *prepared.payload; diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h index 77fe42c062..e719253d75 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h @@ -118,7 +118,7 @@ struct RuntimeOps { */ struct RuntimeArenaLayout { size_t off_sm_handle{0}; - PTO2SchedulerLayout sched; + SchedulerLayout sched; size_t off_scheduler{0}; size_t off_runtime{0}; size_t off_mailbox{0}; @@ -178,7 +178,7 @@ struct RuntimeContext { PTO2OrchestratorState *orchestrator; // Device-only zone: the scheduler state holds no per-run content, so it is // addressed through the arena rather than carried inside this header. - PTO2SchedulerState *scheduler; + SchedulerState *scheduler; AICoreCompletionMailbox *aicore_mailbox; // Mode diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.cpp index 6458fcb747..b54d7bd7b0 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.cpp @@ -42,8 +42,8 @@ uint64_t g_sched_self_atomic_count[PLATFORM_MAX_AICPU_THREADS] = {}; uint64_t g_sched_pop_atomic_count[PLATFORM_MAX_AICPU_THREADS] = {}; uint64_t g_sched_complete_count[PLATFORM_MAX_AICPU_THREADS] = {}; -PTO2SchedProfilingData scheduler_get_profiling(int thread_idx) { - PTO2SchedProfilingData d; +SchedProfilingData scheduler_get_profiling(int thread_idx) { + SchedProfilingData d; d.lock_cycle = std::exchange(g_sched_lock_cycle[thread_idx], 0); d.fanout_cycle = std::exchange(g_sched_fanout_cycle[thread_idx], 0); d.fanin_cycle = std::exchange(g_sched_fanin_cycle[thread_idx], 0); @@ -65,7 +65,7 @@ PTO2SchedProfilingData scheduler_get_profiling(int thread_idx) { // Debug Utilities // ============================================================================= -void PTO2SchedulerState::print_stats() { +void SchedulerState::print_stats() { LOG_DEBUG("=== Scheduler Statistics ==="); #if SIMPLER_SCHED_PROFILING LOG_DEBUG("tasks_completed: %lld", (long long)tasks_completed.load(std::memory_order_relaxed)); @@ -74,8 +74,8 @@ void PTO2SchedulerState::print_stats() { LOG_DEBUG("============================"); } -void PTO2SchedulerState::print_queues() { - PTO2SchedulerState *sched = this; +void SchedulerState::print_queues() { + SchedulerState *sched = this; const char *shape_names[] = {"AIC", "AIV", "MIX"}; for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { LOG_TIMING( diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h index 8655213cc0..6b5385bab5 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -45,8 +45,8 @@ #include "aicpu/device_time.h" // get_sys_cnt_aicpu (weak; used by early-dispatch doorbell timing too) #if SIMPLER_SCHED_PROFILING -#define PTO2_SCHED_CYCLE_START() uint64_t _st0 = get_sys_cnt_aicpu(), _st1 -#define PTO2_SCHED_CYCLE_LAP(acc) \ +#define SCHED_CYCLE_START() uint64_t _st0 = get_sys_cnt_aicpu(), _st1 +#define SCHED_CYCLE_LAP(acc) \ do { \ _st1 = get_sys_cnt_aicpu(); \ acc += (_st1 - _st0); \ @@ -448,11 +448,11 @@ struct CompletionStats { }; /** - * Layout descriptor produced by PTO2SchedulerState::reserve_layout(). Holds + * Layout descriptor produced by SchedulerState::reserve_layout(). Holds * the arena offsets of every sub-region the scheduler needs plus the * capacities used at layout time (init_from_layout reuses them). */ -struct PTO2SchedulerLayout { +struct SchedulerLayout { size_t off_ready_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_ready_sync_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_dummy_ready_queue_slots; @@ -470,7 +470,7 @@ struct PTO2SchedulerLayout { * Separated from shared memory for cache efficiency. * Hot-path methods are defined inline (implicitly inline as member functions). */ -struct PTO2SchedulerState { +struct SchedulerState { // Shared memory access SharedMemoryHeader *sm_header; @@ -1166,7 +1166,7 @@ struct PTO2SchedulerState { // per-ring dep_pool entries) on the supplied arena. // Capacities are baked into the returned layout; init_data_from_layout uses // the same values. - static PTO2SchedulerLayout reserve_layout(DeviceArena &arena); + static SchedulerLayout reserve_layout(DeviceArena &arena); // Phase 3a: write everything *except* arena-internal pointer fields. // `sm_dev_base` is the device address of the SM (only stored, never @@ -1175,12 +1175,12 @@ struct PTO2SchedulerState { // task_window_size for ring task_descriptors address arithmetic; the // scheduler only needs the SM header / ring header base addresses, // both window-size-independent.) - bool init_data_from_layout(const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base); + bool init_data_from_layout(const SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base); // Phase 3b: write the arena-internal pointer fields // (ready_queues[].slots, dummy_ready_queue.slots, dep_pool.base for each // ring). Called on both host and device sides. - void wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena); + void wire_arena_pointers(const SchedulerLayout &layout, DeviceArena &arena); // Phase 3c, device only: bring every queue's slots[] to its empty ramp. The // slot arrays sit past the uploaded range, so this is the only thing that @@ -1195,12 +1195,12 @@ struct PTO2SchedulerState { void print_queues(); }; -// Scheduler cold-path API is declared as PTO2SchedulerState member functions. +// Scheduler cold-path API is declared as SchedulerState member functions. // See init()/destroy()/print_stats()/print_queues() below the struct definition. // try_inline_complete_locked: short-circuit NotDeferred completions seen during // drain so they don't grow entries[]. Defined here (not in async_wait.h) -// because PTO2SchedulerState's on_task_complete signature is only known +// because SchedulerState's on_task_complete signature is only known // after its full definition above. // // Polling: on_task_complete publishes completion + drains the wake list inline, @@ -1210,9 +1210,9 @@ AsyncWaitList::try_inline_complete_locked(AsyncWaitList::DrainCompletionSink &si // Return value (CompletionStats / consumer-walk count) discarded: // async-wait drain path has no Resolve swimlane bar attached. #if SIMPLER_SCHED_PROFILING - PTO2SchedulerState::TaskCompletionOutcome outcome = sink.sched->complete_task(slot_state, sink.thread_idx); + SchedulerState::TaskCompletionOutcome outcome = sink.sched->complete_task(slot_state, sink.thread_idx); #else - PTO2SchedulerState::TaskCompletionOutcome outcome = sink.sched->complete_task(slot_state); + SchedulerState::TaskCompletionOutcome outcome = sink.sched->complete_task(slot_state); #endif if (outcome.error_code != SIMPLER_ERROR_NONE) { sink.error_code = outcome.error_code; @@ -1225,7 +1225,7 @@ AsyncWaitList::try_inline_complete_locked(AsyncWaitList::DrainCompletionSink &si template inline AsyncPollResult AsyncWaitList::poll_and_complete( - AICoreCompletionMailbox *aicore_mailbox, PTO2SchedulerState *sched + AICoreCompletionMailbox *aicore_mailbox, SchedulerState *sched #if SIMPLER_SCHED_PROFILING , int thread_idx @@ -1281,9 +1281,9 @@ inline AsyncPollResult AsyncWaitList::poll_and_complete( // Return value (CompletionStats / consumer-walk count) discarded: // deferred-completion drain has no Resolve swimlane bar attached. #if SIMPLER_SCHED_PROFILING - PTO2SchedulerState::TaskCompletionOutcome outcome = sched->complete_task(*entry.slot_state, thread_idx); + SchedulerState::TaskCompletionOutcome outcome = sched->complete_task(*entry.slot_state, thread_idx); #else - PTO2SchedulerState::TaskCompletionOutcome outcome = sched->complete_task(*entry.slot_state); + SchedulerState::TaskCompletionOutcome outcome = sched->complete_task(*entry.slot_state); #endif if (outcome.error_code != SIMPLER_ERROR_NONE) { result.error_code = outcome.error_code; @@ -1310,7 +1310,7 @@ inline AsyncPollResult AsyncWaitList::poll_and_complete( // ============================================================================= #if SIMPLER_SCHED_PROFILING -struct PTO2SchedProfilingData { +struct SchedProfilingData { // Sub-phase cycle breakdown within on_task_complete uint64_t lock_cycle; // lock_fanout + state store + unlock uint64_t fanout_cycle; // fanout traversal @@ -1336,5 +1336,5 @@ struct PTO2SchedProfilingData { * Get and reset scheduler profiling data for a specific thread. * Returns accumulated profiling data and resets counters. */ -PTO2SchedProfilingData scheduler_get_profiling(int thread_idx); +SchedProfilingData scheduler_get_profiling(int thread_idx); #endif diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index 3070d33c4c..5e97bf3092 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -430,7 +430,7 @@ void SchedulerContext::log_chip_swimlane_summary(int32_t thread_idx, int32_t cur #if SIMPLER_SCHED_PROFILING { - PTO2SchedProfilingData sp = scheduler_get_profiling(thread_idx); + SchedProfilingData sp = scheduler_get_profiling(thread_idx); uint64_t otc_total = sp.lock_cycle + sp.fanout_cycle + sp.fanin_cycle + sp.self_consumed_cycle; uint64_t complete_poll = (chip_swimlane.sched_complete_cycle > otc_total + chip_swimlane.sched_complete_perf_cycle) ? diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index 44e1579ce9..5cfdac0007 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -677,7 +677,7 @@ void SchedulerContext::handle_drain_mode(int32_t thread_idx, [[maybe_unused]] ui // OWNER is acquired before the drain is published and persists through // completion, so every staging thread makes the same gate decision even if // producer release changes early_dispatch_state during the barrier. - bool gated = slot_state->payload != nullptr && PTO2SchedulerState::owns_early_sync_drain(*slot_state->payload); + bool gated = slot_state->payload != nullptr && SchedulerState::owns_early_sync_drain(*slot_state->payload); if (coordinator) { ResourceShape shape = slot_state->active_mask.to_shape(); @@ -764,5 +764,5 @@ void SchedulerContext::handle_drain_mode(int32_t thread_idx, [[maybe_unused]] ui } else { sched_->propagate_dispatch_fanin(*slot_state); } - PTO2SchedulerState::finish_early_sync_drain(*slot_state->payload); + SchedulerState::finish_early_sync_drain(*slot_state->payload); } diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index 5ade11d27a..b80cfd6c8a 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -188,7 +188,7 @@ class SchedulerContext { // ========================================================================= // --- Scheduler binding & per-core runtime state --- - alignas(64) PTO2SchedulerState *sched_{nullptr}; + alignas(64) SchedulerState *sched_{nullptr}; RuntimeContext *rt_{nullptr}; // Per-core execution state, indexed by core_id (= worker_id) diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 38f8b6947d..1713aae3c3 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -132,7 +132,7 @@ void SchedulerContext::build_payload( // A claimed early-stage range stays gated even if producer completion flips // the shared state before this payload is built. All other dispatches run on // pickup. - if (PTO2SchedulerState::should_gate_early_dispatch( + if (SchedulerState::should_gate_early_dispatch( force_gate, payload.early_dispatch_state.load(std::memory_order_relaxed) )) { // Gated task: hand the idle AICore the source payload (non-zero = gate) and @@ -680,12 +680,12 @@ int32_t SchedulerContext::stage_consumer_blocks( for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_cores[w] != 0) { owned[w] = - PTO2SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); + SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); } } for (int i = 0; i < n; i++) { int32_t cid = tracker.get_core_id_by_offset(handles[i].core_offset); - PTO2SchedulerState::ring_claimed_local_doorbell( + SchedulerState::ring_claimed_local_doorbell( owned[cid >> 6], cid, handles[i].reg_addr, handles[i].reg_task_id ); } @@ -825,7 +825,7 @@ int32_t SchedulerContext::try_early_dispatch( if (ChipTaskSlotState *c = sched_->early_sync_start_queue.pop_tagged(&sync_task_id_snapshot)) { bool current_sync_task = static_cast(c->task->task_id.raw) == sync_task_id_snapshot && c->task_attrs.requires_sync_start(); - if (current_sync_task && PTO2SchedulerState::try_claim_early_sync_drain(*c->payload)) { + if (current_sync_task && SchedulerState::try_claim_early_sync_drain(*c->payload)) { if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_STAGING) { sched_->cancel_early_sync_drain(*c); } else if (drain_state_.sync_start_pending.load(std::memory_order_acquire) == 0 && @@ -835,7 +835,7 @@ int32_t SchedulerContext::try_early_dispatch( // From this point onward the operation is all-or-nothing. Only this // scheduler mutates its tracker, and global drain coordinators must // wait for this scheduler's generation-tagged ack before inspecting it. - PTO2SchedulerState::mark_early_sync_drain_armed(*c->payload); + SchedulerState::mark_early_sync_drain_armed(*c->payload); always_assert(c->next_block_idx.load(std::memory_order_seq_cst) == 0); SyncStartStageResult staged = stage_sync_start_cores( c, c->logical_block_num, thread_idx, /*gated=*/true, /*record_drain_phases=*/false @@ -845,10 +845,10 @@ int32_t SchedulerContext::try_early_dispatch( static_cast(staged.running_cores), std::memory_order_seq_cst ); sched_->retry_sync_start_rendezvous_after_staging(*c); - PTO2SchedulerState::finish_early_sync_drain(*c->payload); + SchedulerState::finish_early_sync_drain(*c->payload); total_staged += staged.staged_blocks; } else if (enter_drain_mode(c, c->logical_block_num)) { - PTO2SchedulerState::mark_early_sync_drain_armed(*c->payload); + SchedulerState::mark_early_sync_drain_armed(*c->payload); } else { sched_->cancel_early_sync_drain(*c); } @@ -925,9 +925,9 @@ int32_t SchedulerContext::run_resolution_thread(Runtime *runtime, int32_t thread ChipTaskSlotState *slot; while ((slot = sp_queues_[s].pop()) != nullptr) { #if SIMPLER_SCHED_PROFILING - PTO2SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*slot, thread_idx); + SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*slot, thread_idx); #else - PTO2SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*slot); + SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*slot); #endif if (outcome.error_code != SIMPLER_ERROR_NONE) { fail_scheduler(runtime, thread_idx, outcome.error_code); @@ -971,10 +971,9 @@ int32_t SchedulerContext::run_resolution_thread(Runtime *runtime, int32_t thread while ((dummy_got = sched_->dummy_ready_queue.pop_batch(dummy_batch, DUMMY_DRAIN_BATCH)) > 0) { for (int di = 0; di < dummy_got; di++) { #if SIMPLER_SCHED_PROFILING - PTO2SchedulerState::TaskCompletionOutcome outcome = - sched_->complete_task(*dummy_batch[di], thread_idx); + SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*dummy_batch[di], thread_idx); #else - PTO2SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*dummy_batch[di]); + SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*dummy_batch[di]); #endif if (outcome.error_code != SIMPLER_ERROR_NONE) { fail_scheduler(runtime, thread_idx, outcome.error_code); diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h index 3ba97ee8a7..367ae824eb 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h @@ -19,7 +19,7 @@ #include "runtime_types.h" #include "spin_hint.h" -// host_build_graph host-orch build: RuntimeContext embeds PTO2SchedulerState by +// host_build_graph host-orch build: RuntimeContext embeds SchedulerState by // value, so this header is compiled into the host libhost_runtime.so. The AICPU // spin_hint.h that defines PLATFORM_SCHEDULER_TIMEOUT_MS is not on the host // include path; supply it here. The value only sizes an on-device scheduler diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 29b9a4195f..e98c0b1cf2 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -41,7 +41,7 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Initialize the queue header only. slots[] carries the sequence ramp push // compares against, but it lives past the uploaded range and is seeded on the -// device by PTO2SchedulerState::seed_queue_slots(), so nothing writes it here. +// device by SchedulerState::seed_queue_slots(), so nothing writes it here. void ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, uint64_t capacity) { queue->capacity = capacity; queue->mask = capacity - 1; @@ -63,7 +63,7 @@ void ready_queue_destroy(PTO2ReadyQueue *queue) { // Scheduler // ============================================================================= -bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base) { +bool SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base) { // ring stores the device address of the SM ring header — pure offset // arithmetic, no SM load. ring = sm_layout::ring_header_addr(sm_dev_base); @@ -76,10 +76,10 @@ bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base return true; } -void PTO2SchedulerState::RingSchedState::destroy() { ring = nullptr; } +void SchedulerState::RingSchedState::destroy() { ring = nullptr; } -PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena) { - PTO2SchedulerLayout layout{}; +SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena) { + SchedulerLayout layout{}; layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; // Fixed-capacity early-dispatch queues first, then the PTO2_READY_QUEUE_SIZE @@ -103,10 +103,8 @@ PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena) { return layout; } -bool PTO2SchedulerState::init_data_from_layout( - const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base -) { - PTO2SchedulerState *sched = this; +bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base) { + SchedulerState *sched = this; sched->sm_header = reinterpret_cast(sm_dev_base); #if SIMPLER_SCHED_PROFILING sched->tasks_completed.store(0, std::memory_order_relaxed); @@ -141,8 +139,8 @@ bool PTO2SchedulerState::init_data_from_layout( // wire_arena_pointers / destroy — a queue whose slots[] is never seeded accepts // one push and then reports full, since push claims a slot only when its // sequence already equals the position being claimed. -void PTO2SchedulerState::seed_queue_slots() { - PTO2SchedulerState *sched = this; +void SchedulerState::seed_queue_slots() { + SchedulerState *sched = this; for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_queues[i].seed_slots(); } @@ -158,8 +156,8 @@ void PTO2SchedulerState::seed_queue_slots() { sched->early_sync_start_queue.seed_slots(); } -void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena) { - PTO2SchedulerState *sched = this; +void SchedulerState::wire_arena_pointers(const SchedulerLayout &layout, DeviceArena &arena) { + SchedulerState *sched = this; for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_queues[i], arena, layout.off_ready_queue_slots[i]); } @@ -177,8 +175,8 @@ void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, ready_queue_wire_arena_pointers(&sched->early_sync_start_queue, arena, layout.off_early_sync_start_queue_slots); } -void PTO2SchedulerState::destroy() { - PTO2SchedulerState *sched = this; +void SchedulerState::destroy() { + SchedulerState *sched = this; sched->ring_sched_state.destroy(); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_queues[i]); @@ -200,7 +198,7 @@ void PTO2SchedulerState::destroy() { // ============================================================================= bool PTO2OrchestratorState::init( - void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, PTO2SchedulerState *scheduler_arg + void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, SchedulerState *scheduler_arg ) { auto *orch = this; *orch = PTO2OrchestratorState{}; @@ -245,7 +243,7 @@ bool PTO2OrchestratorState::init( return true; } -void PTO2OrchestratorState::set_scheduler(PTO2SchedulerState *scheduler) { this->scheduler = scheduler; } +void PTO2OrchestratorState::set_scheduler(SchedulerState *scheduler) { this->scheduler = scheduler; } // ============================================================================= // Top-level runtime arena @@ -265,8 +263,8 @@ RuntimeArenaLayout runtime_reserve_layout(DeviceArena &arena, uint64_t task_wind // begin where it ends, making the two adjacent and the upload one copy. layout.off_sm_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); layout.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); - layout.off_scheduler = arena.reserve(sizeof(PTO2SchedulerState), alignof(PTO2SchedulerState)); - layout.sched = PTO2SchedulerState::reserve_layout(arena); + layout.off_scheduler = arena.reserve(sizeof(SchedulerState), alignof(SchedulerState)); + layout.sched = SchedulerState::reserve_layout(arena); layout.off_copied_begin = arena.total_size(); // Padded to a PTO2_ALIGN_SIZE boundary: the shared-memory image starts at @@ -319,7 +317,7 @@ RuntimeContext *runtime_init_data_from_layout( void runtime_wire_arena_pointers(DeviceArena &arena, const RuntimeArenaLayout &layout, RuntimeContext *rt) { rt->sm_handle = static_cast(arena.region_ptr(layout.off_sm_handle)); rt->aicore_mailbox = static_cast(arena.region_ptr(layout.off_mailbox)); - rt->scheduler = static_cast(arena.region_ptr(layout.off_scheduler)); + rt->scheduler = static_cast(arena.region_ptr(layout.off_scheduler)); rt->scheduler->wire_arena_pointers(layout.sched, arena); } diff --git a/src/a2a3/runtime/host_build_graph/runtime/submit_types.h b/src/a2a3/runtime/host_build_graph/runtime/submit_types.h index abcebb86f1..daeabb2776 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/submit_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/submit_types.h @@ -114,7 +114,7 @@ enum class ResourceShape : uint8_t { // Number of *dispatchable* resource shapes (AIC, AIV, MIX). DUMMY does not // allocate a per-shape ready_queue entry / local buffer — it lives in a -// dedicated queue inside PTO2SchedulerState. +// dedicated queue inside SchedulerState. inline constexpr int32_t NUM_RESOURCE_SHAPES = 3; /** diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 0895420d8e..6838dca013 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -126,7 +126,7 @@ struct SharedMemoryHandle { }; ``` -### 4.5 PTO2SchedulerState (modified) +### 4.5 SchedulerState (modified) ```cpp struct RingSchedState { diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_wait.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_wait.h index 5855a97a69..c7a36e6d38 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_wait.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_wait.h @@ -23,7 +23,7 @@ #include "completion_token.h" #include "runtime_types.h" -struct PTO2SchedulerState; +struct SchedulerState; struct CompletionStats; inline constexpr int32_t MAX_ASYNC_WAITS = 64; @@ -178,7 +178,7 @@ struct AsyncWaitList { // NotDeferred tasks inline (without storing a transient entry in // entries[]). struct DrainCompletionSink { - PTO2SchedulerState *sched{nullptr}; + SchedulerState *sched{nullptr}; ChipTaskSlotState **deferred_release_slot_states{nullptr}; int32_t *deferred_release_count{nullptr}; int32_t deferred_release_capacity{0}; @@ -298,7 +298,7 @@ struct AsyncWaitList { template AsyncPollResult poll_and_complete( - AICoreCompletionMailbox *aicore_mailbox, PTO2SchedulerState *sched, + AICoreCompletionMailbox *aicore_mailbox, SchedulerState *sched, ChipTaskSlotState **deferred_release_slot_states, int32_t &deferred_release_count, int32_t deferred_release_capacity #if SIMPLER_SCHED_PROFILING diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 7d7da9adcc..cef22708a6 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -432,7 +432,7 @@ static bool all_claimed_fanin_allow_early_resolve(const PTO2FaninBuilder &fanin_ } void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state) { - PTO2SchedulerState *sched = scheduler; + SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; slot_state.dep_pool_mark = rss.dep_pool.top; #if SIMPLER_DFX @@ -443,7 +443,7 @@ void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state } void PTO2OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin) { - PTO2SchedulerState *sched = scheduler; + SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; TaskPayload *payload = slot_state.payload; slot_state.fanin_count = wfanin + 1; @@ -517,7 +517,7 @@ static ChipTaskSlotState *oldest_open_task_on_current_ring(const PTO2Orchestrato } static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, ChipTaskSlotState &slot_state, int32_t wfanin) { - PTO2SchedulerState *sched = orch->scheduler; + SchedulerState *sched = orch->scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; // dep_pool is orchestrator-exclusive (no lock). ensure_space waits for the @@ -893,7 +893,7 @@ static TaskOutputTensors submit_task_common( return result; } uint8_t ring_id = prepared.task_id.ring(); - PTO2SchedulerState *sched = orch->scheduler; + SchedulerState *sched = orch->scheduler; PTO2RingFlowControl &fc = orch->sm_header->rings[ring_id].fc; TaskId task_id = prepared.task_id; ChipTaskSlotState &cur_slot_state = *prepared.slot_state; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 4cc060dcd1..aa4a6ceb00 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -89,7 +89,7 @@ struct PTO2OrchestratorState { // === SCHEDULER STATE ACCESS === // Same runtime-arena scheduler object; Orch-side wiring mutates dep_pool // and publishes ready tasks through it before scheduler workers dispatch. - PTO2SchedulerState *scheduler; + SchedulerState *scheduler; // Total core counts set once at executor init; used for submit-time deadlock detection. int32_t total_cluster_count{0}; // AIC cores = MIX clusters @@ -169,11 +169,11 @@ struct PTO2OrchestratorState { // scope_begins, rings[].fanin_pool.base, tensor_map.{buckets,entry_pool, // free_entry_list,task_entry_heads}, scheduler reference). // Idempotent — host runs once on the image, AICPU runs once after attach. - void wire_arena_pointers(const PTO2OrchestratorLayout &layout, DeviceArena &arena, PTO2SchedulerState *scheduler); + void wire_arena_pointers(const PTO2OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler); // Forget pointers; arena owns the backing buffers. void destroy(); - void set_scheduler(PTO2SchedulerState *scheduler); + void set_scheduler(SchedulerState *scheduler); void mark_dep_pool_position(ChipTaskSlotState &slot_state); void wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin); void report_fatal(int32_t error_code, const char *func, const char *fmt, ...); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index fcf6251129..eff705d01b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -123,7 +123,7 @@ struct ArenaSizingKey { struct ArenaOffsets { size_t off_sm_handle{0}; PTO2OrchestratorLayout orch; - PTO2SchedulerLayout sched; + SchedulerLayout sched; size_t off_runtime{0}; size_t off_mailbox{0}; @@ -158,7 +158,7 @@ struct RuntimeContext { // Components SharedMemoryHandle *sm_handle; PTO2OrchestratorState orchestrator; - PTO2SchedulerState scheduler; + SchedulerState scheduler; AICoreCompletionMailbox *aicore_mailbox; // GM Heap for output buffers diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp index 23feaf064a..6c5e42adf2 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp @@ -48,8 +48,8 @@ uint64_t g_sched_self_atomic_count[PLATFORM_MAX_AICPU_THREADS] = {}; uint64_t g_sched_pop_atomic_count[PLATFORM_MAX_AICPU_THREADS] = {}; uint64_t g_sched_complete_count[PLATFORM_MAX_AICPU_THREADS] = {}; -PTO2SchedProfilingData scheduler_get_profiling(int thread_idx) { - PTO2SchedProfilingData d; +SchedProfilingData scheduler_get_profiling(int thread_idx) { + SchedProfilingData d; d.lock_cycle = std::exchange(g_sched_lock_cycle[thread_idx], 0); d.fanout_cycle = std::exchange(g_sched_fanout_cycle[thread_idx], 0); d.fanin_cycle = std::exchange(g_sched_fanin_cycle[thread_idx], 0); @@ -71,8 +71,8 @@ PTO2SchedProfilingData scheduler_get_profiling(int thread_idx) { // Debug Utilities // ============================================================================= -void PTO2SchedulerState::print_stats() { - PTO2SchedulerState *sched = this; +void SchedulerState::print_stats() { + SchedulerState *sched = this; LOG_DEBUG("=== Scheduler Statistics ==="); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { if (sched->ring_sched_states[r].last_task_alive > 0) { @@ -94,8 +94,8 @@ void PTO2SchedulerState::print_stats() { LOG_DEBUG("============================"); } -void PTO2SchedulerState::print_queues() { - PTO2SchedulerState *sched = this; +void SchedulerState::print_queues() { + SchedulerState *sched = this; LOG_DEBUG("=== Ready Queues ==="); const char *shape_names[] = {"AIC", "AIV", "MIX"}; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index b68f78e046..0a150c8b67 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -41,8 +41,8 @@ #include "aicpu/device_time.h" // get_sys_cnt_aicpu (weak; used by early-dispatch doorbell timing too) #if SIMPLER_SCHED_PROFILING -#define PTO2_SCHED_CYCLE_START() uint64_t _st0 = get_sys_cnt_aicpu(), _st1 -#define PTO2_SCHED_CYCLE_LAP(acc) \ +#define SCHED_CYCLE_START() uint64_t _st0 = get_sys_cnt_aicpu(), _st1 +#define SCHED_CYCLE_LAP(acc) \ do { \ _st1 = get_sys_cnt_aicpu(); \ acc += (_st1 - _st0); \ @@ -419,11 +419,11 @@ struct CompletionStats { }; /** - * Layout descriptor produced by PTO2SchedulerState::reserve_layout(). Holds + * Layout descriptor produced by SchedulerState::reserve_layout(). Holds * the arena offsets of every sub-region the scheduler needs plus the * capacities used at layout time (init_data_from_layout reuses them). */ -struct PTO2SchedulerLayout { +struct SchedulerLayout { size_t off_ready_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_ready_sync_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_dummy_ready_queue_slots; @@ -441,7 +441,7 @@ struct PTO2SchedulerLayout { * Separated from shared memory for cache efficiency. * Hot-path methods are defined inline (implicitly inline as member functions). */ -struct PTO2SchedulerState { +struct SchedulerState { // Shared memory access SharedMemoryHeader *sm_header; @@ -1256,7 +1256,7 @@ struct PTO2SchedulerState { extern uint64_t g_sched_lock_atomic_count[], g_sched_lock_wait_cycle[]; extern uint64_t g_sched_fanout_atomic_count[], g_sched_push_wait_cycle[]; uint64_t lock_atomics = 0, lock_wait = 0; - PTO2_SCHED_CYCLE_START(); + SCHED_CYCLE_START(); #endif #if SIMPLER_SCHED_PROFILING @@ -1272,7 +1272,7 @@ struct PTO2SchedulerState { lock_atomics += 3; // task_state.store + lifecycle_flags.fetch_or + unlock.store g_sched_lock_atomic_count[thread_idx] += lock_atomics; g_sched_lock_wait_cycle[thread_idx] += lock_wait; - PTO2_SCHED_CYCLE_LAP(g_sched_lock_cycle[thread_idx]); + SCHED_CYCLE_LAP(g_sched_lock_cycle[thread_idx]); #endif // Fanout: notify consumers. A pre-staged consumer that becomes ready has @@ -1315,7 +1315,7 @@ struct PTO2SchedulerState { #if SIMPLER_SCHED_PROFILING g_sched_fanout_atomic_count[thread_idx] += fanout_atomics; g_sched_push_wait_cycle[thread_idx] += push_wait; - PTO2_SCHED_CYCLE_LAP(g_sched_fanout_cycle[thread_idx]); + SCHED_CYCLE_LAP(g_sched_fanout_cycle[thread_idx]); return stats; #else return consumer_walk_count; @@ -1330,7 +1330,7 @@ struct PTO2SchedulerState { #if SIMPLER_SCHED_PROFILING int32_t on_task_release(ChipTaskSlotState &slot_state, int32_t thread_idx) { - PTO2_SCHED_CYCLE_START(); + SCHED_CYCLE_START(); extern uint64_t g_sched_fanin_cycle[], g_sched_fanin_atomic_count[]; extern uint64_t g_sched_self_atomic_count[]; extern uint64_t g_sched_self_consumed_cycle[]; @@ -1358,7 +1358,7 @@ struct PTO2SchedulerState { }); #if SIMPLER_SCHED_PROFILING g_sched_fanin_atomic_count[thread_idx] += fanin_atomics; - PTO2_SCHED_CYCLE_LAP(g_sched_fanin_cycle[thread_idx]); + SCHED_CYCLE_LAP(g_sched_fanin_cycle[thread_idx]); #endif // Self consumed check @@ -1366,7 +1366,7 @@ struct PTO2SchedulerState { uint64_t self_atomics = 0; check_and_handle_consumed(slot_state, self_atomics); g_sched_self_atomic_count[thread_idx] += self_atomics; - PTO2_SCHED_CYCLE_LAP(g_sched_self_consumed_cycle[thread_idx]); + SCHED_CYCLE_LAP(g_sched_self_consumed_cycle[thread_idx]); g_sched_complete_count[thread_idx]++; #else check_and_handle_consumed(slot_state); @@ -1380,9 +1380,8 @@ struct PTO2SchedulerState { // per-ring dep_pool entries) on the supplied arena. // Capacities are baked into the returned layout; init_data_from_layout uses // the same values. - static PTO2SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE); - static PTO2SchedulerLayout - reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]); + static SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE); + static SchedulerLayout reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]); // Phase 3a: write everything *except* arena-internal pointer fields. // `sm_dev_base` is the device address of the SM (only stored, never @@ -1391,13 +1390,13 @@ struct PTO2SchedulerState { // task_window_size for ring task_descriptors address arithmetic; the // scheduler only needs the SM header / ring header base addresses, // both window-size-independent.) - bool init_data_from_layout(const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base); - void reset_for_reuse(const PTO2SchedulerLayout &layout, void *sm_dev_base); + bool init_data_from_layout(const SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base); + void reset_for_reuse(const SchedulerLayout &layout, void *sm_dev_base); // Phase 3b: write the arena-internal pointer fields // (ready_queues[].slots, dummy_ready_queue.slots, dep_pool.base for each // ring). Called on both host and device sides. - void wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena); + void wire_arena_pointers(const SchedulerLayout &layout, DeviceArena &arena); // Forget per-region pointers; arena owns the backing memory. void destroy(); @@ -1405,12 +1404,12 @@ struct PTO2SchedulerState { void print_queues(); }; -// Scheduler cold-path API is declared as PTO2SchedulerState member functions. +// Scheduler cold-path API is declared as SchedulerState member functions. // See init()/destroy()/print_stats()/print_queues() below the struct definition. // try_inline_complete_locked: short-circuit NotDeferred completions seen during // drain so they don't grow entries[]. Defined here (not in async_wait.h) -// because PTO2SchedulerState's on_task_complete signature is only known +// because SchedulerState's on_task_complete signature is only known // after its full definition above. // // When the deferred_release_slot_states[] buffer is full, drain it via @@ -1444,8 +1443,8 @@ AsyncWaitList::try_inline_complete_locked(AsyncWaitList::DrainCompletionSink &si template inline AsyncPollResult AsyncWaitList::poll_and_complete( - AICoreCompletionMailbox *aicore_mailbox, PTO2SchedulerState *sched, - ChipTaskSlotState **deferred_release_slot_states, int32_t &deferred_release_count, int32_t deferred_release_capacity + AICoreCompletionMailbox *aicore_mailbox, SchedulerState *sched, ChipTaskSlotState **deferred_release_slot_states, + int32_t &deferred_release_count, int32_t deferred_release_capacity #if SIMPLER_SCHED_PROFILING , int thread_idx @@ -1539,7 +1538,7 @@ inline AsyncPollResult AsyncWaitList::poll_and_complete( // ============================================================================= #if SIMPLER_SCHED_PROFILING -struct PTO2SchedProfilingData { +struct SchedProfilingData { // Sub-phase cycle breakdown within on_task_complete uint64_t lock_cycle; // lock_fanout + state store + unlock uint64_t fanout_cycle; // fanout traversal @@ -1565,5 +1564,5 @@ struct PTO2SchedProfilingData { * Get and reset scheduler profiling data for a specific thread. * Returns accumulated profiling data and resets counters. */ -PTO2SchedProfilingData scheduler_get_profiling(int thread_idx); +SchedProfilingData scheduler_get_profiling(int thread_idx); #endif diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 142a0496b4..8a58633127 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -506,7 +506,7 @@ void SchedulerContext::log_chip_swimlane_summary(int32_t thread_idx, [[maybe_unu if (sched_total == 0) sched_total = 1; { - PTO2SchedProfilingData sp = scheduler_get_profiling(thread_idx); + SchedProfilingData sp = scheduler_get_profiling(thread_idx); uint64_t otc_total = sp.lock_cycle + sp.fanout_cycle + sp.fanin_cycle + sp.self_consumed_cycle; uint64_t complete_poll = (chip_swimlane.sched_complete_cycle > otc_total + chip_swimlane.sched_complete_perf_cycle) ? diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index cd6856bc29..847e642a09 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -731,7 +731,7 @@ void SchedulerContext::handle_drain_mode( // OWNER is acquired before the drain is published and persists through // completion, so every staging thread makes the same gate decision even if // producer release changes early_dispatch_state during the barrier. - bool gated = slot_state->payload != nullptr && PTO2SchedulerState::owns_early_sync_drain(*slot_state->payload); + bool gated = slot_state->payload != nullptr && SchedulerState::owns_early_sync_drain(*slot_state->payload); if (coordinator) { ResourceShape shape = slot_state->active_mask.to_shape(); @@ -819,5 +819,5 @@ void SchedulerContext::handle_drain_mode( } else { sched_->propagate_dispatch_fanin(*slot_state); } - PTO2SchedulerState::finish_early_sync_drain(*slot_state->payload); + SchedulerState::finish_early_sync_drain(*slot_state->payload); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index bca0e7079a..197aa80d43 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -143,7 +143,7 @@ class SchedulerContext { // ========================================================================= // --- Scheduler binding & per-core runtime state --- - alignas(64) PTO2SchedulerState *sched_{nullptr}; + alignas(64) SchedulerState *sched_{nullptr}; RuntimeContext *rt_{nullptr}; // Per-core execution state, indexed by core_id (= worker_id) diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index a935608aee..f18ca65512 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -134,7 +134,7 @@ void SchedulerContext::build_payload( // A claimed early-stage range stays gated even if producer completion flips // the shared state before this payload is built. All other dispatches run on // pickup. - if (PTO2SchedulerState::should_gate_early_dispatch( + if (SchedulerState::should_gate_early_dispatch( force_gate, payload.early_dispatch_state.load(std::memory_order_relaxed) )) { // Gated task: hand the idle AICore the source payload (non-zero = gate) and @@ -681,12 +681,12 @@ int32_t SchedulerContext::stage_consumer_blocks( for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_cores[w] != 0) { owned[w] = - PTO2SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); + SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); } } for (int i = 0; i < n; i++) { int32_t cid = tracker.get_core_id_by_offset(handles[i].core_offset); - PTO2SchedulerState::ring_claimed_local_doorbell( + SchedulerState::ring_claimed_local_doorbell( owned[cid >> 6], cid, handles[i].reg_addr, handles[i].reg_task_id ); } @@ -832,7 +832,7 @@ int32_t SchedulerContext::try_early_dispatch( if (ChipTaskSlotState *c = sched_->early_sync_start_queue.pop_tagged(&sync_task_id_snapshot)) { bool current_sync_task = static_cast(c->task->task_id.raw) == sync_task_id_snapshot && c->task_attrs.requires_sync_start(); - if (current_sync_task && PTO2SchedulerState::try_claim_early_sync_drain(*c->payload)) { + if (current_sync_task && SchedulerState::try_claim_early_sync_drain(*c->payload)) { if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_STAGING) { sched_->cancel_early_sync_drain(*c); } else if (drain_state_.sync_start_pending.load(std::memory_order_acquire) == 0 && @@ -842,7 +842,7 @@ int32_t SchedulerContext::try_early_dispatch( // From this point onward the operation is all-or-nothing. Only this // scheduler mutates its tracker, and global drain coordinators must // wait for this scheduler's generation-tagged ack before inspecting it. - PTO2SchedulerState::mark_early_sync_drain_armed(*c->payload); + SchedulerState::mark_early_sync_drain_armed(*c->payload); always_assert(c->next_block_idx.load(std::memory_order_seq_cst) == 0); SyncStartStageResult staged = stage_sync_start_cores( c, c->logical_block_num, thread_idx, /*gated=*/true, /*record_drain_phases=*/false @@ -852,10 +852,10 @@ int32_t SchedulerContext::try_early_dispatch( static_cast(staged.running_cores), std::memory_order_seq_cst ); sched_->retry_sync_start_rendezvous_after_staging(*c); - PTO2SchedulerState::finish_early_sync_drain(*c->payload); + SchedulerState::finish_early_sync_drain(*c->payload); total_staged += staged.staged_blocks; } else if (enter_drain_mode(c, c->logical_block_num)) { - PTO2SchedulerState::mark_early_sync_drain_armed(*c->payload); + SchedulerState::mark_early_sync_drain_armed(*c->payload); } else { sched_->cancel_early_sync_drain(*c); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 302da4e1c2..96c0ceaedc 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -85,7 +85,7 @@ void ready_queue_destroy(PTO2ReadyQueue *queue) { // Scheduler // ============================================================================= -bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base, int32_t ring_id) { +bool SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base, int32_t ring_id) { // ring stores the device address of the SM ring header — pure offset // arithmetic, no SM load. ring = sm_layout::ring_header_addr(sm_dev_base, ring_id); @@ -104,7 +104,7 @@ bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base return true; } -void PTO2SchedulerState::RingSchedState::reset_for_reuse( +void SchedulerState::RingSchedState::reset_for_reuse( void *sm_dev_base, int32_t ring_id, std::atomic *orch_err ) { ring = sm_layout::ring_header_addr(sm_dev_base, ring_id); @@ -117,9 +117,9 @@ void PTO2SchedulerState::RingSchedState::reset_for_reuse( #endif } -void PTO2SchedulerState::RingSchedState::destroy() { ring = nullptr; } +void SchedulerState::RingSchedState::destroy() { ring = nullptr; } -PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity) { +SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity) { int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { dep_pool_capacities[r] = dep_pool_capacity; @@ -127,9 +127,9 @@ PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena, int32 return reserve_layout(arena, dep_pool_capacities); } -PTO2SchedulerLayout -PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]) { - PTO2SchedulerLayout layout{}; +SchedulerLayout +SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]) { + SchedulerLayout layout{}; layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { layout.dep_pool_capacities[r] = dep_pool_capacities[r]; @@ -155,10 +155,8 @@ PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_ca return layout; } -bool PTO2SchedulerState::init_data_from_layout( - const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base -) { - PTO2SchedulerState *sched = this; +bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base) { + SchedulerState *sched = this; sched->sm_header = reinterpret_cast(sm_dev_base); sched->advance_pending_mask.store(0, std::memory_order_relaxed); #if SIMPLER_SCHED_PROFILING @@ -216,8 +214,8 @@ bool PTO2SchedulerState::init_data_from_layout( return true; } -void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void *sm_dev_base) { - PTO2SchedulerState *sched = this; +void SchedulerState::reset_for_reuse(const SchedulerLayout &layout, void *sm_dev_base) { + SchedulerState *sched = this; sched->sm_header = reinterpret_cast(sm_dev_base); sched->advance_pending_mask.store(0, std::memory_order_relaxed); #if SIMPLER_SCHED_PROFILING @@ -246,8 +244,8 @@ void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void (void)layout; } -void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena) { - PTO2SchedulerState *sched = this; +void SchedulerState::wire_arena_pointers(const SchedulerLayout &layout, DeviceArena &arena) { + SchedulerState *sched = this; for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_queues[i], arena, layout.off_ready_queue_slots[i]); } @@ -267,8 +265,8 @@ void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, } } -void PTO2SchedulerState::destroy() { - PTO2SchedulerState *sched = this; +void SchedulerState::destroy() { + SchedulerState *sched = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { sched->ring_sched_states[r].destroy(); sched->ring_sched_states[r].dep_pool.base = nullptr; @@ -477,7 +475,7 @@ bool PTO2OrchestratorState::reset_for_reuse( } void PTO2OrchestratorState::wire_arena_pointers( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, PTO2SchedulerState *scheduler_arg + const PTO2OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler_arg ) { auto *orch = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -501,7 +499,7 @@ void PTO2OrchestratorState::destroy() { orch->scope_begins = nullptr; } -void PTO2OrchestratorState::set_scheduler(PTO2SchedulerState *scheduler) { this->scheduler = scheduler; } +void PTO2OrchestratorState::set_scheduler(SchedulerState *scheduler) { this->scheduler = scheduler; } // ============================================================================= // Top-level runtime arena @@ -537,7 +535,7 @@ RuntimeArenaLayout runtime_reserve_layout( task_window_sizes_i32[r] = static_cast(task_window_sizes[r]); } layout.offsets.orch = PTO2OrchestratorState::reserve_layout(arena, task_window_sizes_i32, dep_pool_capacities); - layout.offsets.sched = PTO2SchedulerState::reserve_layout(arena, dep_pool_capacities); + layout.offsets.sched = SchedulerState::reserve_layout(arena, dep_pool_capacities); layout.offsets.off_runtime = arena.reserve(sizeof(RuntimeContext), PTO2_ALIGN_SIZE); layout.offsets.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h index fa2645c45e..e0f01abe0d 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h @@ -114,7 +114,7 @@ enum class ResourceShape : uint8_t { // Number of *dispatchable* resource shapes (AIC, AIV, MIX). DUMMY does not // allocate a per-shape ready_queue entry / local buffer — it lives in a -// dedicated queue inside PTO2SchedulerState. +// dedicated queue inside SchedulerState. inline constexpr int32_t NUM_RESOURCE_SHAPES = 3; /** diff --git a/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md b/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md index c8ebfd16d3..724a6763c6 100644 --- a/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md @@ -102,7 +102,7 @@ reserves them in this order: | Zone | Regions | Copied | Allocated on device | Written by | | ---- | ------- | ------ | ------------------- | ---------- | -| device-only | `sm_handle`, the completion mailbox, `PTO2SchedulerState` and its thirteen queue slot arrays | never | yes | AICPU at boot | +| device-only | `sm_handle`, the completion mailbox, `SchedulerState` and its thirteen queue slot arrays | never | yes | AICPU at boot | | copied | `[off_copied_begin, off_copied_end)`: the runtime header | whole zone, one copy | yes | host | The copied zone comes last, so `bind` is a single contiguous `copy_to_device` @@ -122,7 +122,7 @@ putting an owning member back inside it. The one orchestrator value the device-s scheduler reads, the count of tasks completed inline during orchestration, is a scalar `rt_orchestration_done` publishes into the runtime header. -**Why the scheduler state is device-written.** `PTO2SchedulerState` holds no +**Why the scheduler state is device-written.** `SchedulerState` holds no per-run content: `sm_header` and the ring pointer derive from a pooled SM base, queue capacities are compile-time constants, hbg never advances `last_task_alive`, and it has no host-side entry point at all. So the host would @@ -137,7 +137,7 @@ when that slot's `sequence` already equals `pos`, so an empty queue is a match and every later position reads a lower sequence, which is the full-queue signal, so such a queue accepts one push and then reports full. The ramp is mandatory but it is a function of `capacity` alone, so -`PTO2SchedulerState::seed_queue_slots()` writes it on the device rather than `bind` +`SchedulerState::seed_queue_slots()` writes it on the device rather than `bind` shipping 1,775,616 bytes of it. The ready queues are still *not* bounded to `total_tasks`: graph execution expands a GRAPH task into on-device nodes that push past the host task count, so every slot must carry a valid sequence. diff --git a/src/a5/runtime/host_build_graph/runtime/async_wait.h b/src/a5/runtime/host_build_graph/runtime/async_wait.h index 2fe5e335f6..db86ba05d1 100644 --- a/src/a5/runtime/host_build_graph/runtime/async_wait.h +++ b/src/a5/runtime/host_build_graph/runtime/async_wait.h @@ -22,7 +22,7 @@ #include "completion_token.h" #include "runtime_types.h" -struct PTO2SchedulerState; +struct SchedulerState; struct CompletionStats; inline constexpr int32_t MAX_ASYNC_WAITS = 64; @@ -172,7 +172,7 @@ struct AsyncWaitList { // NotDeferred tasks inline (without storing a transient entry in // entries[]). struct DrainCompletionSink { - PTO2SchedulerState *sched{nullptr}; + SchedulerState *sched{nullptr}; int32_t inline_completed{0}; int32_t inline_resolved{0}; int32_t error_code{SIMPLER_ERROR_NONE}; @@ -293,7 +293,7 @@ struct AsyncWaitList { template AsyncPollResult poll_and_complete( - AICoreCompletionMailbox *aicore_mailbox, PTO2SchedulerState *sched + AICoreCompletionMailbox *aicore_mailbox, SchedulerState *sched #if SIMPLER_SCHED_PROFILING , int thread_idx diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator.h b/src/a5/runtime/host_build_graph/runtime/orchestrator.h index 816345b169..abed45ef46 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator.h @@ -80,7 +80,7 @@ struct PTO2OrchestratorState { // === SCHEDULER REFERENCE === // Note: In simulated mode, orchestrator and scheduler share address space // In real mode, they communicate via shared memory only - PTO2SchedulerState *scheduler; // For simulated mode only + SchedulerState *scheduler; // For simulated mode only // Total core counts set once at executor init; used for submit-time deadlock detection. int32_t total_cluster_count{0}; // AIC cores = MIX clusters @@ -138,10 +138,9 @@ struct PTO2OrchestratorState { // // Returns false when an allocation fails; the caller then has no hazard map // and must not orchestrate. - bool - init(void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, PTO2SchedulerState *scheduler); + bool init(void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, SchedulerState *scheduler); - void set_scheduler(PTO2SchedulerState *scheduler); + void set_scheduler(SchedulerState *scheduler); void report_fatal(int32_t error_code, const char *func, const char *fmt, ...); void begin_scope(PTO2ScopeMode mode = PTO2ScopeMode::AUTO); void end_scope(); diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 55e08f0154..e229f0cd41 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1460,7 +1460,7 @@ static TaskOutputTensors submit_task_common( if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; } - PTO2SchedulerState *sched = orch->scheduler; + SchedulerState *sched = orch->scheduler; TaskId task_id = prepared.task_id; TaskDescriptor &task = *prepared.task; TaskPayload &payload = *prepared.payload; diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_core.h b/src/a5/runtime/host_build_graph/runtime/runtime_core.h index f47a43161c..04cacd46af 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_core.h @@ -119,7 +119,7 @@ struct RuntimeOps { */ struct RuntimeArenaLayout { size_t off_sm_handle{0}; - PTO2SchedulerLayout sched; + SchedulerLayout sched; size_t off_scheduler{0}; size_t off_runtime{0}; size_t off_mailbox{0}; @@ -179,7 +179,7 @@ struct RuntimeContext { PTO2OrchestratorState *orchestrator; // Device-only zone: the scheduler state holds no per-run content, so it is // addressed through the arena rather than carried inside this header. - PTO2SchedulerState *scheduler; + SchedulerState *scheduler; AICoreCompletionMailbox *aicore_mailbox; // Mode diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.cpp index 6458fcb747..b54d7bd7b0 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.cpp @@ -42,8 +42,8 @@ uint64_t g_sched_self_atomic_count[PLATFORM_MAX_AICPU_THREADS] = {}; uint64_t g_sched_pop_atomic_count[PLATFORM_MAX_AICPU_THREADS] = {}; uint64_t g_sched_complete_count[PLATFORM_MAX_AICPU_THREADS] = {}; -PTO2SchedProfilingData scheduler_get_profiling(int thread_idx) { - PTO2SchedProfilingData d; +SchedProfilingData scheduler_get_profiling(int thread_idx) { + SchedProfilingData d; d.lock_cycle = std::exchange(g_sched_lock_cycle[thread_idx], 0); d.fanout_cycle = std::exchange(g_sched_fanout_cycle[thread_idx], 0); d.fanin_cycle = std::exchange(g_sched_fanin_cycle[thread_idx], 0); @@ -65,7 +65,7 @@ PTO2SchedProfilingData scheduler_get_profiling(int thread_idx) { // Debug Utilities // ============================================================================= -void PTO2SchedulerState::print_stats() { +void SchedulerState::print_stats() { LOG_DEBUG("=== Scheduler Statistics ==="); #if SIMPLER_SCHED_PROFILING LOG_DEBUG("tasks_completed: %lld", (long long)tasks_completed.load(std::memory_order_relaxed)); @@ -74,8 +74,8 @@ void PTO2SchedulerState::print_stats() { LOG_DEBUG("============================"); } -void PTO2SchedulerState::print_queues() { - PTO2SchedulerState *sched = this; +void SchedulerState::print_queues() { + SchedulerState *sched = this; const char *shape_names[] = {"AIC", "AIV", "MIX"}; for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { LOG_TIMING( diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h index 8655213cc0..6b5385bab5 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -45,8 +45,8 @@ #include "aicpu/device_time.h" // get_sys_cnt_aicpu (weak; used by early-dispatch doorbell timing too) #if SIMPLER_SCHED_PROFILING -#define PTO2_SCHED_CYCLE_START() uint64_t _st0 = get_sys_cnt_aicpu(), _st1 -#define PTO2_SCHED_CYCLE_LAP(acc) \ +#define SCHED_CYCLE_START() uint64_t _st0 = get_sys_cnt_aicpu(), _st1 +#define SCHED_CYCLE_LAP(acc) \ do { \ _st1 = get_sys_cnt_aicpu(); \ acc += (_st1 - _st0); \ @@ -448,11 +448,11 @@ struct CompletionStats { }; /** - * Layout descriptor produced by PTO2SchedulerState::reserve_layout(). Holds + * Layout descriptor produced by SchedulerState::reserve_layout(). Holds * the arena offsets of every sub-region the scheduler needs plus the * capacities used at layout time (init_from_layout reuses them). */ -struct PTO2SchedulerLayout { +struct SchedulerLayout { size_t off_ready_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_ready_sync_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_dummy_ready_queue_slots; @@ -470,7 +470,7 @@ struct PTO2SchedulerLayout { * Separated from shared memory for cache efficiency. * Hot-path methods are defined inline (implicitly inline as member functions). */ -struct PTO2SchedulerState { +struct SchedulerState { // Shared memory access SharedMemoryHeader *sm_header; @@ -1166,7 +1166,7 @@ struct PTO2SchedulerState { // per-ring dep_pool entries) on the supplied arena. // Capacities are baked into the returned layout; init_data_from_layout uses // the same values. - static PTO2SchedulerLayout reserve_layout(DeviceArena &arena); + static SchedulerLayout reserve_layout(DeviceArena &arena); // Phase 3a: write everything *except* arena-internal pointer fields. // `sm_dev_base` is the device address of the SM (only stored, never @@ -1175,12 +1175,12 @@ struct PTO2SchedulerState { // task_window_size for ring task_descriptors address arithmetic; the // scheduler only needs the SM header / ring header base addresses, // both window-size-independent.) - bool init_data_from_layout(const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base); + bool init_data_from_layout(const SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base); // Phase 3b: write the arena-internal pointer fields // (ready_queues[].slots, dummy_ready_queue.slots, dep_pool.base for each // ring). Called on both host and device sides. - void wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena); + void wire_arena_pointers(const SchedulerLayout &layout, DeviceArena &arena); // Phase 3c, device only: bring every queue's slots[] to its empty ramp. The // slot arrays sit past the uploaded range, so this is the only thing that @@ -1195,12 +1195,12 @@ struct PTO2SchedulerState { void print_queues(); }; -// Scheduler cold-path API is declared as PTO2SchedulerState member functions. +// Scheduler cold-path API is declared as SchedulerState member functions. // See init()/destroy()/print_stats()/print_queues() below the struct definition. // try_inline_complete_locked: short-circuit NotDeferred completions seen during // drain so they don't grow entries[]. Defined here (not in async_wait.h) -// because PTO2SchedulerState's on_task_complete signature is only known +// because SchedulerState's on_task_complete signature is only known // after its full definition above. // // Polling: on_task_complete publishes completion + drains the wake list inline, @@ -1210,9 +1210,9 @@ AsyncWaitList::try_inline_complete_locked(AsyncWaitList::DrainCompletionSink &si // Return value (CompletionStats / consumer-walk count) discarded: // async-wait drain path has no Resolve swimlane bar attached. #if SIMPLER_SCHED_PROFILING - PTO2SchedulerState::TaskCompletionOutcome outcome = sink.sched->complete_task(slot_state, sink.thread_idx); + SchedulerState::TaskCompletionOutcome outcome = sink.sched->complete_task(slot_state, sink.thread_idx); #else - PTO2SchedulerState::TaskCompletionOutcome outcome = sink.sched->complete_task(slot_state); + SchedulerState::TaskCompletionOutcome outcome = sink.sched->complete_task(slot_state); #endif if (outcome.error_code != SIMPLER_ERROR_NONE) { sink.error_code = outcome.error_code; @@ -1225,7 +1225,7 @@ AsyncWaitList::try_inline_complete_locked(AsyncWaitList::DrainCompletionSink &si template inline AsyncPollResult AsyncWaitList::poll_and_complete( - AICoreCompletionMailbox *aicore_mailbox, PTO2SchedulerState *sched + AICoreCompletionMailbox *aicore_mailbox, SchedulerState *sched #if SIMPLER_SCHED_PROFILING , int thread_idx @@ -1281,9 +1281,9 @@ inline AsyncPollResult AsyncWaitList::poll_and_complete( // Return value (CompletionStats / consumer-walk count) discarded: // deferred-completion drain has no Resolve swimlane bar attached. #if SIMPLER_SCHED_PROFILING - PTO2SchedulerState::TaskCompletionOutcome outcome = sched->complete_task(*entry.slot_state, thread_idx); + SchedulerState::TaskCompletionOutcome outcome = sched->complete_task(*entry.slot_state, thread_idx); #else - PTO2SchedulerState::TaskCompletionOutcome outcome = sched->complete_task(*entry.slot_state); + SchedulerState::TaskCompletionOutcome outcome = sched->complete_task(*entry.slot_state); #endif if (outcome.error_code != SIMPLER_ERROR_NONE) { result.error_code = outcome.error_code; @@ -1310,7 +1310,7 @@ inline AsyncPollResult AsyncWaitList::poll_and_complete( // ============================================================================= #if SIMPLER_SCHED_PROFILING -struct PTO2SchedProfilingData { +struct SchedProfilingData { // Sub-phase cycle breakdown within on_task_complete uint64_t lock_cycle; // lock_fanout + state store + unlock uint64_t fanout_cycle; // fanout traversal @@ -1336,5 +1336,5 @@ struct PTO2SchedProfilingData { * Get and reset scheduler profiling data for a specific thread. * Returns accumulated profiling data and resets counters. */ -PTO2SchedProfilingData scheduler_get_profiling(int thread_idx); +SchedProfilingData scheduler_get_profiling(int thread_idx); #endif diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index 3070d33c4c..5e97bf3092 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -430,7 +430,7 @@ void SchedulerContext::log_chip_swimlane_summary(int32_t thread_idx, int32_t cur #if SIMPLER_SCHED_PROFILING { - PTO2SchedProfilingData sp = scheduler_get_profiling(thread_idx); + SchedProfilingData sp = scheduler_get_profiling(thread_idx); uint64_t otc_total = sp.lock_cycle + sp.fanout_cycle + sp.fanin_cycle + sp.self_consumed_cycle; uint64_t complete_poll = (chip_swimlane.sched_complete_cycle > otc_total + chip_swimlane.sched_complete_perf_cycle) ? diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index e94b50d8af..a1ddd50333 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -682,7 +682,7 @@ void SchedulerContext::handle_drain_mode(int32_t thread_idx, [[maybe_unused]] ui // OWNER is acquired before the drain is published and persists through // completion, so every staging thread makes the same gate decision even if // producer release changes early_dispatch_state during the barrier. - bool gated = slot_state->payload != nullptr && PTO2SchedulerState::owns_early_sync_drain(*slot_state->payload); + bool gated = slot_state->payload != nullptr && SchedulerState::owns_early_sync_drain(*slot_state->payload); if (coordinator) { ResourceShape shape = slot_state->active_mask.to_shape(); @@ -769,5 +769,5 @@ void SchedulerContext::handle_drain_mode(int32_t thread_idx, [[maybe_unused]] ui } else { sched_->propagate_dispatch_fanin(*slot_state); } - PTO2SchedulerState::finish_early_sync_drain(*slot_state->payload); + SchedulerState::finish_early_sync_drain(*slot_state->payload); } diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index 727aa205a7..f2fdba1daa 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -188,7 +188,7 @@ class SchedulerContext { // ========================================================================= // --- Scheduler binding & per-core runtime state --- - alignas(64) PTO2SchedulerState *sched_{nullptr}; + alignas(64) SchedulerState *sched_{nullptr}; RuntimeContext *rt_{nullptr}; // Per-core execution state, indexed by core_id (= worker_id) diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 38f8b6947d..1713aae3c3 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -132,7 +132,7 @@ void SchedulerContext::build_payload( // A claimed early-stage range stays gated even if producer completion flips // the shared state before this payload is built. All other dispatches run on // pickup. - if (PTO2SchedulerState::should_gate_early_dispatch( + if (SchedulerState::should_gate_early_dispatch( force_gate, payload.early_dispatch_state.load(std::memory_order_relaxed) )) { // Gated task: hand the idle AICore the source payload (non-zero = gate) and @@ -680,12 +680,12 @@ int32_t SchedulerContext::stage_consumer_blocks( for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_cores[w] != 0) { owned[w] = - PTO2SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); + SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); } } for (int i = 0; i < n; i++) { int32_t cid = tracker.get_core_id_by_offset(handles[i].core_offset); - PTO2SchedulerState::ring_claimed_local_doorbell( + SchedulerState::ring_claimed_local_doorbell( owned[cid >> 6], cid, handles[i].reg_addr, handles[i].reg_task_id ); } @@ -825,7 +825,7 @@ int32_t SchedulerContext::try_early_dispatch( if (ChipTaskSlotState *c = sched_->early_sync_start_queue.pop_tagged(&sync_task_id_snapshot)) { bool current_sync_task = static_cast(c->task->task_id.raw) == sync_task_id_snapshot && c->task_attrs.requires_sync_start(); - if (current_sync_task && PTO2SchedulerState::try_claim_early_sync_drain(*c->payload)) { + if (current_sync_task && SchedulerState::try_claim_early_sync_drain(*c->payload)) { if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_STAGING) { sched_->cancel_early_sync_drain(*c); } else if (drain_state_.sync_start_pending.load(std::memory_order_acquire) == 0 && @@ -835,7 +835,7 @@ int32_t SchedulerContext::try_early_dispatch( // From this point onward the operation is all-or-nothing. Only this // scheduler mutates its tracker, and global drain coordinators must // wait for this scheduler's generation-tagged ack before inspecting it. - PTO2SchedulerState::mark_early_sync_drain_armed(*c->payload); + SchedulerState::mark_early_sync_drain_armed(*c->payload); always_assert(c->next_block_idx.load(std::memory_order_seq_cst) == 0); SyncStartStageResult staged = stage_sync_start_cores( c, c->logical_block_num, thread_idx, /*gated=*/true, /*record_drain_phases=*/false @@ -845,10 +845,10 @@ int32_t SchedulerContext::try_early_dispatch( static_cast(staged.running_cores), std::memory_order_seq_cst ); sched_->retry_sync_start_rendezvous_after_staging(*c); - PTO2SchedulerState::finish_early_sync_drain(*c->payload); + SchedulerState::finish_early_sync_drain(*c->payload); total_staged += staged.staged_blocks; } else if (enter_drain_mode(c, c->logical_block_num)) { - PTO2SchedulerState::mark_early_sync_drain_armed(*c->payload); + SchedulerState::mark_early_sync_drain_armed(*c->payload); } else { sched_->cancel_early_sync_drain(*c); } @@ -925,9 +925,9 @@ int32_t SchedulerContext::run_resolution_thread(Runtime *runtime, int32_t thread ChipTaskSlotState *slot; while ((slot = sp_queues_[s].pop()) != nullptr) { #if SIMPLER_SCHED_PROFILING - PTO2SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*slot, thread_idx); + SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*slot, thread_idx); #else - PTO2SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*slot); + SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*slot); #endif if (outcome.error_code != SIMPLER_ERROR_NONE) { fail_scheduler(runtime, thread_idx, outcome.error_code); @@ -971,10 +971,9 @@ int32_t SchedulerContext::run_resolution_thread(Runtime *runtime, int32_t thread while ((dummy_got = sched_->dummy_ready_queue.pop_batch(dummy_batch, DUMMY_DRAIN_BATCH)) > 0) { for (int di = 0; di < dummy_got; di++) { #if SIMPLER_SCHED_PROFILING - PTO2SchedulerState::TaskCompletionOutcome outcome = - sched_->complete_task(*dummy_batch[di], thread_idx); + SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*dummy_batch[di], thread_idx); #else - PTO2SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*dummy_batch[di]); + SchedulerState::TaskCompletionOutcome outcome = sched_->complete_task(*dummy_batch[di]); #endif if (outcome.error_code != SIMPLER_ERROR_NONE) { fail_scheduler(runtime, thread_idx, outcome.error_code); diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h index 3ba97ee8a7..367ae824eb 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h @@ -19,7 +19,7 @@ #include "runtime_types.h" #include "spin_hint.h" -// host_build_graph host-orch build: RuntimeContext embeds PTO2SchedulerState by +// host_build_graph host-orch build: RuntimeContext embeds SchedulerState by // value, so this header is compiled into the host libhost_runtime.so. The AICPU // spin_hint.h that defines PLATFORM_SCHEDULER_TIMEOUT_MS is not on the host // include path; supply it here. The value only sizes an on-device scheduler diff --git a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 29b9a4195f..e98c0b1cf2 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -41,7 +41,7 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Initialize the queue header only. slots[] carries the sequence ramp push // compares against, but it lives past the uploaded range and is seeded on the -// device by PTO2SchedulerState::seed_queue_slots(), so nothing writes it here. +// device by SchedulerState::seed_queue_slots(), so nothing writes it here. void ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, uint64_t capacity) { queue->capacity = capacity; queue->mask = capacity - 1; @@ -63,7 +63,7 @@ void ready_queue_destroy(PTO2ReadyQueue *queue) { // Scheduler // ============================================================================= -bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base) { +bool SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base) { // ring stores the device address of the SM ring header — pure offset // arithmetic, no SM load. ring = sm_layout::ring_header_addr(sm_dev_base); @@ -76,10 +76,10 @@ bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base return true; } -void PTO2SchedulerState::RingSchedState::destroy() { ring = nullptr; } +void SchedulerState::RingSchedState::destroy() { ring = nullptr; } -PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena) { - PTO2SchedulerLayout layout{}; +SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena) { + SchedulerLayout layout{}; layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; // Fixed-capacity early-dispatch queues first, then the PTO2_READY_QUEUE_SIZE @@ -103,10 +103,8 @@ PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena) { return layout; } -bool PTO2SchedulerState::init_data_from_layout( - const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base -) { - PTO2SchedulerState *sched = this; +bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base) { + SchedulerState *sched = this; sched->sm_header = reinterpret_cast(sm_dev_base); #if SIMPLER_SCHED_PROFILING sched->tasks_completed.store(0, std::memory_order_relaxed); @@ -141,8 +139,8 @@ bool PTO2SchedulerState::init_data_from_layout( // wire_arena_pointers / destroy — a queue whose slots[] is never seeded accepts // one push and then reports full, since push claims a slot only when its // sequence already equals the position being claimed. -void PTO2SchedulerState::seed_queue_slots() { - PTO2SchedulerState *sched = this; +void SchedulerState::seed_queue_slots() { + SchedulerState *sched = this; for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { sched->ready_queues[i].seed_slots(); } @@ -158,8 +156,8 @@ void PTO2SchedulerState::seed_queue_slots() { sched->early_sync_start_queue.seed_slots(); } -void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena) { - PTO2SchedulerState *sched = this; +void SchedulerState::wire_arena_pointers(const SchedulerLayout &layout, DeviceArena &arena) { + SchedulerState *sched = this; for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_queues[i], arena, layout.off_ready_queue_slots[i]); } @@ -177,8 +175,8 @@ void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, ready_queue_wire_arena_pointers(&sched->early_sync_start_queue, arena, layout.off_early_sync_start_queue_slots); } -void PTO2SchedulerState::destroy() { - PTO2SchedulerState *sched = this; +void SchedulerState::destroy() { + SchedulerState *sched = this; sched->ring_sched_state.destroy(); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_queues[i]); @@ -200,7 +198,7 @@ void PTO2SchedulerState::destroy() { // ============================================================================= bool PTO2OrchestratorState::init( - void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, PTO2SchedulerState *scheduler_arg + void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, SchedulerState *scheduler_arg ) { auto *orch = this; *orch = PTO2OrchestratorState{}; @@ -245,7 +243,7 @@ bool PTO2OrchestratorState::init( return true; } -void PTO2OrchestratorState::set_scheduler(PTO2SchedulerState *scheduler) { this->scheduler = scheduler; } +void PTO2OrchestratorState::set_scheduler(SchedulerState *scheduler) { this->scheduler = scheduler; } // ============================================================================= // Top-level runtime arena @@ -265,8 +263,8 @@ RuntimeArenaLayout runtime_reserve_layout(DeviceArena &arena, uint64_t task_wind // begin where it ends, making the two adjacent and the upload one copy. layout.off_sm_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); layout.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); - layout.off_scheduler = arena.reserve(sizeof(PTO2SchedulerState), alignof(PTO2SchedulerState)); - layout.sched = PTO2SchedulerState::reserve_layout(arena); + layout.off_scheduler = arena.reserve(sizeof(SchedulerState), alignof(SchedulerState)); + layout.sched = SchedulerState::reserve_layout(arena); layout.off_copied_begin = arena.total_size(); // Padded to a PTO2_ALIGN_SIZE boundary: the shared-memory image starts at @@ -319,7 +317,7 @@ RuntimeContext *runtime_init_data_from_layout( void runtime_wire_arena_pointers(DeviceArena &arena, const RuntimeArenaLayout &layout, RuntimeContext *rt) { rt->sm_handle = static_cast(arena.region_ptr(layout.off_sm_handle)); rt->aicore_mailbox = static_cast(arena.region_ptr(layout.off_mailbox)); - rt->scheduler = static_cast(arena.region_ptr(layout.off_scheduler)); + rt->scheduler = static_cast(arena.region_ptr(layout.off_scheduler)); rt->scheduler->wire_arena_pointers(layout.sched, arena); } diff --git a/src/a5/runtime/host_build_graph/runtime/submit_types.h b/src/a5/runtime/host_build_graph/runtime/submit_types.h index abcebb86f1..daeabb2776 100644 --- a/src/a5/runtime/host_build_graph/runtime/submit_types.h +++ b/src/a5/runtime/host_build_graph/runtime/submit_types.h @@ -114,7 +114,7 @@ enum class ResourceShape : uint8_t { // Number of *dispatchable* resource shapes (AIC, AIV, MIX). DUMMY does not // allocate a per-shape ready_queue entry / local buffer — it lives in a -// dedicated queue inside PTO2SchedulerState. +// dedicated queue inside SchedulerState. inline constexpr int32_t NUM_RESOURCE_SHAPES = 3; /** diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index dc363705df..a93e319afc 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -126,7 +126,7 @@ struct SharedMemoryHandle { }; ``` -### 4.5 PTO2SchedulerState (modified) +### 4.5 SchedulerState (modified) ```cpp struct RingSchedState { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/async_wait.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/async_wait.h index 4f3a518108..6c282cb065 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/async_wait.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/async_wait.h @@ -24,7 +24,7 @@ #include "completion_token.h" #include "runtime_types.h" -struct PTO2SchedulerState; +struct SchedulerState; struct CompletionStats; inline constexpr int32_t MAX_ASYNC_WAITS = 64; @@ -184,7 +184,7 @@ struct AsyncWaitList { // NotDeferred tasks inline (without storing a transient entry in // entries[]). struct DrainCompletionSink { - PTO2SchedulerState *sched{nullptr}; + SchedulerState *sched{nullptr}; ChipTaskSlotState **deferred_release_slot_states{nullptr}; int32_t *deferred_release_count{nullptr}; int32_t deferred_release_capacity{0}; @@ -304,7 +304,7 @@ struct AsyncWaitList { template AsyncPollResult poll_and_complete( - AICoreCompletionMailbox *aicore_mailbox, PTO2SchedulerState *sched, + AICoreCompletionMailbox *aicore_mailbox, SchedulerState *sched, ChipTaskSlotState **deferred_release_slot_states, int32_t &deferred_release_count, int32_t deferred_release_capacity #if SIMPLER_SCHED_PROFILING diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index cc13d7ba3d..1477edd4cd 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -425,7 +425,7 @@ static bool all_claimed_fanin_allow_early_resolve(const PTO2FaninBuilder &fanin_ } void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state) { - PTO2SchedulerState *sched = scheduler; + SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; slot_state.dep_pool_mark = rss.dep_pool.top; #if SIMPLER_DFX @@ -436,7 +436,7 @@ void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state } void PTO2OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin) { - PTO2SchedulerState *sched = scheduler; + SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; TaskPayload *payload = slot_state.payload; slot_state.fanin_count = wfanin + 1; @@ -510,7 +510,7 @@ static ChipTaskSlotState *oldest_open_task_on_current_ring(const PTO2Orchestrato } static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, ChipTaskSlotState &slot_state, int32_t wfanin) { - PTO2SchedulerState *sched = orch->scheduler; + SchedulerState *sched = orch->scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; // dep_pool is orchestrator-exclusive (no lock). ensure_space waits for the @@ -785,7 +785,7 @@ static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t neede uint32_t all_ring_bits = 0; for (int32_t r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - all_ring_bits |= PTO2SchedulerState::ring_advance_pending_bit(r); + all_ring_bits |= SchedulerState::ring_advance_pending_bit(r); } // No acknowledgment is consumed here because TensorMap reclamation has no // structural head classification; repeated watermark reads are sufficient. @@ -895,7 +895,7 @@ static TaskOutputTensors submit_task_common( return result; } uint8_t ring_id = prepared.task_id.ring(); - PTO2SchedulerState *sched = orch->scheduler; + SchedulerState *sched = orch->scheduler; PTO2RingFlowControl &fc = orch->sm_header->rings[ring_id].fc; TaskId task_id = prepared.task_id; ChipTaskSlotState &cur_slot_state = *prepared.slot_state; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 4cc060dcd1..aa4a6ceb00 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -89,7 +89,7 @@ struct PTO2OrchestratorState { // === SCHEDULER STATE ACCESS === // Same runtime-arena scheduler object; Orch-side wiring mutates dep_pool // and publishes ready tasks through it before scheduler workers dispatch. - PTO2SchedulerState *scheduler; + SchedulerState *scheduler; // Total core counts set once at executor init; used for submit-time deadlock detection. int32_t total_cluster_count{0}; // AIC cores = MIX clusters @@ -169,11 +169,11 @@ struct PTO2OrchestratorState { // scope_begins, rings[].fanin_pool.base, tensor_map.{buckets,entry_pool, // free_entry_list,task_entry_heads}, scheduler reference). // Idempotent — host runs once on the image, AICPU runs once after attach. - void wire_arena_pointers(const PTO2OrchestratorLayout &layout, DeviceArena &arena, PTO2SchedulerState *scheduler); + void wire_arena_pointers(const PTO2OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler); // Forget pointers; arena owns the backing buffers. void destroy(); - void set_scheduler(PTO2SchedulerState *scheduler); + void set_scheduler(SchedulerState *scheduler); void mark_dep_pool_position(ChipTaskSlotState &slot_state); void wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin); void report_fatal(int32_t error_code, const char *func, const char *fmt, ...); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index fcf6251129..eff705d01b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -123,7 +123,7 @@ struct ArenaSizingKey { struct ArenaOffsets { size_t off_sm_handle{0}; PTO2OrchestratorLayout orch; - PTO2SchedulerLayout sched; + SchedulerLayout sched; size_t off_runtime{0}; size_t off_mailbox{0}; @@ -158,7 +158,7 @@ struct RuntimeContext { // Components SharedMemoryHandle *sm_handle; PTO2OrchestratorState orchestrator; - PTO2SchedulerState scheduler; + SchedulerState scheduler; AICoreCompletionMailbox *aicore_mailbox; // GM Heap for output buffers diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp index 23feaf064a..6c5e42adf2 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.cpp @@ -48,8 +48,8 @@ uint64_t g_sched_self_atomic_count[PLATFORM_MAX_AICPU_THREADS] = {}; uint64_t g_sched_pop_atomic_count[PLATFORM_MAX_AICPU_THREADS] = {}; uint64_t g_sched_complete_count[PLATFORM_MAX_AICPU_THREADS] = {}; -PTO2SchedProfilingData scheduler_get_profiling(int thread_idx) { - PTO2SchedProfilingData d; +SchedProfilingData scheduler_get_profiling(int thread_idx) { + SchedProfilingData d; d.lock_cycle = std::exchange(g_sched_lock_cycle[thread_idx], 0); d.fanout_cycle = std::exchange(g_sched_fanout_cycle[thread_idx], 0); d.fanin_cycle = std::exchange(g_sched_fanin_cycle[thread_idx], 0); @@ -71,8 +71,8 @@ PTO2SchedProfilingData scheduler_get_profiling(int thread_idx) { // Debug Utilities // ============================================================================= -void PTO2SchedulerState::print_stats() { - PTO2SchedulerState *sched = this; +void SchedulerState::print_stats() { + SchedulerState *sched = this; LOG_DEBUG("=== Scheduler Statistics ==="); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { if (sched->ring_sched_states[r].last_task_alive > 0) { @@ -94,8 +94,8 @@ void PTO2SchedulerState::print_stats() { LOG_DEBUG("============================"); } -void PTO2SchedulerState::print_queues() { - PTO2SchedulerState *sched = this; +void SchedulerState::print_queues() { + SchedulerState *sched = this; LOG_DEBUG("=== Ready Queues ==="); const char *shape_names[] = {"AIC", "AIV", "MIX"}; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index a7cb9ce1d9..027a8aed6c 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -41,8 +41,8 @@ #if SIMPLER_SCHED_PROFILING #include "aicpu/device_time.h" -#define PTO2_SCHED_CYCLE_START() uint64_t _st0 = get_sys_cnt_aicpu(), _st1 -#define PTO2_SCHED_CYCLE_LAP(acc) \ +#define SCHED_CYCLE_START() uint64_t _st0 = get_sys_cnt_aicpu(), _st1 +#define SCHED_CYCLE_LAP(acc) \ do { \ _st1 = get_sys_cnt_aicpu(); \ acc += (_st1 - _st0); \ @@ -420,11 +420,11 @@ struct CompletionStats { }; /** - * Layout descriptor produced by PTO2SchedulerState::reserve_layout(). Holds + * Layout descriptor produced by SchedulerState::reserve_layout(). Holds * the arena offsets of every sub-region the scheduler needs plus the * capacities used at layout time (init_data_from_layout reuses them). */ -struct PTO2SchedulerLayout { +struct SchedulerLayout { size_t off_ready_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_ready_sync_queue_slots[NUM_RESOURCE_SHAPES]; size_t off_dummy_ready_queue_slots; @@ -442,7 +442,7 @@ struct PTO2SchedulerLayout { * Separated from shared memory for cache efficiency. * Hot-path methods are defined inline (implicitly inline as member functions). */ -struct PTO2SchedulerState { +struct SchedulerState { // Shared memory access SharedMemoryHeader *sm_header; @@ -1178,7 +1178,7 @@ struct PTO2SchedulerState { extern uint64_t g_sched_lock_atomic_count[], g_sched_lock_wait_cycle[]; extern uint64_t g_sched_fanout_atomic_count[], g_sched_push_wait_cycle[]; uint64_t lock_atomics = 0, lock_wait = 0; - PTO2_SCHED_CYCLE_START(); + SCHED_CYCLE_START(); #endif #if SIMPLER_SCHED_PROFILING @@ -1194,7 +1194,7 @@ struct PTO2SchedulerState { lock_atomics += 3; // task_state.store + lifecycle_flags.fetch_or + unlock.store g_sched_lock_atomic_count[thread_idx] += lock_atomics; g_sched_lock_wait_cycle[thread_idx] += lock_wait; - PTO2_SCHED_CYCLE_LAP(g_sched_lock_cycle[thread_idx]); + SCHED_CYCLE_LAP(g_sched_lock_cycle[thread_idx]); #endif // Fanout: notify consumers @@ -1222,7 +1222,7 @@ struct PTO2SchedulerState { #if SIMPLER_SCHED_PROFILING g_sched_fanout_atomic_count[thread_idx] += fanout_atomics; g_sched_push_wait_cycle[thread_idx] += push_wait; - PTO2_SCHED_CYCLE_LAP(g_sched_fanout_cycle[thread_idx]); + SCHED_CYCLE_LAP(g_sched_fanout_cycle[thread_idx]); return stats; #else return consumer_walk_count; @@ -1237,7 +1237,7 @@ struct PTO2SchedulerState { #if SIMPLER_SCHED_PROFILING int32_t on_task_release(ChipTaskSlotState &slot_state, int32_t thread_idx) { - PTO2_SCHED_CYCLE_START(); + SCHED_CYCLE_START(); extern uint64_t g_sched_fanin_cycle[], g_sched_fanin_atomic_count[]; extern uint64_t g_sched_self_atomic_count[]; extern uint64_t g_sched_self_consumed_cycle[]; @@ -1265,7 +1265,7 @@ struct PTO2SchedulerState { }); #if SIMPLER_SCHED_PROFILING g_sched_fanin_atomic_count[thread_idx] += fanin_atomics; - PTO2_SCHED_CYCLE_LAP(g_sched_fanin_cycle[thread_idx]); + SCHED_CYCLE_LAP(g_sched_fanin_cycle[thread_idx]); #endif // Self consumed check @@ -1273,7 +1273,7 @@ struct PTO2SchedulerState { uint64_t self_atomics = 0; check_and_handle_consumed(slot_state, self_atomics); g_sched_self_atomic_count[thread_idx] += self_atomics; - PTO2_SCHED_CYCLE_LAP(g_sched_self_consumed_cycle[thread_idx]); + SCHED_CYCLE_LAP(g_sched_self_consumed_cycle[thread_idx]); g_sched_complete_count[thread_idx]++; #else check_and_handle_consumed(slot_state); @@ -1287,9 +1287,8 @@ struct PTO2SchedulerState { // per-ring dep_pool entries) on the supplied arena. // Capacities are baked into the returned layout; init_data_from_layout uses // the same values. - static PTO2SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE); - static PTO2SchedulerLayout - reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]); + static SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE); + static SchedulerLayout reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]); // Phase 3a: write everything *except* arena-internal pointer fields. // `sm_dev_base` is the device address of the SM (only stored, never @@ -1298,14 +1297,14 @@ struct PTO2SchedulerState { // task_window_size for ring task_descriptors address arithmetic; the // scheduler only needs the SM header / ring header base addresses, // both window-size-independent.) - bool init_data_from_layout(const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base); - void reset_for_reuse(const PTO2SchedulerLayout &layout, void *sm_dev_base); + bool init_data_from_layout(const SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base); + void reset_for_reuse(const SchedulerLayout &layout, void *sm_dev_base); // Phase 3b: write the arena-internal pointer fields // (ready_queues[].slots, dummy_ready_queue.slots, and dep_pool base plus // reclaim-publication pointers for each ring). Called on both host and // device sides. - void wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena); + void wire_arena_pointers(const SchedulerLayout &layout, DeviceArena &arena); // Forget per-region pointers; arena owns the backing memory. void destroy(); @@ -1313,7 +1312,7 @@ struct PTO2SchedulerState { void print_queues(); }; -// Scheduler cold-path API is declared as PTO2SchedulerState member functions. +// Scheduler cold-path API is declared as SchedulerState member functions. // See init()/destroy()/print_stats()/print_queues() below the struct definition. // Short-circuit NotDeferred completions seen during drain so they don't grow @@ -1343,8 +1342,8 @@ AsyncWaitList::try_inline_complete_locked(AsyncWaitList::DrainCompletionSink &si template inline AsyncPollResult AsyncWaitList::poll_and_complete( - AICoreCompletionMailbox *aicore_mailbox, PTO2SchedulerState *sched, - ChipTaskSlotState **deferred_release_slot_states, int32_t &deferred_release_count, int32_t deferred_release_capacity + AICoreCompletionMailbox *aicore_mailbox, SchedulerState *sched, ChipTaskSlotState **deferred_release_slot_states, + int32_t &deferred_release_count, int32_t deferred_release_capacity #if SIMPLER_SCHED_PROFILING , int thread_idx @@ -1423,7 +1422,7 @@ inline AsyncPollResult AsyncWaitList::poll_and_complete( // ============================================================================= #if SIMPLER_SCHED_PROFILING -struct PTO2SchedProfilingData { +struct SchedProfilingData { // Sub-phase cycle breakdown within on_task_complete uint64_t lock_cycle; // lock_fanout + state store + unlock uint64_t fanout_cycle; // fanout traversal @@ -1449,5 +1448,5 @@ struct PTO2SchedProfilingData { * Get and reset scheduler profiling data for a specific thread. * Returns accumulated profiling data and resets counters. */ -PTO2SchedProfilingData scheduler_get_profiling(int thread_idx); +SchedProfilingData scheduler_get_profiling(int thread_idx); #endif diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 0f2f554882..e9303f4b71 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -512,7 +512,7 @@ void SchedulerContext::log_chip_swimlane_summary(int32_t thread_idx, [[maybe_unu if (sched_total == 0) sched_total = 1; { - PTO2SchedProfilingData sp = scheduler_get_profiling(thread_idx); + SchedProfilingData sp = scheduler_get_profiling(thread_idx); uint64_t otc_total = sp.lock_cycle + sp.fanout_cycle + sp.fanin_cycle + sp.self_consumed_cycle; uint64_t complete_poll = (chip_swimlane.sched_complete_cycle > otc_total + chip_swimlane.sched_complete_perf_cycle) ? diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index 7176b1cb9f..af4c5f3178 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -617,7 +617,7 @@ void SchedulerContext::handle_drain_mode( bool coordinator = thread_idx == 0; ChipTaskSlotState *slot_state = drain_state_.pending_task.load(std::memory_order_acquire); - bool gated = slot_state->payload != nullptr && PTO2SchedulerState::owns_early_sync_drain(*slot_state->payload); + bool gated = slot_state->payload != nullptr && SchedulerState::owns_early_sync_drain(*slot_state->payload); if (coordinator) { ResourceShape shape = slot_state->active_mask.to_shape(); @@ -680,5 +680,5 @@ void SchedulerContext::handle_drain_mode( } else { sched_->propagate_dispatch_fanin(*slot_state); } - PTO2SchedulerState::finish_early_sync_drain(*slot_state->payload); + SchedulerState::finish_early_sync_drain(*slot_state->payload); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index a5f80f8b55..496a11c920 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -142,7 +142,7 @@ class SchedulerContext { // ========================================================================= // --- Scheduler binding & per-core runtime state --- - alignas(64) PTO2SchedulerState *sched_{nullptr}; + alignas(64) SchedulerState *sched_{nullptr}; RuntimeContext *rt_{nullptr}; // Per-core execution state, indexed by core_id (= worker_id) diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 6962605b6c..a475c6b99e 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -120,7 +120,7 @@ void SchedulerContext::build_payload( const CoreCallable *callable = reinterpret_cast(callable_addr); dispatch_payload.function_bin_addr = callable->resolved_addr(); auto &payload = *slot_state.payload; - if (PTO2SchedulerState::should_gate_early_dispatch( + if (SchedulerState::should_gate_early_dispatch( force_gate, payload.early_dispatch_state.load(std::memory_order_relaxed) )) { dispatch_payload.src_payload = reinterpret_cast(&payload); @@ -637,12 +637,12 @@ int32_t SchedulerContext::stage_consumer_blocks( for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_cores[w] != 0) { owned[w] = - PTO2SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); + SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); } } for (int i = 0; i < n; i++) { int32_t cid = tracker.get_core_id_by_offset(handles[i].core_offset); - PTO2SchedulerState::ring_claimed_local_doorbell( + SchedulerState::ring_claimed_local_doorbell( owned[cid >> 6], cid, handles[i].reg_addr, handles[i].reg_task_id ); } @@ -785,7 +785,7 @@ int32_t SchedulerContext::try_early_dispatch( if (ChipTaskSlotState *c = sched_->early_sync_start_queue.pop_tagged(&sync_task_id_snapshot)) { bool current_sync_task = static_cast(c->task->task_id.raw) == sync_task_id_snapshot && c->task_attrs.requires_sync_start(); - if (current_sync_task && PTO2SchedulerState::try_claim_early_sync_drain(*c->payload)) { + if (current_sync_task && SchedulerState::try_claim_early_sync_drain(*c->payload)) { if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_STAGING) { sched_->cancel_early_sync_drain(*c); } else if (drain_state_.sync_start_pending.load(std::memory_order_acquire) == 0 && @@ -797,7 +797,7 @@ int32_t SchedulerContext::try_early_dispatch( // payload becomes visible. Once staging starts, the local // capacity invariant guarantees completion; never fall back // after publishing a partial cohort. - PTO2SchedulerState::mark_early_sync_drain_armed(*c->payload); + SchedulerState::mark_early_sync_drain_armed(*c->payload); always_assert(c->next_block_idx.load(std::memory_order_seq_cst) == 0); SyncStartStageResult staged = stage_sync_start_cores( c, c->logical_block_num, thread_idx, /*gated=*/true, /*record_drain_phases=*/false @@ -807,10 +807,10 @@ int32_t SchedulerContext::try_early_dispatch( static_cast(staged.running_cores), std::memory_order_seq_cst ); sched_->retry_sync_start_rendezvous_after_staging(*c); - PTO2SchedulerState::finish_early_sync_drain(*c->payload); + SchedulerState::finish_early_sync_drain(*c->payload); total_staged += staged.staged_blocks; } else if (enter_drain_mode(c, c->logical_block_num)) { - PTO2SchedulerState::mark_early_sync_drain_armed(*c->payload); + SchedulerState::mark_early_sync_drain_armed(*c->payload); } else { sched_->cancel_early_sync_drain(*c); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index cacdf0b61d..afb7c44ec0 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -85,7 +85,7 @@ void ready_queue_destroy(PTO2ReadyQueue *queue) { // Scheduler // ============================================================================= -bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base, int32_t ring_id) { +bool SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base, int32_t ring_id) { // ring stores the device address of the SM ring header — pure offset // arithmetic, no SM load. ring = sm_layout::ring_header_addr(sm_dev_base, ring_id); @@ -106,7 +106,7 @@ bool PTO2SchedulerState::RingSchedState::init_data_from_layout(void *sm_dev_base return true; } -void PTO2SchedulerState::RingSchedState::reset_for_reuse( +void SchedulerState::RingSchedState::reset_for_reuse( void *sm_dev_base, int32_t ring_id, std::atomic *orch_err ) { ring = sm_layout::ring_header_addr(sm_dev_base, ring_id); @@ -121,12 +121,12 @@ void PTO2SchedulerState::RingSchedState::reset_for_reuse( #endif } -void PTO2SchedulerState::RingSchedState::destroy() { +void SchedulerState::RingSchedState::destroy() { publication_batching_enabled = false; ring = nullptr; } -PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity) { +SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity) { int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { dep_pool_capacities[r] = dep_pool_capacity; @@ -134,9 +134,9 @@ PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena, int32 return reserve_layout(arena, dep_pool_capacities); } -PTO2SchedulerLayout -PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]) { - PTO2SchedulerLayout layout{}; +SchedulerLayout +SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]) { + SchedulerLayout layout{}; layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { layout.dep_pool_capacities[r] = dep_pool_capacities[r]; @@ -162,10 +162,8 @@ PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_ca return layout; } -bool PTO2SchedulerState::init_data_from_layout( - const PTO2SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base -) { - PTO2SchedulerState *sched = this; +bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, DeviceArena &arena, void *sm_dev_base) { + SchedulerState *sched = this; sched->sm_header = reinterpret_cast(sm_dev_base); sched->advance_pending_mask.store(0, std::memory_order_relaxed); sched->publication_request_mask.store(0, std::memory_order_relaxed); @@ -228,8 +226,8 @@ bool PTO2SchedulerState::init_data_from_layout( return true; } -void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void *sm_dev_base) { - PTO2SchedulerState *sched = this; +void SchedulerState::reset_for_reuse(const SchedulerLayout &layout, void *sm_dev_base) { + SchedulerState *sched = this; sched->sm_header = reinterpret_cast(sm_dev_base); sched->advance_pending_mask.store(0, std::memory_order_relaxed); sched->publication_request_mask.store(0, std::memory_order_relaxed); @@ -260,8 +258,8 @@ void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void (void)layout; } -void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena) { - PTO2SchedulerState *sched = this; +void SchedulerState::wire_arena_pointers(const SchedulerLayout &layout, DeviceArena &arena) { + SchedulerState *sched = this; for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { ready_queue_wire_arena_pointers(&sched->ready_queues[i], arena, layout.off_ready_queue_slots[i]); } @@ -284,8 +282,8 @@ void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, } } -void PTO2SchedulerState::destroy() { - PTO2SchedulerState *sched = this; +void SchedulerState::destroy() { + SchedulerState *sched = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { sched->ring_sched_states[r].destroy(); sched->ring_sched_states[r].dep_pool.base = nullptr; @@ -499,7 +497,7 @@ bool PTO2OrchestratorState::reset_for_reuse( } void PTO2OrchestratorState::wire_arena_pointers( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, PTO2SchedulerState *scheduler_arg + const PTO2OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler_arg ) { auto *orch = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -523,7 +521,7 @@ void PTO2OrchestratorState::destroy() { orch->scope_begins = nullptr; } -void PTO2OrchestratorState::set_scheduler(PTO2SchedulerState *scheduler_arg) { +void PTO2OrchestratorState::set_scheduler(SchedulerState *scheduler_arg) { scheduler = scheduler_arg; if (scheduler == nullptr) return; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -570,7 +568,7 @@ RuntimeArenaLayout runtime_reserve_layout( task_window_sizes_i32[r] = static_cast(task_window_sizes[r]); } layout.offsets.orch = PTO2OrchestratorState::reserve_layout(arena, task_window_sizes_i32, dep_pool_capacities); - layout.offsets.sched = PTO2SchedulerState::reserve_layout(arena, dep_pool_capacities); + layout.offsets.sched = SchedulerState::reserve_layout(arena, dep_pool_capacities); layout.offsets.off_runtime = arena.reserve(sizeof(RuntimeContext), PTO2_ALIGN_SIZE); layout.offsets.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h index 40c4c80907..5a49a0b717 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h @@ -114,7 +114,7 @@ enum class ResourceShape : uint8_t { // Number of *dispatchable* resource shapes (AIC, AIV, MIX). DUMMY does not // allocate a per-shape ready_queue entry / local buffer — it lives in a -// dedicated queue inside PTO2SchedulerState. +// dedicated queue inside SchedulerState. inline constexpr int32_t NUM_RESOURCE_SHAPES = 3; /** diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 3e835252ef..109139b4a7 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -1004,7 +1004,7 @@ add_a5_hbg_runtime_test(test_a5_graph_cache common/test_hbg_graph_cache.cpp) target_sources(test_a5_graph_cache PRIVATE ${CMAKE_SOURCE_DIR}/../../../src/common/host_build_graph/graph_execution.cpp ) -# Incremental-activation tests init a real PTO2SchedulerState (ready queues), so +# Incremental-activation tests init a real SchedulerState (ready queues), so # they link the scheduler + shared runtime out-of-line members. add_a2a3_hbg_runtime_test(test_graph_activation a2a3/test_graph_activation.cpp) target_sources(test_graph_activation PRIVATE diff --git a/tests/ut/cpp/a2a3/test_graph_activation.cpp b/tests/ut/cpp/a2a3/test_graph_activation.cpp index 6d33bed597..9937218ad5 100644 --- a/tests/ut/cpp/a2a3/test_graph_activation.cpp +++ b/tests/ut/cpp/a2a3/test_graph_activation.cpp @@ -34,13 +34,13 @@ class GraphActivationTest : public ::testing::Test { DeviceArena sm_arena; DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; - PTO2SchedulerState sched{}; - PTO2SchedulerLayout sched_layout{}; + SchedulerState sched{}; + SchedulerLayout sched_layout{}; void SetUp() override { sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); - sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); + sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); sched.wire_arena_pointers(sched_layout, runtime_arena); diff --git a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp index 7039a6b3bc..a6af42ee8f 100644 --- a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp @@ -48,8 +48,8 @@ class HbgSubmitPoisonTest : public ::testing::Test { DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; - PTO2SchedulerState sched{}; - PTO2SchedulerLayout sched_layout{}; + SchedulerState sched{}; + SchedulerLayout sched_layout{}; std::vector gm_heap; void SetUp() override { @@ -57,7 +57,7 @@ class HbgSubmitPoisonTest : public ::testing::Test { ASSERT_NE(sm_handle, nullptr); gm_heap.resize(4096); - sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); + sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); diff --git a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp index 085ee024ff..30de4f6de5 100644 --- a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp @@ -25,9 +25,9 @@ class OrchestratorFaninTest : public ::testing::Test { DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; - PTO2SchedulerState sched{}; + SchedulerState sched{}; PTO2OrchestratorLayout orch_layout{}; - PTO2SchedulerLayout sched_layout{}; + SchedulerLayout sched_layout{}; std::vector gm_heap; void SetUp() override { @@ -41,7 +41,7 @@ class OrchestratorFaninTest : public ::testing::Test { } orch_layout = PTO2OrchestratorState::reserve_layout(runtime_arena, task_window_sizes); - sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); + sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(orch.init_data_from_layout( diff --git a/tests/ut/cpp/a2a3/test_scheduler_state.cpp b/tests/ut/cpp/a2a3/test_scheduler_state.cpp index 357c537f4a..2f0a446c3b 100644 --- a/tests/ut/cpp/a2a3/test_scheduler_state.cpp +++ b/tests/ut/cpp/a2a3/test_scheduler_state.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2SchedulerState from scheduler.h + * Unit tests for SchedulerState from scheduler.h * * Tests task state transitions, fanin/fanout logic, subtask completion. */ @@ -53,7 +53,7 @@ TEST(SyncStartDrainAttemptTest, LateAckCannotSatisfyNextAttemptBarrier) { class SchedulerStateTest : public ::testing::Test { protected: - PTO2SchedulerState sched; + SchedulerState sched; SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -68,7 +68,7 @@ class SchedulerStateTest : public ::testing::Test { void SetUp() override { sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); - auto layout = PTO2SchedulerState::reserve_layout(sched_arena); + auto layout = SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(layout, sched_arena, sm_handle->header)); sched.wire_arena_pointers(layout, sched_arena); @@ -130,7 +130,7 @@ class SchedulerStateTest : public ::testing::Test { void setup_ring_for_reclaim_race(int32_t ring_id, int32_t current_task_index, int32_t blocked_task_id) { SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ring.fc.current_task_index.store(current_task_index, std::memory_order_release); ring.fc.last_task_alive.store(0, std::memory_order_release); @@ -146,7 +146,7 @@ class SchedulerStateTest : public ::testing::Test { void setup_contended_head_case(int32_t ring_id, int32_t head_task_id) { SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ring.fc.current_task_index.store(head_task_id + 2, std::memory_order_release); ring.fc.last_task_alive.store(head_task_id, std::memory_order_release); ring_sched.last_task_alive = head_task_id; @@ -185,9 +185,9 @@ TEST_F(SchedulerStateTest, ConsumedHeadAdvancesAfterContendedAdvanceLock) { setup_ring_for_reclaim_race(ring_id, /*current_task_index=*/1, head_task_id); SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); - uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); + uint32_t pending_bit = SchedulerState::ring_advance_pending_bit(ring_id); ring_sched.advance_lock.store(1, std::memory_order_release); std::thread unlocker([&]() { @@ -231,9 +231,9 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadSetsPendingAndIdleDrainAdvances) setup_contended_head_case(ring_id, head_task_id); SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); - uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); + uint32_t pending_bit = SchedulerState::ring_advance_pending_bit(ring_id); ring_sched.advance_lock.store(1, std::memory_order_release); sched.check_and_handle_consumed(head); @@ -262,7 +262,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadIdleDrainStress) { setup_contended_head_case(ring_id, head_task_id); SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); ring_sched.advance_lock.store(1, std::memory_order_release); diff --git a/tests/ut/cpp/a2a3/test_task_state.cpp b/tests/ut/cpp/a2a3/test_task_state.cpp index 23caa3181b..587e89e5b6 100644 --- a/tests/ut/cpp/a2a3/test_task_state.cpp +++ b/tests/ut/cpp/a2a3/test_task_state.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for ChipTaskSlotState lifecycle through PTO2SchedulerState API. + * Unit tests for ChipTaskSlotState lifecycle through SchedulerState API. * * These tests drive state transitions via src methods (release_fanin, * on_subtask_complete, check_and_handle_consumed) rather than manually @@ -33,7 +33,7 @@ class TaskStateTest : public ::testing::Test { protected: - PTO2SchedulerState sched; + SchedulerState sched; SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -48,7 +48,7 @@ class TaskStateTest : public ::testing::Test { void SetUp() override { sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); - auto layout = PTO2SchedulerState::reserve_layout(sched_arena); + auto layout = SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(layout, sched_arena, sm_handle->header)); sched.wire_arena_pointers(layout, sched_arena); diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index 0f89a0bc5d..de09175de7 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -44,7 +44,7 @@ uint64_t get_test_reg_stub_base_addr(); class WiringTest : public ::testing::Test { protected: PTO2OrchestratorState orch{}; - PTO2SchedulerState sched{}; + SchedulerState sched{}; SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -60,7 +60,7 @@ class WiringTest : public ::testing::Test { void SetUp() override { sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); - auto layout = PTO2SchedulerState::reserve_layout(sched_arena); + auto layout = SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(layout, sched_arena, sm_handle->header)); sched.wire_arena_pointers(layout, sched_arena); @@ -633,14 +633,14 @@ TEST_F(WiringTest, EarlyDispatchDoorbellBitsHaveOneOwner) { constexpr uint64_t late_bits = 0b1010; std::atomic release_first{all_bits}; - uint64_t release_owned = PTO2SchedulerState::claim_all_staged_doorbell_bits(release_first); - uint64_t late_owned = PTO2SchedulerState::claim_late_staged_doorbell_bits(release_first, late_bits); + uint64_t release_owned = SchedulerState::claim_all_staged_doorbell_bits(release_first); + uint64_t late_owned = SchedulerState::claim_late_staged_doorbell_bits(release_first, late_bits); EXPECT_EQ(release_owned, all_bits); EXPECT_EQ(late_owned, 0); std::atomic late_first{all_bits}; - late_owned = PTO2SchedulerState::claim_late_staged_doorbell_bits(late_first, late_bits); - release_owned = PTO2SchedulerState::claim_all_staged_doorbell_bits(late_first); + late_owned = SchedulerState::claim_late_staged_doorbell_bits(late_first, late_bits); + release_owned = SchedulerState::claim_all_staged_doorbell_bits(late_first); EXPECT_EQ(late_owned, late_bits); EXPECT_EQ(release_owned, all_bits & ~late_bits); EXPECT_EQ(release_owned & late_owned, 0); @@ -648,18 +648,18 @@ TEST_F(WiringTest, EarlyDispatchDoorbellBitsHaveOneOwner) { EXPECT_EQ(late_first.load(std::memory_order_acquire), 0); std::atomic published_after_release{0}; - release_owned = PTO2SchedulerState::claim_all_staged_doorbell_bits(published_after_release); + release_owned = SchedulerState::claim_all_staged_doorbell_bits(published_after_release); published_after_release.fetch_or(late_bits, std::memory_order_seq_cst); - late_owned = PTO2SchedulerState::claim_late_staged_doorbell_bits(published_after_release, late_bits); + late_owned = SchedulerState::claim_late_staged_doorbell_bits(published_after_release, late_bits); EXPECT_EQ(release_owned, 0); EXPECT_EQ(late_owned, late_bits); EXPECT_EQ(published_after_release.load(std::memory_order_acquire), 0); } TEST_F(WiringTest, EarlyDispatchClaimStaysGatedAfterRelease) { - EXPECT_TRUE(PTO2SchedulerState::should_gate_early_dispatch(true, PTO2_EARLY_DISPATCH_DISPATCHED)); - EXPECT_TRUE(PTO2SchedulerState::should_gate_early_dispatch(false, PTO2_EARLY_DISPATCH_STAGING)); - EXPECT_FALSE(PTO2SchedulerState::should_gate_early_dispatch(false, PTO2_EARLY_DISPATCH_DISPATCHED)); + EXPECT_TRUE(SchedulerState::should_gate_early_dispatch(true, PTO2_EARLY_DISPATCH_DISPATCHED)); + EXPECT_TRUE(SchedulerState::should_gate_early_dispatch(false, PTO2_EARLY_DISPATCH_STAGING)); + EXPECT_FALSE(SchedulerState::should_gate_early_dispatch(false, PTO2_EARLY_DISPATCH_DISPATCHED)); } TEST_F(WiringTest, EarlyDispatchLaunchHasSingleOwner) { @@ -671,7 +671,7 @@ TEST_F(WiringTest, EarlyDispatchLaunchHasSingleOwner) { for (int i = 0; i < 2; i++) { contenders[i] = std::thread([&, i] { while (!start.load(std::memory_order_acquire)) {} - won[i] = PTO2SchedulerState::try_claim_early_dispatch_launch(payload); + won[i] = SchedulerState::try_claim_early_dispatch_launch(payload); }); } start.store(true, std::memory_order_release); @@ -724,7 +724,7 @@ TEST_F(WiringTest, LateStagerRingsCapturedDoorbellAfterTableReuse) { sched.early_dispatch_doorbell_table[core_id].token = reused_token; uint64_t claimed = 1ULL << core_id; - EXPECT_TRUE(PTO2SchedulerState::ring_claimed_local_doorbell(claimed, core_id, captured_addr, captured_token)); + EXPECT_TRUE(SchedulerState::ring_claimed_local_doorbell(claimed, core_id, captured_addr, captured_token)); EXPECT_EQ(get_test_reg_stub_base_addr(), captured_addr); EXPECT_EQ(get_test_reg_stub_value(), (static_cast(captured_token) << 32) | captured_token); } @@ -779,11 +779,11 @@ TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { std::thread first([&] { while (!start.load(std::memory_order_acquire)) {} - first_won = PTO2SchedulerState::try_claim_early_dispatch_launch(payload); + first_won = SchedulerState::try_claim_early_dispatch_launch(payload); }); std::thread second([&] { while (!start.load(std::memory_order_acquire)) {} - second_won = PTO2SchedulerState::try_claim_early_dispatch_launch(payload); + second_won = SchedulerState::try_claim_early_dispatch_launch(payload); }); start.store(true, std::memory_order_release); @@ -873,8 +873,8 @@ TEST_F(WiringTest, ArmedEarlySyncDrainOwnsFinalReadyRoute) { sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); - ASSERT_TRUE(PTO2SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); - PTO2SchedulerState::mark_early_sync_drain_armed(*sync_consumer.payload); + ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); + SchedulerState::mark_early_sync_drain_armed(*sync_consumer.payload); EXPECT_TRUE(sched.release_fanin_and_check_ready(sync_consumer)); auto shape = static_cast(sync_consumer.active_mask.to_shape()); EXPECT_EQ(sched.ready_sync_queues[shape].pop(), nullptr); @@ -889,8 +889,8 @@ TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { alignas(64) TaskPayload payload{}; payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); - ASSERT_TRUE(PTO2SchedulerState::try_claim_early_sync_drain(payload)); - PTO2SchedulerState::mark_early_sync_drain_armed(payload); + ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(payload)); + SchedulerState::mark_early_sync_drain_armed(payload); std::atomic start{false}; std::atomic before_ready{0}; @@ -903,8 +903,8 @@ TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { if (i >= 2) { while (!ready_published.load(std::memory_order_acquire)) {} } - bool force_gate = PTO2SchedulerState::owns_early_sync_drain(payload); - gated[i] = PTO2SchedulerState::should_gate_early_dispatch( + bool force_gate = SchedulerState::owns_early_sync_drain(payload); + gated[i] = SchedulerState::should_gate_early_dispatch( force_gate, payload.early_dispatch_state.load(std::memory_order_relaxed) ); if (i < 2) before_ready.fetch_add(1, std::memory_order_release); @@ -913,7 +913,7 @@ TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { start.store(true, std::memory_order_release); while (before_ready.load(std::memory_order_acquire) != 2) {} payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst); - bool ready_has_owner = PTO2SchedulerState::publish_ready_to_early_sync_drain(payload); + bool ready_has_owner = SchedulerState::publish_ready_to_early_sync_drain(payload); ready_published.store(true, std::memory_order_release); for (auto &stager : stagers) stager.join(); @@ -924,11 +924,11 @@ TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { alignas(64) TaskPayload normal_ready{}; normal_ready.early_dispatch_state.store(PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_relaxed); - EXPECT_FALSE(PTO2SchedulerState::publish_ready_to_early_sync_drain(normal_ready)); - EXPECT_FALSE(PTO2SchedulerState::owns_early_sync_drain(normal_ready)); + EXPECT_FALSE(SchedulerState::publish_ready_to_early_sync_drain(normal_ready)); + EXPECT_FALSE(SchedulerState::owns_early_sync_drain(normal_ready)); EXPECT_FALSE( - PTO2SchedulerState::should_gate_early_dispatch( - PTO2SchedulerState::owns_early_sync_drain(normal_ready), + SchedulerState::should_gate_early_dispatch( + SchedulerState::owns_early_sync_drain(normal_ready), normal_ready.early_dispatch_state.load(std::memory_order_relaxed) ) ); @@ -942,15 +942,15 @@ TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); - ASSERT_TRUE(PTO2SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); - PTO2SchedulerState::mark_early_sync_drain_armed(*sync_consumer.payload); + ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); + SchedulerState::mark_early_sync_drain_armed(*sync_consumer.payload); // READY publication may lag the STAGING-to-DISPATCHED release transition. EXPECT_FALSE(sched.try_early_dispatch_release(sync_consumer)); sync_consumer.next_block_idx.store(sync_consumer.logical_block_num, std::memory_order_seq_cst); - PTO2SchedulerState::finish_early_sync_drain(*sync_consumer.payload); + SchedulerState::finish_early_sync_drain(*sync_consumer.payload); - EXPECT_TRUE(PTO2SchedulerState::publish_ready_to_early_sync_drain(*sync_consumer.payload)); + EXPECT_TRUE(SchedulerState::publish_ready_to_early_sync_drain(*sync_consumer.payload)); EXPECT_EQ( sync_consumer.payload->early_sync_drain_state.load(), PTO2_EARLY_SYNC_DRAIN_OWNER | PTO2_EARLY_SYNC_DRAIN_ARMED | PTO2_EARLY_SYNC_DRAIN_READY | @@ -966,7 +966,7 @@ TEST_F(WiringTest, CancelledEarlySyncDrainRoutesProducerRelease) { sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); - ASSERT_TRUE(PTO2SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); + ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); sched.cancel_early_sync_drain(sync_consumer); EXPECT_TRUE(sched.release_fanin_and_check_ready(sync_consumer)); @@ -984,7 +984,7 @@ TEST_F(WiringTest, ProducerReleaseTransfersReadyRouteToCancellingDrain) { sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); - ASSERT_TRUE(PTO2SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); + ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); EXPECT_TRUE(sched.release_fanin_and_check_ready(sync_consumer)); auto shape = static_cast(sync_consumer.active_mask.to_shape()); EXPECT_EQ(sched.ready_sync_queues[shape].pop(), nullptr); diff --git a/tests/ut/cpp/a5/test_graph_activation.cpp b/tests/ut/cpp/a5/test_graph_activation.cpp index 6d33bed597..9937218ad5 100644 --- a/tests/ut/cpp/a5/test_graph_activation.cpp +++ b/tests/ut/cpp/a5/test_graph_activation.cpp @@ -34,13 +34,13 @@ class GraphActivationTest : public ::testing::Test { DeviceArena sm_arena; DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; - PTO2SchedulerState sched{}; - PTO2SchedulerLayout sched_layout{}; + SchedulerState sched{}; + SchedulerLayout sched_layout{}; void SetUp() override { sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); - sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); + sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); sched.wire_arena_pointers(sched_layout, runtime_arena); diff --git a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp index 7039a6b3bc..a6af42ee8f 100644 --- a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp @@ -48,8 +48,8 @@ class HbgSubmitPoisonTest : public ::testing::Test { DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; - PTO2SchedulerState sched{}; - PTO2SchedulerLayout sched_layout{}; + SchedulerState sched{}; + SchedulerLayout sched_layout{}; std::vector gm_heap; void SetUp() override { @@ -57,7 +57,7 @@ class HbgSubmitPoisonTest : public ::testing::Test { ASSERT_NE(sm_handle, nullptr); gm_heap.resize(4096); - sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); + sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); diff --git a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp index a53a363fac..087791cdd8 100644 --- a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp @@ -25,9 +25,9 @@ class OrchestratorFaninTest : public ::testing::Test { DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; - PTO2SchedulerState sched{}; + SchedulerState sched{}; PTO2OrchestratorLayout orch_layout{}; - PTO2SchedulerLayout sched_layout{}; + SchedulerLayout sched_layout{}; std::vector gm_heap; void SetUp() override { @@ -41,7 +41,7 @@ class OrchestratorFaninTest : public ::testing::Test { } orch_layout = PTO2OrchestratorState::reserve_layout(runtime_arena, task_window_sizes); - sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); + sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(orch.init_data_from_layout( diff --git a/tests/ut/cpp/a5/test_scheduler_state.cpp b/tests/ut/cpp/a5/test_scheduler_state.cpp index a8be096e6d..ece7e6ba9a 100644 --- a/tests/ut/cpp/a5/test_scheduler_state.cpp +++ b/tests/ut/cpp/a5/test_scheduler_state.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2SchedulerState from scheduler.h + * Unit tests for SchedulerState from scheduler.h * * Tests task state transitions, fanin/fanout logic, subtask completion. */ @@ -53,7 +53,7 @@ TEST(SyncStartDrainAttemptTest, LateAckCannotSatisfyNextAttemptBarrier) { class SchedulerStateTest : public ::testing::Test { protected: - PTO2SchedulerState sched; + SchedulerState sched; SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -68,7 +68,7 @@ class SchedulerStateTest : public ::testing::Test { void SetUp() override { sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); - auto layout = PTO2SchedulerState::reserve_layout(sched_arena); + auto layout = SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(layout, sched_arena, sm_handle->header)); sched.wire_arena_pointers(layout, sched_arena); @@ -130,7 +130,7 @@ class SchedulerStateTest : public ::testing::Test { void setup_ring_for_reclaim_race(int32_t ring_id, int32_t current_task_index, int32_t blocked_task_id) { SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ring.fc.current_task_index.store(current_task_index, std::memory_order_release); ring.fc.last_task_alive.store(0, std::memory_order_release); @@ -146,7 +146,7 @@ class SchedulerStateTest : public ::testing::Test { void setup_contended_head_case(int32_t ring_id, int32_t head_task_id) { SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ring.fc.current_task_index.store(head_task_id + 2, std::memory_order_release); ring.fc.last_task_alive.store(head_task_id, std::memory_order_release); ring_sched.last_task_alive = head_task_id; @@ -164,7 +164,7 @@ class SchedulerStateTest : public ::testing::Test { TEST_F(SchedulerStateTest, UnvalidatedWiringPublishesEveryAdvance) { constexpr int32_t ring_id = 0; SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; EXPECT_FALSE(ring_sched.publication_batching_enabled); ring.fc.current_task_index.store(2, std::memory_order_release); @@ -201,9 +201,9 @@ TEST_F(SchedulerStateTest, ConsumedHeadAdvancesAfterContendedAdvanceLock) { setup_ring_for_reclaim_race(ring_id, /*current_task_index=*/1, head_task_id); SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); - uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); + uint32_t pending_bit = SchedulerState::ring_advance_pending_bit(ring_id); ring_sched.advance_lock.store(1, std::memory_order_release); std::thread unlocker([&]() { @@ -247,9 +247,9 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadSetsPendingAndIdleDrainAdvances) setup_contended_head_case(ring_id, head_task_id); SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); - uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); + uint32_t pending_bit = SchedulerState::ring_advance_pending_bit(ring_id); ring_sched.advance_lock.store(1, std::memory_order_release); sched.check_and_handle_consumed(head); @@ -275,9 +275,9 @@ TEST_F(SchedulerStateTest, DeferredAdvanceDoesNotAcknowledgePublication) { setup_contended_head_case(ring_id, head_task_id); SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); - uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); + uint32_t pending_bit = SchedulerState::ring_advance_pending_bit(ring_id); sched.publication_ack_mask.store(0, std::memory_order_release); ring_sched.advance_lock.store(1, std::memory_order_release); @@ -293,7 +293,7 @@ TEST_F(SchedulerStateTest, PublicationRequestDoesNotConsumeDeferredAdvance) { constexpr int32_t ring_id = CHIP_MAX_RING_DEPTH - 1; setup_ring_for_reclaim_race(ring_id, /*current_task_index=*/1, /*blocked_task_id=*/1); - uint32_t pending_bit = PTO2SchedulerState::ring_advance_pending_bit(ring_id); + uint32_t pending_bit = SchedulerState::ring_advance_pending_bit(ring_id); sched.advance_pending_mask.fetch_or(pending_bit, std::memory_order_release); sched.publication_request_mask.fetch_or(pending_bit, std::memory_order_release); @@ -315,7 +315,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadIdleDrainStress) { setup_contended_head_case(ring_id, head_task_id); SharedMemoryRingHeader &ring = sm_handle->header->rings[ring_id]; - PTO2SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; + SchedulerState::RingSchedState &ring_sched = sched.ring_sched_states[ring_id]; ChipTaskSlotState &head = ring.get_slot_state_by_task_id(head_task_id); ring_sched.advance_lock.store(1, std::memory_order_release); diff --git a/tests/ut/cpp/a5/test_task_state.cpp b/tests/ut/cpp/a5/test_task_state.cpp index 23caa3181b..587e89e5b6 100644 --- a/tests/ut/cpp/a5/test_task_state.cpp +++ b/tests/ut/cpp/a5/test_task_state.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for ChipTaskSlotState lifecycle through PTO2SchedulerState API. + * Unit tests for ChipTaskSlotState lifecycle through SchedulerState API. * * These tests drive state transitions via src methods (release_fanin, * on_subtask_complete, check_and_handle_consumed) rather than manually @@ -33,7 +33,7 @@ class TaskStateTest : public ::testing::Test { protected: - PTO2SchedulerState sched; + SchedulerState sched; SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -48,7 +48,7 @@ class TaskStateTest : public ::testing::Test { void SetUp() override { sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); - auto layout = PTO2SchedulerState::reserve_layout(sched_arena); + auto layout = SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(layout, sched_arena, sm_handle->header)); sched.wire_arena_pointers(layout, sched_arena); diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index 3299b24a46..83013bdb5f 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -40,7 +40,7 @@ class WiringTest : public ::testing::Test { protected: PTO2OrchestratorState orch{}; - PTO2SchedulerState sched{}; + SchedulerState sched{}; SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; DeviceArena sched_arena; @@ -56,7 +56,7 @@ class WiringTest : public ::testing::Test { void SetUp() override { sm_handle = SharedMemoryHandle::create_and_init_default(sm_arena); ASSERT_NE(sm_handle, nullptr); - auto layout = PTO2SchedulerState::reserve_layout(sched_arena); + auto layout = SchedulerState::reserve_layout(sched_arena); ASSERT_NE(sched_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(layout, sched_arena, sm_handle->header)); sched.wire_arena_pointers(layout, sched_arena); @@ -290,11 +290,11 @@ TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { std::thread first([&] { while (!start.load(std::memory_order_acquire)) {} - first_won = PTO2SchedulerState::try_claim_early_dispatch_launch(payload); + first_won = SchedulerState::try_claim_early_dispatch_launch(payload); }); std::thread second([&] { while (!start.load(std::memory_order_acquire)) {} - second_won = PTO2SchedulerState::try_claim_early_dispatch_launch(payload); + second_won = SchedulerState::try_claim_early_dispatch_launch(payload); }); start.store(true, std::memory_order_release); @@ -382,14 +382,14 @@ TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); - ASSERT_TRUE(PTO2SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); - PTO2SchedulerState::mark_early_sync_drain_armed(*sync_consumer.payload); + ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); + SchedulerState::mark_early_sync_drain_armed(*sync_consumer.payload); EXPECT_FALSE(sched.try_early_dispatch_release(sync_consumer)); sync_consumer.next_block_idx.store(sync_consumer.logical_block_num, std::memory_order_seq_cst); - PTO2SchedulerState::finish_early_sync_drain(*sync_consumer.payload); + SchedulerState::finish_early_sync_drain(*sync_consumer.payload); - EXPECT_TRUE(PTO2SchedulerState::publish_ready_to_early_sync_drain(*sync_consumer.payload)); + EXPECT_TRUE(SchedulerState::publish_ready_to_early_sync_drain(*sync_consumer.payload)); EXPECT_EQ( sync_consumer.payload->early_sync_drain_state.load(), PTO2_EARLY_SYNC_DRAIN_OWNER | PTO2_EARLY_SYNC_DRAIN_ARMED | PTO2_EARLY_SYNC_DRAIN_READY | diff --git a/tests/ut/cpp/common/test_hbg_graph_cache.cpp b/tests/ut/cpp/common/test_hbg_graph_cache.cpp index 59758e6bda..dc81ed9be9 100644 --- a/tests/ut/cpp/common/test_hbg_graph_cache.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_cache.cpp @@ -596,7 +596,7 @@ TEST(GraphExecutionActivationState, RetriesDoNotClearReadiness) { } TEST(GraphExecutionActivationState, ExternalReadyBeforePrepareActivatesAtMeet) { - PTO2SchedulerState scheduler{}; + SchedulerState scheduler{}; GraphExecution execution{}; ChipTaskSlotState outer_slot{}; outer_slot.task_kind = TaskKind::GRAPH; @@ -614,7 +614,7 @@ TEST(GraphExecutionActivationState, ExternalReadyBeforePrepareActivatesAtMeet) { } TEST(GraphExecutionActivationState, PrepareBeforeExternalReadyActivatesAtMeet) { - PTO2SchedulerState scheduler{}; + SchedulerState scheduler{}; GraphExecution execution{}; ChipTaskSlotState outer_slot{}; outer_slot.task_kind = TaskKind::GRAPH; @@ -636,7 +636,7 @@ TEST(GraphExecutionErrors, ReadyQueueOverflowHasTriageText) { TEST(GraphExecutionErrors, GraphReadyQueueOverflowIsReported) { SharedMemoryHeader header{}; - PTO2SchedulerState scheduler{}; + SchedulerState scheduler{}; scheduler.sm_header = &header; PTO2ReadyQueueSlot queue_slots[2]{}; queue_slots[0].sequence.store(0, std::memory_order_relaxed); @@ -660,7 +660,7 @@ TEST(GraphExecutionErrors, GraphReadyQueueOverflowIsReported) { TEST(GraphExecutionErrors, GraphPrepareQueueOverflowIsReported) { SharedMemoryHeader header{}; - PTO2SchedulerState scheduler{}; + SchedulerState scheduler{}; scheduler.sm_header = &header; PTO2ReadyQueueSlot queue_slots[2]{}; queue_slots[0].sequence.store(0, std::memory_order_relaxed); @@ -682,18 +682,18 @@ TEST(GraphExecutionErrors, GraphPrepareQueueOverflowIsReported) { } TEST(GraphExecutionErrors, InvalidNodeCompletionIsReported) { - PTO2SchedulerState scheduler{}; + SchedulerState scheduler{}; ChipTaskSlotState slot{}; slot.task_kind = TaskKind::GRAPH_NODE; - const PTO2SchedulerState::TaskCompletionOutcome outcome = scheduler.complete_task(slot); + const SchedulerState::TaskCompletionOutcome outcome = scheduler.complete_task(slot); EXPECT_EQ(outcome.error_code, SIMPLER_ERROR_INVALID_ARGS); EXPECT_EQ(outcome.stream_tasks_completed, 0); } TEST(GraphExecutionProgress, InternalNodeResolutionIsNotAHostCompletion) { - PTO2SchedulerState scheduler{}; + SchedulerState scheduler{}; GraphDefinition definition{}; GraphNodeStorage node{}; GraphExecution execution{}; diff --git a/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp b/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp index 909d3a3973..ceb6fdfe5a 100644 --- a/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp @@ -43,8 +43,8 @@ class HbgGraphDefinitionArenaTest : public ::testing::Test { DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; - PTO2SchedulerState sched{}; - PTO2SchedulerLayout sched_layout{}; + SchedulerState sched{}; + SchedulerLayout sched_layout{}; GraphHostStatePtr graph_state; std::vector gm_heap; std::vector staging; @@ -57,7 +57,7 @@ class HbgGraphDefinitionArenaTest : public ::testing::Test { ASSERT_NE(sm_handle, nullptr); gm_heap.resize(HEAP_BYTES); - sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); + sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); diff --git a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp index 328ca26043..f0566841e0 100644 --- a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp @@ -35,8 +35,8 @@ class HbgGraphSubmitFailureTest : public ::testing::Test { DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; - PTO2SchedulerState sched{}; - PTO2SchedulerLayout sched_layout{}; + SchedulerState sched{}; + SchedulerLayout sched_layout{}; GraphHostStatePtr graph_state; std::vector gm_heap; // The Definition objects are built in here, as a bind's retained staging. @@ -66,7 +66,7 @@ class HbgGraphSubmitFailureTest : public ::testing::Test { ASSERT_NE(sm_handle, nullptr); gm_heap.resize(HEAP_BYTES); - sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); + sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); diff --git a/tests/ut/cpp/common/test_hbg_slot_claim.cpp b/tests/ut/cpp/common/test_hbg_slot_claim.cpp index e2d19d5380..e29e6268f7 100644 --- a/tests/ut/cpp/common/test_hbg_slot_claim.cpp +++ b/tests/ut/cpp/common/test_hbg_slot_claim.cpp @@ -34,8 +34,8 @@ class HbgSlotClaimTest : public ::testing::Test { DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; PTO2OrchestratorState orch{}; - PTO2SchedulerState sched{}; - PTO2SchedulerLayout sched_layout{}; + SchedulerState sched{}; + SchedulerLayout sched_layout{}; GraphHostStatePtr graph_state; std::vector gm_heap; // Stands in for a bind's retained staging: the Definition objects are built @@ -50,7 +50,7 @@ class HbgSlotClaimTest : public ::testing::Test { ASSERT_NE(sm_handle, nullptr); gm_heap.resize(HEAP_BYTES); - sched_layout = PTO2SchedulerState::reserve_layout(runtime_arena); + sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); diff --git a/tests/ut/cpp/stubs/test_stubs.cpp b/tests/ut/cpp/stubs/test_stubs.cpp index f3be49e11d..2f964b76f5 100644 --- a/tests/ut/cpp/stubs/test_stubs.cpp +++ b/tests/ut/cpp/stubs/test_stubs.cpp @@ -110,7 +110,7 @@ void flush_range_impl(const void * /* addr */, size_t /* size */) {} // platform_regs.h stub (get_reg_ptr) // ============================================================================= -// PTO2SchedulerState::ring_one_doorbell (scheduler.h, speculative +// SchedulerState::ring_one_doorbell (scheduler.h, speculative // early-dispatch) is an inline that resolves a register id to its MMIO pointer // via get_reg_ptr and writes a 64-bit token through it. There is no MMIO on the // host UT runner; hand back writable static storage (8 bytes — the doorbell is a From 6577ae83e89e77964d8eca516d7cab39a3ab4816 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:18:54 -0700 Subject: [PATCH 05/22] Refactor: retire the PTO2 prefix on the orchestrator-state structs `PTO2OrchestratorState` -> `OrchestratorState`, `PTO2OrchestratorLayout` -> `OrchestratorLayout`, `PTO2OrchProfilingData` -> `OrchProfilingData`. All plain strips; 236 occurrences across 42 files. Co-Authored-By: Claude Opus 5 (1M context) --- docs/manual-scope.md | 2 +- .../host_build_graph/docs/RUNTIME_LOGIC.md | 2 +- .../host_build_graph/host/runtime_maker.cpp | 10 +-- .../host_build_graph/runtime/orchestrator.h | 6 +- .../orchestrator_core/orchestrator.cpp | 66 +++++++++---------- .../orchestrator_core/runtime_core.cpp | 2 +- .../host_build_graph/runtime/runtime_core.h | 4 +- .../runtime/shared/runtime_init.cpp | 6 +- .../aicpu/aicpu_executor.cpp | 2 +- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 8 +-- .../runtime/orchestrator.cpp | 50 +++++++------- .../runtime/orchestrator.h | 22 +++---- .../runtime/runtime_core.cpp | 2 +- .../runtime/runtime_core.h | 4 +- .../runtime/shared/runtime_init.cpp | 32 ++++----- .../host_build_graph/docs/RUNTIME_LOGIC.md | 2 +- .../host_build_graph/host/runtime_maker.cpp | 10 +-- .../host_build_graph/runtime/orchestrator.h | 6 +- .../orchestrator_core/orchestrator.cpp | 66 +++++++++---------- .../orchestrator_core/runtime_core.cpp | 2 +- .../host_build_graph/runtime/runtime_core.h | 4 +- .../runtime/shared/runtime_init.cpp | 6 +- .../aicpu/aicpu_executor.cpp | 2 +- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 8 +-- .../runtime/orchestrator.cpp | 50 +++++++------- .../runtime/orchestrator.h | 22 +++---- .../runtime/runtime_core.cpp | 2 +- .../runtime/runtime_core.h | 4 +- .../runtime/shared/runtime_init.cpp | 32 ++++----- src/common/hierarchical/scope.h | 2 +- tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp | 2 +- tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp | 8 +-- tests/ut/cpp/a2a3/test_shared_memory.cpp | 2 +- tests/ut/cpp/a2a3/test_wiring.cpp | 2 +- tests/ut/cpp/a5/test_hbg_submit_poison.cpp | 2 +- tests/ut/cpp/a5/test_orchestrator_fanin.cpp | 8 +-- tests/ut/cpp/a5/test_shared_memory.cpp | 2 +- tests/ut/cpp/a5/test_wiring.cpp | 2 +- .../test_hbg_graph_definition_arena.cpp | 2 +- .../common/test_hbg_graph_submit_failure.cpp | 2 +- tests/ut/cpp/common/test_hbg_slot_claim.cpp | 2 +- 43 files changed, 235 insertions(+), 239 deletions(-) diff --git a/docs/manual-scope.md b/docs/manual-scope.md index 16ab969c5c..ba9f4a2997 100644 --- a/docs/manual-scope.md +++ b/docs/manual-scope.md @@ -93,7 +93,7 @@ Manual scope v0 is: `manual_begin_depth` is the only manual-scope-specific state, and it decides whether the current submit is in manual mode. It is explicitly initialized in -`PTO2OrchestratorState::init()` and reset in `mark_done()` so a reused orchestrator +`OrchestratorState::init()` and reset in `mark_done()` so a reused orchestrator starts cleanly on the next run. Whether a scope keeps a task list at all is runtime-specific, and manual mode does diff --git a/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md index 724a6763c6..7e4f1ac7be 100644 --- a/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md @@ -114,7 +114,7 @@ a boundary from which region happens to be reserved first — **Why the orchestrator is not in the arena at all.** hbg has no device-side orchestrator, so nothing on the device reads its state: not the `fanin_seen_epoch` table, not the scope arrays, not the TensorMap (~9.3 MB between them). It is -therefore a plain host object that owns those arrays — `PTO2OrchestratorState::init` +therefore a plain host object that owns those arrays — `OrchestratorState::init` allocates them — and `RuntimeContext` reaches it through a pointer that `bind` drops before the copied zone is uploaded, so no host address crosses the boundary. A `static_assert` keeps `RuntimeContext` trivially copyable, which is what forbids diff --git a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp index c937a90628..65ea3c00c1 100644 --- a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp @@ -534,13 +534,13 @@ bool bind_graph_definitions(const HostApi *api, GraphHostState &graph_state, Def } struct GraphHostStateBinding { - explicit GraphHostStateBinding(PTO2OrchestratorState &orchestrator, GraphHostState *state) : + explicit GraphHostStateBinding(OrchestratorState &orchestrator, GraphHostState *state) : orchestrator(orchestrator) { orchestrator.graph_host_state = state; } ~GraphHostStateBinding() { orchestrator.graph_host_state = nullptr; } - PTO2OrchestratorState &orchestrator; + OrchestratorState &orchestrator; }; int32_t run_host_orchestration( @@ -568,7 +568,7 @@ int32_t run_host_orchestration( // Re-point the orchestrator half at the host SM (scheduler keeps device SM). // Host-owned and destroyed with this frame, so rt->orchestrator is dropped on // every exit — it must never outlive the object it names. - PTO2OrchestratorState orchestrator; + OrchestratorState orchestrator; rt->orchestrator = &orchestrator; RAIIScopeGuard orchestrator_binding([rt]() { rt->orchestrator = nullptr; @@ -653,7 +653,7 @@ int32_t run_host_orchestration( // as spans rather than LOG_INFO because INFO is suppressed at the default log // level. Like the phase spans these are summed cost shares, not intervals. { - const PTO2OrchProfilingData prof = orchestrator_get_profiling(); + const OrchProfilingData prof = orchestrator_get_profiling(); const std::pair steps[] = { {"alloc", prof.alloc_cycle}, {"args", prof.args_cycle}, {"lookup", prof.lookup_cycle}, {"insert", prof.insert_cycle}, {"fanin", prof.fanin_cycle}, @@ -744,7 +744,7 @@ int32_t run_host_orchestration( // What this bind actually put in the pools. The orchestrator's cursors are the // exact populated extent of each one — no scan of the mirror is needed, and the // image ships that much rather than the worst case the mirror is dimensioned for. - const PTO2OrchestratorState &orch_state = orchestrator; + const OrchestratorState &orch_state = orchestrator; const sm_layout::BindUsage bind_usage{ nt, static_cast(orch_state.fanin_pool_cursor), diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h index 99827fde70..faa174a66b 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h @@ -56,7 +56,7 @@ struct GraphHostState; * holds it by pointer — a by-value member would put non-trivially-copyable state * inside the struct bind copies to the device. */ -struct PTO2OrchestratorState { +struct OrchestratorState { // === SHARED MEMORY ACCESS === SharedMemoryHeader *sm_header; @@ -165,7 +165,7 @@ struct PTO2OrchestratorState { // ============================================================================= #if SIMPLER_ORCH_PROFILING -struct PTO2OrchProfilingData { +struct OrchProfilingData { uint64_t alloc_cycle; // Combined task slot + heap allocation uint64_t args_cycle; uint64_t lookup_cycle; @@ -178,5 +178,5 @@ struct PTO2OrchProfilingData { uint64_t args_atomic_count; }; -PTO2OrchProfilingData orchestrator_get_profiling(); +OrchProfilingData orchestrator_get_profiling(); #endif diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index e229f0cd41..a5a0bede66 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -233,7 +233,7 @@ __attribute__((weak, visibility("hidden"))) uint64_t host_phase_now_ns() { retur } while (0) #endif -static int32_t orch_mark_fatal(PTO2OrchestratorState *orch, int32_t error_code) { +static int32_t orch_mark_fatal(OrchestratorState *orch, int32_t error_code) { always_assert(orch != nullptr); orch->fatal = true; if (error_code == SIMPLER_ERROR_NONE || orch->sm_header == nullptr) { @@ -249,7 +249,7 @@ static int32_t orch_mark_fatal(PTO2OrchestratorState *orch, int32_t error_code) } static void -orch_report_fatal_v(PTO2OrchestratorState *orch, int32_t error_code, const char *func, const char *fmt, va_list args) { +orch_report_fatal_v(OrchestratorState *orch, int32_t error_code, const char *func, const char *fmt, va_list args) { int32_t latched_code = orch_mark_fatal(orch, error_code); if (fmt == nullptr || fmt[0] == '\0') { @@ -270,7 +270,7 @@ orch_report_fatal_v(PTO2OrchestratorState *orch, int32_t error_code, const char unified_log_error(func, "FATAL(code=%d): %s", error_code, message.data()); } -void PTO2OrchestratorState::report_fatal(int32_t error_code, const char *func, const char *fmt, ...) { +void OrchestratorState::report_fatal(int32_t error_code, const char *func, const char *fmt, ...) { auto *orch = this; va_list args; va_start(args, fmt); @@ -558,7 +558,7 @@ struct GraphHostState { namespace { -GraphHostState *graph_state_from(PTO2OrchestratorState *orch) { +GraphHostState *graph_state_from(OrchestratorState *orch) { return orch == nullptr ? nullptr : static_cast(orch->graph_host_state); } @@ -571,7 +571,7 @@ thread_local GraphRecording *g_active_graph_recording = nullptr; // part of the thread-local identity rather than implied by it. thread_local GraphHostState *g_active_graph_owner = nullptr; -GraphRecording *active_graph_recording(PTO2OrchestratorState *orch) { +GraphRecording *active_graph_recording(OrchestratorState *orch) { GraphHostState *state = graph_state_from(orch); if (state == nullptr || state != g_active_graph_owner) return nullptr; return g_active_graph_recording; @@ -1121,7 +1121,7 @@ GraphHostDefinitionList graph_host_definitions(GraphHostState &state) { return list; } -static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { +static uint32_t next_fanin_seen_epoch(OrchestratorState *orch) { uint32_t next = orch->fanin_seen_current_epoch + 1; if (next == 0) { memset( @@ -1141,7 +1141,7 @@ static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { // (the consumer), used to bump each producer's last_consumer_local_id (the // reclaim gate the host wait_for_consumers polls via completed_watermark). struct PTO2FaninBuilder { - PTO2FaninBuilder(PTO2OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : + PTO2FaninBuilder(OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : count(0), orch(orch), seen_epoch(seen_epoch), @@ -1149,7 +1149,7 @@ struct PTO2FaninBuilder { payload(payload), slots(payload->fanin_data()) {} int32_t count{0}; - PTO2OrchestratorState *orch{nullptr}; + OrchestratorState *orch{nullptr}; uint32_t seen_epoch{0}; int32_t self_local{0}; TaskPayload *payload{nullptr}; @@ -1178,7 +1178,7 @@ struct PTO2FaninBuilder { }; static bool append_fanin_or_fail( - PTO2OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, + OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, TaskId producer_task_id, PTO2FaninBuilder *fanin_builder ) { // Skip a stale/reused producer slot: the cached owner id no longer resolves @@ -1239,7 +1239,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { } static bool prepare_task( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, + OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, TaskAttrs task_attrs, PreparedTask *out ) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit task outside a scope"); @@ -1339,7 +1339,7 @@ static bool prepare_task( // Scope Management // ============================================================================= -void PTO2OrchestratorState::begin_scope(PTO2ScopeMode mode) { +void OrchestratorState::begin_scope(PTO2ScopeMode mode) { auto *orch = this; if (orch->fatal) { return; @@ -1384,7 +1384,7 @@ void PTO2OrchestratorState::begin_scope(PTO2ScopeMode mode) { } } -void PTO2OrchestratorState::end_scope() { +void OrchestratorState::end_scope() { auto *orch = this; if (orch->fatal) { return; @@ -1420,7 +1420,7 @@ void PTO2OrchestratorState::end_scope() { // SIMPLER_ERROR_TENSORMAP_OVERFLOW and bail rather than letting new_entry()'s hard // assert fire mid-registration. Returns false when the pool is exhausted or a // fatal is already latched by another party. -static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t needed) { +static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { PTO2TensorMap &tm = orch->tensor_map; if (tm.free_entries() >= needed) { return true; @@ -1449,7 +1449,7 @@ static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t neede // computation (explicit_deps + auto), output registration, slot init, and // Orch-side wiring/ready publication. static TaskOutputTensors submit_task_common( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, + OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id ) { CYCLE_COUNT_START(); @@ -1751,7 +1751,7 @@ void graph_reset_outer_payload(TaskPayload &payload) { } bool graph_submit_outer( - PTO2OrchestratorState *orch, GraphHostState *state, uint64_t full_key, uint64_t definition_hash, int32_t owned_heap, + OrchestratorState *orch, GraphHostState *state, uint64_t full_key, uint64_t definition_hash, int32_t owned_heap, bool defer_heap, const GraphTaskArgs &args, TaskId *submitted_id ) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit Graph outside a scope"); @@ -1896,7 +1896,7 @@ bool graph_submit_outer( } bool graph_submit_definition( - PTO2OrchestratorState *orch, GraphHostState *state, const GraphDefinition *definition, const GraphTaskArgs &args, + OrchestratorState *orch, GraphHostState *state, const GraphDefinition *definition, const GraphTaskArgs &args, TaskId *submitted_id ) { if (definition == nullptr || !graph_boundary_matches(*definition, args) || @@ -1913,13 +1913,12 @@ bool graph_submit_definition( } bool graph_submit_pending_definition( - PTO2OrchestratorState *orch, GraphHostState *state, uint64_t full_key, const GraphTaskArgs &args, - TaskId *submitted_id + OrchestratorState *orch, GraphHostState *state, uint64_t full_key, const GraphTaskArgs &args, TaskId *submitted_id ) { return graph_submit_outer(orch, state, full_key, 0, 0, true, args, submitted_id); } -bool graph_finalize_pending_submissions(PTO2OrchestratorState *orch, GraphHostState *state, uint64_t *failed_key) { +bool graph_finalize_pending_submissions(OrchestratorState *orch, GraphHostState *state, uint64_t *failed_key) { for (GraphPendingUpload &pending : state->pending_uploads) { if (!pending.deferred_heap) continue; auto definition_it = state->definitions.find(pending.full_key); @@ -1963,7 +1962,7 @@ bool graph_finalize_pending_submissions(PTO2OrchestratorState *orch, GraphHostSt // recording.nodes keeps those addresses valid because the inner buffer is // transferred, not copied. TaskOutputTensors graph_record_submit_node( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, + OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id ) { ORCH_PHASE_START(); @@ -2222,8 +2221,7 @@ TaskOutputTensors graph_record_submit_node( } // namespace -GraphScopeResult -PTO2OrchestratorState::graph_begin(uint64_t graph_key, const GraphTaskArgs &args, uint64_t callable_hash) { +GraphScopeResult OrchestratorState::graph_begin(uint64_t graph_key, const GraphTaskArgs &args, uint64_t callable_hash) { ORCH_PHASE_START_SPANNING(); const GraphScopeResult result = graph_begin_inner(graph_key, args, callable_hash); ORCH_PHASE_END_SPANNING(HostPhaseKind::OrchGraphBegin, graph_key); @@ -2231,7 +2229,7 @@ PTO2OrchestratorState::graph_begin(uint64_t graph_key, const GraphTaskArgs &args } GraphScopeResult -PTO2OrchestratorState::graph_begin_inner(uint64_t graph_key, const GraphTaskArgs &args, uint64_t callable_hash) { +OrchestratorState::graph_begin_inner(uint64_t graph_key, const GraphTaskArgs &args, uint64_t callable_hash) { auto *orch = this; GraphScopeResult result; GraphHostState *state = graph_state_from(orch); @@ -2348,7 +2346,7 @@ PTO2OrchestratorState::graph_begin_inner(uint64_t graph_key, const GraphTaskArgs return result; } -bool PTO2OrchestratorState::graph_prepare(void *recording_handle, const GraphTaskArgs &args) { +bool OrchestratorState::graph_prepare(void *recording_handle, const GraphTaskArgs &args) { GraphHostState *state = graph_state_from(this); if (state == nullptr || recording_handle == nullptr || g_active_graph_recording != nullptr) return false; auto *entry = static_cast(recording_handle); @@ -2389,7 +2387,7 @@ bool PTO2OrchestratorState::graph_prepare(void *recording_handle, const GraphTas return true; } -void PTO2OrchestratorState::graph_abort(void *recording_handle) { +void OrchestratorState::graph_abort(void *recording_handle) { GraphHostState *state = graph_state_from(this); auto *entry = static_cast(recording_handle); if (state == nullptr || entry == nullptr) return; @@ -2411,7 +2409,7 @@ void PTO2OrchestratorState::graph_abort(void *recording_handle) { // Finish the background recording and publish the Definition. The main // thread finalizes the already-submitted outer Graph tasks in graph_commit. -bool PTO2OrchestratorState::graph_end() { +bool OrchestratorState::graph_end() { GraphHostState *state = graph_state_from(this); GraphRecording *recording = active_graph_recording(this); GraphInflightRecording *entry = g_active_graph_entry; @@ -2471,13 +2469,13 @@ bool PTO2OrchestratorState::graph_end() { // Join every recording in flight and back-patch all deferred shells in submit // order. Orchestration completion is the only normal-path barrier. -void PTO2OrchestratorState::graph_commit() { +void OrchestratorState::graph_commit() { ORCH_PHASE_START_SPANNING(); graph_commit_inner(); ORCH_PHASE_END_SPANNING(HostPhaseKind::OrchGraphCommit, 0); } -void PTO2OrchestratorState::graph_commit_inner() { +void OrchestratorState::graph_commit_inner() { if (active_graph_recording(this) != nullptr) return; GraphHostState *state = graph_state_from(this); if (state == nullptr || state->inflight_count.load(std::memory_order_acquire) == 0) return; @@ -2517,7 +2515,7 @@ void PTO2OrchestratorState::graph_commit_inner() { } } -TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args) { auto *orch = this; // Orchestration API should short-circuit after fatal, but keep this entry @@ -2609,7 +2607,7 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k // AICore dispatch. Empty active_mask routes the slot to the DUMMY ready // bucket; dispatch loop short-circuits to completion. Accepts the same Arg // shape as submit_task; scalars are permitted but never consumed. -TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::submit_dummy_task(const CoreTaskArgs &args) { auto *orch = this; if (orch->fatal) { @@ -2645,7 +2643,7 @@ TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const CoreTaskArgs &a ); } -TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { auto *orch = this; // Orchestration API should short-circuit after fatal, but keep this entry // robust as a no-op in case a caller reaches it directly. @@ -2775,7 +2773,7 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) // Flow Control // ============================================================================= -void PTO2OrchestratorState::mark_done() { +void OrchestratorState::mark_done() { auto *orch = this; int32_t total_tasks = orch->task_allocator.active_count(); if (total_tasks > 0) { @@ -2790,8 +2788,8 @@ void PTO2OrchestratorState::mark_done() { } #if SIMPLER_ORCH_PROFILING -PTO2OrchProfilingData orchestrator_get_profiling() { - PTO2OrchProfilingData d; +OrchProfilingData orchestrator_get_profiling() { + OrchProfilingData d; d.alloc_cycle = g_orch_alloc_cycle; d.args_cycle = g_orch_args_cycle; d.lookup_cycle = g_orch_lookup_cycle; diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 4dac7bd3cd..ab38c79b4d 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -168,7 +168,7 @@ MAYBE_UNINITIALIZED_BEGIN static bool wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_for_consumers, const char *caller) { TaskId owner = tensor.owner_task_id; - PTO2OrchestratorState &orch = *rt->orchestrator; + OrchestratorState &orch = *rt->orchestrator; // Segmented wait: collect up to kSegmentCap producer slots, then flush by // spinning on each. When the segment fills, we wait for the accumulated diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h index e719253d75..25a7863cd9 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h @@ -144,7 +144,7 @@ struct RuntimeArenaLayout { // adjacent on the device and travel as one copy. // // The orchestrator is NOT here. It runs on the host, owns its own scratch, and - // no device code reads any of it — see PTO2OrchestratorState. + // no device code reads any of it — see OrchestratorState. size_t off_copied_begin{0}; size_t off_copied_end{0}; @@ -175,7 +175,7 @@ struct RuntimeContext { // the orchestrator runs on the host and owns non-trivial scratch. Null on the // device — bind drops it before the copied zone is uploaded, so no device code // may dereference it. - PTO2OrchestratorState *orchestrator; + OrchestratorState *orchestrator; // Device-only zone: the scheduler state holds no per-run content, so it is // addressed through the arena rather than carried inside this header. SchedulerState *scheduler; diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp index e98c0b1cf2..5b87b39445 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -197,11 +197,11 @@ void SchedulerState::destroy() { // Orchestrator // ============================================================================= -bool PTO2OrchestratorState::init( +bool OrchestratorState::init( void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, SchedulerState *scheduler_arg ) { auto *orch = this; - *orch = PTO2OrchestratorState{}; + *orch = OrchestratorState{}; // A power-of-two window lets pto2_task_slot() mask instead of dividing. always_assert(task_window_size > 0 && (task_window_size & (task_window_size - 1)) == 0); @@ -243,7 +243,7 @@ bool PTO2OrchestratorState::init( return true; } -void PTO2OrchestratorState::set_scheduler(SchedulerState *scheduler) { this->scheduler = scheduler; } +void OrchestratorState::set_scheduler(SchedulerState *scheduler) { this->scheduler = scheduler; } // ============================================================================= // Top-level runtime arena diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index b9eb0cc4c8..e567347f76 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -737,7 +737,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { // Print orchestrator profiling data #if SIMPLER_ORCH_PROFILING - PTO2OrchProfilingData p = orchestrator_get_profiling(); + OrchProfilingData p = orchestrator_get_profiling(); uint64_t total = p.sync_cycle + p.alloc_cycle + p.args_cycle + p.lookup_cycle + p.insert_cycle + p.fanin_cycle; if (total == 0) total = 1; // avoid div-by-zero diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 6838dca013..75588a7a44 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -58,7 +58,7 @@ struct PTO2RingSet { }; ``` -### 4.2 PTO2OrchestratorState (modified) +### 4.2 OrchestratorState (modified) ```cpp // Before: single ring diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index b5bb017121..22de58bf83 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -335,7 +335,7 @@ This guarantees lookup only traverses valid entries — O(valid_entries_in_bucke **Layer 2 — Periodic Batch Cleanup** (`cleanup_retired`, per-task): -Every time the orchestrator submits a task (Step 0 of `PTO2OrchestratorState::submit_task`), it calls `PTO2TensorMap::sync_tensormap`. When `last_task_alive` has advanced by more than `PTO2_TENSORMAP_CLEANUP_INTERVAL` (default 64) tasks since the last cleanup, `PTO2TensorMap::cleanup_retired` runs: +Every time the orchestrator submits a task (Step 0 of `OrchestratorState::submit_task`), it calls `PTO2TensorMap::sync_tensormap`. When `last_task_alive` has advanced by more than `PTO2_TENSORMAP_CLEANUP_INTERVAL` (default 64) tasks since the last cleanup, `PTO2TensorMap::cleanup_retired` runs: This uses the **per-task entry chain** (`task_entry_head[task_slot]`) — each task's entries are doubly-linked together at insert time via `next_in_task`/`prev_in_task`. A slot's chain can hold more than one task's entries: a task at `local_id + N * window` reuses the slot and prepends to the chain already there, and cleanup can lag that reuse. Cleanup therefore walks the chain and frees only the entries whose `producer_task_id` matches the retiring task, unlinking each and leaving the rest linked — O(entries_in_slot), with no scan of the entire pool or all buckets. Freed entries are returned to `free_entry_list` for immediate reuse. @@ -357,7 +357,7 @@ In steady state, the number of valid TensorMap entries ≈ `active_tasks × avg_ ### 5.5 Dependency Discovery Flow -When `PTO2OrchestratorState::submit_task` processes parameters: +When `OrchestratorState::submit_task` processes parameters: 1. **INPUT/INOUT**: `PTO2TensorMap::lookup` searches for overlapping producers (with chain truncation) 2. For each producer found: `append_fanin_or_fail` adds the dependency @@ -415,7 +415,7 @@ In the scheduler's `task_state[]` array (`std::atomic`): ## 7. Orchestrator -### 7.1 PTO2OrchestratorState +### 7.1 OrchestratorState The orchestrator runs on AICPU Thread 3 and builds the task graph by calling the user-provided orchestration function. @@ -427,7 +427,7 @@ Key members: - `scheduler`: pointer to scheduler state (for Orch-side wiring helpers and ready queue access) - `gm_heap_base`, `gm_heap_size`: GM heap for output buffers -### 7.2 Task Submission Flow (`PTO2OrchestratorState::submit_task`) +### 7.2 Task Submission Flow (`OrchestratorState::submit_task`) | Step | Operation | | ---- | --------- | diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index cef22708a6..3e5d39019c 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -166,7 +166,7 @@ static uint32_t g_orch_submit_idx = 0; #define CYCLE_COUNT_ORCH_SUBMIT_RECORD(tid) #endif -static int32_t orch_mark_fatal(PTO2OrchestratorState *orch, int32_t error_code) { +static int32_t orch_mark_fatal(OrchestratorState *orch, int32_t error_code) { always_assert(orch != nullptr); orch->fatal = true; if (error_code == SIMPLER_ERROR_NONE || orch->sm_header == nullptr) { @@ -182,7 +182,7 @@ static int32_t orch_mark_fatal(PTO2OrchestratorState *orch, int32_t error_code) } static void -orch_report_fatal_v(PTO2OrchestratorState *orch, int32_t error_code, const char *func, const char *fmt, va_list args) { +orch_report_fatal_v(OrchestratorState *orch, int32_t error_code, const char *func, const char *fmt, va_list args) { int32_t latched_code = orch_mark_fatal(orch, error_code); #if SIMPLER_DFX @@ -211,7 +211,7 @@ orch_report_fatal_v(PTO2OrchestratorState *orch, int32_t error_code, const char unified_log_error(func, "FATAL(code=%d): %s", error_code, message); } -void PTO2OrchestratorState::report_fatal(int32_t error_code, const char *func, const char *fmt, ...) { +void OrchestratorState::report_fatal(int32_t error_code, const char *func, const char *fmt, ...) { auto *orch = this; va_list args; va_start(args, fmt); @@ -219,7 +219,7 @@ void PTO2OrchestratorState::report_fatal(int32_t error_code, const char *func, c va_end(args); } -static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { +static uint32_t next_fanin_seen_epoch(OrchestratorState *orch) { uint32_t next = orch->fanin_seen_current_epoch + 1; if (next == 0) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -235,7 +235,7 @@ static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { } struct PTO2FaninBuilder { - PTO2FaninBuilder(PTO2OrchestratorState *orch, PTO2FaninPool &spill_pool, uint32_t seen_epoch) : + PTO2FaninBuilder(OrchestratorState *orch, PTO2FaninPool &spill_pool, uint32_t seen_epoch) : count(0), wait_count(0), spill_start(0), @@ -245,7 +245,7 @@ struct PTO2FaninBuilder { int32_t count{0}; // total fanin edges (all flag combinations) int32_t wait_count{0}; // edges carrying DEP_WAIT — sizes readiness accounting int32_t spill_start{0}; - PTO2OrchestratorState *orch{nullptr}; + OrchestratorState *orch{nullptr}; uint32_t seen_epoch{0}; PTO2FaninPool &spill_pool; PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP]; @@ -316,7 +316,7 @@ struct PTO2FaninBuilder { }; static bool append_fanin_or_fail( - PTO2OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, + OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, TaskId producer_task_id, PTO2FaninBuilder *fanin_builder, uint8_t ring_id, DepFlags kind ) { // Decide-and-claim under the producer's fanout_lock. Two conditions make this @@ -431,7 +431,7 @@ static bool all_claimed_fanin_allow_early_resolve(const PTO2FaninBuilder &fanin_ }); } -void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state) { +void OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state) { SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; slot_state.dep_pool_mark = rss.dep_pool.top; @@ -442,7 +442,7 @@ void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state #endif } -void PTO2OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin) { +void OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin) { SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; TaskPayload *payload = slot_state.payload; @@ -508,7 +508,7 @@ void PTO2OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32 } } -static ChipTaskSlotState *oldest_open_task_on_current_ring(const PTO2OrchestratorState *orch) { +static ChipTaskSlotState *oldest_open_task_on_current_ring(const OrchestratorState *orch) { // Scope depth maps directly to a ring until the deepest ring, where all // further nested scopes share that ring. Its shallowest open scope begin // therefore marks the oldest task pinned by any open scope on this ring. @@ -516,7 +516,7 @@ static ChipTaskSlotState *oldest_open_task_on_current_ring(const PTO2Orchestrato return begin < orch->scope_tasks_size ? orch->scope_tasks[begin] : nullptr; } -static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, ChipTaskSlotState &slot_state, int32_t wfanin) { +static bool orch_wire_live_fanin_task(OrchestratorState *orch, ChipTaskSlotState &slot_state, int32_t wfanin) { SchedulerState *sched = orch->scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; @@ -535,7 +535,7 @@ static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, ChipTaskSlotS return true; } -static void scope_tasks_push(PTO2OrchestratorState *orch, ChipTaskSlotState *task_slot_state); +static void scope_tasks_push(OrchestratorState *orch, ChipTaskSlotState *task_slot_state); struct PreparedTask { TaskId task_id = TaskId::invalid(); @@ -559,7 +559,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { return layout; } -static bool check_scope_can_accept_task(PTO2OrchestratorState *orch, PTO2TaskAllocator &allocator, uint8_t ring_id) { +static bool check_scope_can_accept_task(OrchestratorState *orch, PTO2TaskAllocator &allocator, uint8_t ring_id) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit task outside a scope"); int32_t scope_task_count = orch->scope_tasks_size - orch->scope_begins[orch->scope_stack_top]; @@ -593,7 +593,7 @@ static bool check_scope_can_accept_task(PTO2OrchestratorState *orch, PTO2TaskAll } static bool prepare_task( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, + OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, TaskAttrs task_attrs, PreparedTask *out ) { uint8_t ring_id = orch->current_ring_id(); @@ -668,7 +668,7 @@ static bool prepare_task( // Scope Management // ============================================================================= -static void scope_tasks_push(PTO2OrchestratorState *orch, ChipTaskSlotState *task_slot_state) { +static void scope_tasks_push(OrchestratorState *orch, ChipTaskSlotState *task_slot_state) { if (orch->scope_tasks_size >= orch->scope_tasks_capacity) { // scope_tasks lives in the per-Worker arena (single backing allocation), // so realloc is not legal. Capacity is the total in-flight slot budget @@ -684,7 +684,7 @@ static void scope_tasks_push(PTO2OrchestratorState *orch, ChipTaskSlotState *tas orch->scope_tasks[orch->scope_tasks_size++] = task_slot_state; } -void PTO2OrchestratorState::begin_scope(PTO2ScopeMode mode) { +void OrchestratorState::begin_scope(PTO2ScopeMode mode) { auto *orch = this; if (orch->fatal) { return; @@ -723,7 +723,7 @@ void PTO2OrchestratorState::begin_scope(PTO2ScopeMode mode) { #endif } -void PTO2OrchestratorState::end_scope() { +void OrchestratorState::end_scope() { auto *orch = this; if (orch->fatal) { return; @@ -789,7 +789,7 @@ void PTO2OrchestratorState::end_scope() { // (no entry freed) is a genuine deadlock: latch SIMPLER_ERROR_TENSORMAP_OVERFLOW // and bail. Returns false on deadlock or on a fatal already latched by another // party. Cold path — the fast path returns immediately when the pool has room. -static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t needed) { +static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { PTO2TensorMap &tm = orch->tensor_map; if (tm.free_entries() >= needed) { return true; @@ -882,7 +882,7 @@ static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t neede // computation (explicit_deps + auto), output registration, slot init, and // Orch-side wiring/ready publication. static TaskOutputTensors submit_task_common( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, + OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id ) { CYCLE_COUNT_START(); @@ -1143,7 +1143,7 @@ static TaskOutputTensors submit_task_common( return result; } -TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args) { auto *orch = this; // Orchestration API should short-circuit after fatal, but keep this entry @@ -1223,7 +1223,7 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k // AICore dispatch. Empty active_mask routes the slot to the DUMMY ready // bucket; dispatch loop short-circuits to completion. Accepts the same Arg // shape as submit_task; scalars are permitted but never consumed. -TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::submit_dummy_task(const CoreTaskArgs &args) { auto *orch = this; if (orch->fatal) { @@ -1253,7 +1253,7 @@ TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const CoreTaskArgs &a ); } -TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { auto *orch = this; // Orchestration API should short-circuit after fatal, but keep this entry // robust as a no-op in case a caller reaches it directly. @@ -1365,7 +1365,7 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) // Flow Control // ============================================================================= -void PTO2OrchestratorState::mark_done() { +void OrchestratorState::mark_done() { auto *orch = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { int32_t total_tasks = orch->rings[r].task_allocator.active_count(); @@ -1390,8 +1390,8 @@ void PTO2OrchestratorState::mark_done() { } #if SIMPLER_ORCH_PROFILING -PTO2OrchProfilingData orchestrator_get_profiling() { - PTO2OrchProfilingData d; +OrchProfilingData orchestrator_get_profiling() { + OrchProfilingData d; d.sync_cycle = g_orch_sync_cycle; d.alloc_cycle = g_orch_alloc_cycle; d.args_cycle = g_orch_args_cycle; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index aa4a6ceb00..9a8ecd5e88 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -38,11 +38,11 @@ #include "types.h" /** - * Layout descriptor produced by PTO2OrchestratorState::reserve_layout(). Holds + * Layout descriptor produced by OrchestratorState::reserve_layout(). Holds * arena offsets for every sub-region the orchestrator owns (per-ring fanin * pools, scope arrays, plus the nested PTO2TensorMap layout). */ -struct PTO2OrchestratorLayout { +struct OrchestratorLayout { size_t off_fanin_pool[CHIP_MAX_RING_DEPTH]; size_t off_fanin_seen_epoch[CHIP_MAX_RING_DEPTH]; size_t off_scope_tasks; @@ -62,7 +62,7 @@ struct PTO2OrchestratorLayout { * * Contains all state needed for task graph construction and buffer management. */ -struct PTO2OrchestratorState { +struct OrchestratorState { // === SHARED MEMORY ACCESS === SharedMemoryHeader *sm_header; @@ -139,11 +139,11 @@ struct PTO2OrchestratorState { // tensor_map sub-layout) on the supplied arena. task_window_sizes feeds // the nested tensor_map layout. Returned layout is consumed by // init_data_from_layout. - static PTO2OrchestratorLayout reserve_layout( + static OrchestratorLayout reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE ); - static PTO2OrchestratorLayout reserve_layout( + static OrchestratorLayout reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH] ); @@ -153,15 +153,15 @@ struct PTO2OrchestratorState { // task_window_size feeds the per-ring SM address arithmetic. Safe to call // on a host arena that holds the prebuilt image. bool init_data_from_layout( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, uint64_t heap_size, + const OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size ); bool init_data_from_layout( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, + const OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ); bool reset_for_reuse( - const PTO2OrchestratorLayout &layout, void *sm_dev_base, void *gm_heap, + const OrchestratorLayout &layout, void *sm_dev_base, void *gm_heap, const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ); @@ -169,7 +169,7 @@ struct PTO2OrchestratorState { // scope_begins, rings[].fanin_pool.base, tensor_map.{buckets,entry_pool, // free_entry_list,task_entry_heads}, scheduler reference). // Idempotent — host runs once on the image, AICPU runs once after attach. - void wire_arena_pointers(const PTO2OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler); + void wire_arena_pointers(const OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler); // Forget pointers; arena owns the backing buffers. void destroy(); @@ -190,7 +190,7 @@ struct PTO2OrchestratorState { // ============================================================================= #if SIMPLER_ORCH_PROFILING -struct PTO2OrchProfilingData { +struct OrchProfilingData { uint64_t sync_cycle; uint64_t alloc_cycle; // Combined task slot + heap allocation uint64_t args_cycle; @@ -208,5 +208,5 @@ struct PTO2OrchProfilingData { uint64_t scope_end_atomic_count; }; -PTO2OrchProfilingData orchestrator_get_profiling(); +OrchProfilingData orchestrator_get_profiling(); #endif diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 8793534fdf..8ddd8c3827 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -99,7 +99,7 @@ MAYBE_UNINITIALIZED_BEGIN static bool wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_for_consumers, const char *caller) { TaskId owner = tensor.owner_task_id; - PTO2OrchestratorState &orch = rt->orchestrator; + OrchestratorState &orch = rt->orchestrator; // Segmented wait: collect up to kSegmentCap producer slots, then flush by // spinning on each. When the segment fills, we wait for the accumulated diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index eff705d01b..94ec967711 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -122,7 +122,7 @@ struct ArenaSizingKey { */ struct ArenaOffsets { size_t off_sm_handle{0}; - PTO2OrchestratorLayout orch; + OrchestratorLayout orch; SchedulerLayout sched; size_t off_runtime{0}; size_t off_mailbox{0}; @@ -157,7 +157,7 @@ struct RuntimeContext { // Components SharedMemoryHandle *sm_handle; - PTO2OrchestratorState orchestrator; + OrchestratorState orchestrator; SchedulerState scheduler; AICoreCompletionMailbox *aicore_mailbox; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 96c0ceaedc..f515786fcb 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -288,7 +288,7 @@ void SchedulerState::destroy() { // Orchestrator // ============================================================================= -PTO2OrchestratorLayout PTO2OrchestratorState::reserve_layout( +OrchestratorLayout OrchestratorState::reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], int32_t dep_pool_capacity ) { int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]; @@ -298,11 +298,11 @@ PTO2OrchestratorLayout PTO2OrchestratorState::reserve_layout( return reserve_layout(arena, task_window_sizes, dep_pool_capacities); } -PTO2OrchestratorLayout PTO2OrchestratorState::reserve_layout( +OrchestratorLayout OrchestratorState::reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH] ) { - PTO2OrchestratorLayout layout{}; + OrchestratorLayout layout{}; // scope_tasks holds every task in the open scope across all rings, so its cap // is the real in-flight budget = sum of the (runtime) per-ring windows. Using // the compile-time PTO2_SCOPE_TASKS_CAP instead under-sized the buffer when @@ -342,8 +342,8 @@ PTO2OrchestratorLayout PTO2OrchestratorState::reserve_layout( return layout; } -bool PTO2OrchestratorState::init_data_from_layout( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, uint64_t heap_size, +bool OrchestratorState::init_data_from_layout( + const OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size ) { uint64_t heap_sizes[CHIP_MAX_RING_DEPTH]; @@ -355,12 +355,12 @@ bool PTO2OrchestratorState::init_data_from_layout( return init_data_from_layout(layout, arena, sm_dev_base, gm_heap, heap_sizes, task_window_sizes); } -bool PTO2OrchestratorState::init_data_from_layout( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, +bool OrchestratorState::init_data_from_layout( + const OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ) { auto *orch = this; - *orch = PTO2OrchestratorState{}; + *orch = OrchestratorState{}; orch->sm_header = reinterpret_cast(sm_dev_base); orch->gm_heap_base = gm_heap; @@ -414,9 +414,9 @@ bool PTO2OrchestratorState::init_data_from_layout( return true; } -bool PTO2OrchestratorState::reset_for_reuse( - const PTO2OrchestratorLayout &layout, void *sm_dev_base, void *gm_heap, - const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] +bool OrchestratorState::reset_for_reuse( + const OrchestratorLayout &layout, void *sm_dev_base, void *gm_heap, const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], + const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ) { auto *orch = this; orch->sm_header = reinterpret_cast(sm_dev_base); @@ -474,8 +474,8 @@ bool PTO2OrchestratorState::reset_for_reuse( return true; } -void PTO2OrchestratorState::wire_arena_pointers( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler_arg +void OrchestratorState::wire_arena_pointers( + const OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler_arg ) { auto *orch = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -488,7 +488,7 @@ void PTO2OrchestratorState::wire_arena_pointers( orch->scheduler = scheduler_arg; } -void PTO2OrchestratorState::destroy() { +void OrchestratorState::destroy() { auto *orch = this; orch->tensor_map.destroy(); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -499,7 +499,7 @@ void PTO2OrchestratorState::destroy() { orch->scope_begins = nullptr; } -void PTO2OrchestratorState::set_scheduler(SchedulerState *scheduler) { this->scheduler = scheduler; } +void OrchestratorState::set_scheduler(SchedulerState *scheduler) { this->scheduler = scheduler; } // ============================================================================= // Top-level runtime arena @@ -534,7 +534,7 @@ RuntimeArenaLayout runtime_reserve_layout( for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { task_window_sizes_i32[r] = static_cast(task_window_sizes[r]); } - layout.offsets.orch = PTO2OrchestratorState::reserve_layout(arena, task_window_sizes_i32, dep_pool_capacities); + layout.offsets.orch = OrchestratorState::reserve_layout(arena, task_window_sizes_i32, dep_pool_capacities); layout.offsets.sched = SchedulerState::reserve_layout(arena, dep_pool_capacities); layout.offsets.off_runtime = arena.reserve(sizeof(RuntimeContext), PTO2_ALIGN_SIZE); layout.offsets.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); diff --git a/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md b/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md index 724a6763c6..7e4f1ac7be 100644 --- a/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md @@ -114,7 +114,7 @@ a boundary from which region happens to be reserved first — **Why the orchestrator is not in the arena at all.** hbg has no device-side orchestrator, so nothing on the device reads its state: not the `fanin_seen_epoch` table, not the scope arrays, not the TensorMap (~9.3 MB between them). It is -therefore a plain host object that owns those arrays — `PTO2OrchestratorState::init` +therefore a plain host object that owns those arrays — `OrchestratorState::init` allocates them — and `RuntimeContext` reaches it through a pointer that `bind` drops before the copied zone is uploaded, so no host address crosses the boundary. A `static_assert` keeps `RuntimeContext` trivially copyable, which is what forbids diff --git a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp index e25a729b46..50821a6e7f 100644 --- a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp @@ -539,13 +539,13 @@ bool bind_graph_definitions(const HostApi *api, GraphHostState &graph_state, Def } struct GraphHostStateBinding { - explicit GraphHostStateBinding(PTO2OrchestratorState &orchestrator, GraphHostState *state) : + explicit GraphHostStateBinding(OrchestratorState &orchestrator, GraphHostState *state) : orchestrator(orchestrator) { orchestrator.graph_host_state = state; } ~GraphHostStateBinding() { orchestrator.graph_host_state = nullptr; } - PTO2OrchestratorState &orchestrator; + OrchestratorState &orchestrator; }; int32_t run_host_orchestration( @@ -574,7 +574,7 @@ int32_t run_host_orchestration( // Re-point the orchestrator half at the host SM (scheduler keeps device SM). // Host-owned and destroyed with this frame, so rt->orchestrator is dropped on // every exit — it must never outlive the object it names. - PTO2OrchestratorState orchestrator; + OrchestratorState orchestrator; rt->orchestrator = &orchestrator; RAIIScopeGuard orchestrator_binding([rt]() { rt->orchestrator = nullptr; @@ -669,7 +669,7 @@ int32_t run_host_orchestration( // as spans rather than LOG_INFO because INFO is suppressed at the default log // level. Like the phase spans these are summed cost shares, not intervals. { - const PTO2OrchProfilingData prof = orchestrator_get_profiling(); + const OrchProfilingData prof = orchestrator_get_profiling(); const std::pair steps[] = { {"alloc", prof.alloc_cycle}, {"args", prof.args_cycle}, {"lookup", prof.lookup_cycle}, {"insert", prof.insert_cycle}, {"fanin", prof.fanin_cycle}, @@ -760,7 +760,7 @@ int32_t run_host_orchestration( // What this bind actually put in the pools. The orchestrator's cursors are the // exact populated extent of each one — no scan of the mirror is needed, and the // image ships that much rather than the worst case the mirror is dimensioned for. - const PTO2OrchestratorState &orch_state = orchestrator; + const OrchestratorState &orch_state = orchestrator; const sm_layout::BindUsage bind_usage{ nt, static_cast(orch_state.fanin_pool_cursor), diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator.h b/src/a5/runtime/host_build_graph/runtime/orchestrator.h index abed45ef46..c469578ca4 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator.h @@ -56,7 +56,7 @@ struct GraphHostState; * holds it by pointer — a by-value member would put non-trivially-copyable state * inside the struct bind copies to the device. */ -struct PTO2OrchestratorState { +struct OrchestratorState { // === SHARED MEMORY ACCESS === SharedMemoryHeader *sm_header; @@ -164,7 +164,7 @@ struct PTO2OrchestratorState { // ============================================================================= #if SIMPLER_ORCH_PROFILING -struct PTO2OrchProfilingData { +struct OrchProfilingData { uint64_t alloc_cycle; // Combined task slot + heap allocation uint64_t args_cycle; uint64_t lookup_cycle; @@ -177,5 +177,5 @@ struct PTO2OrchProfilingData { uint64_t args_atomic_count; }; -PTO2OrchProfilingData orchestrator_get_profiling(); +OrchProfilingData orchestrator_get_profiling(); #endif diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index e229f0cd41..a5a0bede66 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -233,7 +233,7 @@ __attribute__((weak, visibility("hidden"))) uint64_t host_phase_now_ns() { retur } while (0) #endif -static int32_t orch_mark_fatal(PTO2OrchestratorState *orch, int32_t error_code) { +static int32_t orch_mark_fatal(OrchestratorState *orch, int32_t error_code) { always_assert(orch != nullptr); orch->fatal = true; if (error_code == SIMPLER_ERROR_NONE || orch->sm_header == nullptr) { @@ -249,7 +249,7 @@ static int32_t orch_mark_fatal(PTO2OrchestratorState *orch, int32_t error_code) } static void -orch_report_fatal_v(PTO2OrchestratorState *orch, int32_t error_code, const char *func, const char *fmt, va_list args) { +orch_report_fatal_v(OrchestratorState *orch, int32_t error_code, const char *func, const char *fmt, va_list args) { int32_t latched_code = orch_mark_fatal(orch, error_code); if (fmt == nullptr || fmt[0] == '\0') { @@ -270,7 +270,7 @@ orch_report_fatal_v(PTO2OrchestratorState *orch, int32_t error_code, const char unified_log_error(func, "FATAL(code=%d): %s", error_code, message.data()); } -void PTO2OrchestratorState::report_fatal(int32_t error_code, const char *func, const char *fmt, ...) { +void OrchestratorState::report_fatal(int32_t error_code, const char *func, const char *fmt, ...) { auto *orch = this; va_list args; va_start(args, fmt); @@ -558,7 +558,7 @@ struct GraphHostState { namespace { -GraphHostState *graph_state_from(PTO2OrchestratorState *orch) { +GraphHostState *graph_state_from(OrchestratorState *orch) { return orch == nullptr ? nullptr : static_cast(orch->graph_host_state); } @@ -571,7 +571,7 @@ thread_local GraphRecording *g_active_graph_recording = nullptr; // part of the thread-local identity rather than implied by it. thread_local GraphHostState *g_active_graph_owner = nullptr; -GraphRecording *active_graph_recording(PTO2OrchestratorState *orch) { +GraphRecording *active_graph_recording(OrchestratorState *orch) { GraphHostState *state = graph_state_from(orch); if (state == nullptr || state != g_active_graph_owner) return nullptr; return g_active_graph_recording; @@ -1121,7 +1121,7 @@ GraphHostDefinitionList graph_host_definitions(GraphHostState &state) { return list; } -static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { +static uint32_t next_fanin_seen_epoch(OrchestratorState *orch) { uint32_t next = orch->fanin_seen_current_epoch + 1; if (next == 0) { memset( @@ -1141,7 +1141,7 @@ static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { // (the consumer), used to bump each producer's last_consumer_local_id (the // reclaim gate the host wait_for_consumers polls via completed_watermark). struct PTO2FaninBuilder { - PTO2FaninBuilder(PTO2OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : + PTO2FaninBuilder(OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : count(0), orch(orch), seen_epoch(seen_epoch), @@ -1149,7 +1149,7 @@ struct PTO2FaninBuilder { payload(payload), slots(payload->fanin_data()) {} int32_t count{0}; - PTO2OrchestratorState *orch{nullptr}; + OrchestratorState *orch{nullptr}; uint32_t seen_epoch{0}; int32_t self_local{0}; TaskPayload *payload{nullptr}; @@ -1178,7 +1178,7 @@ struct PTO2FaninBuilder { }; static bool append_fanin_or_fail( - PTO2OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, + OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, TaskId producer_task_id, PTO2FaninBuilder *fanin_builder ) { // Skip a stale/reused producer slot: the cached owner id no longer resolves @@ -1239,7 +1239,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { } static bool prepare_task( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, + OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, TaskAttrs task_attrs, PreparedTask *out ) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit task outside a scope"); @@ -1339,7 +1339,7 @@ static bool prepare_task( // Scope Management // ============================================================================= -void PTO2OrchestratorState::begin_scope(PTO2ScopeMode mode) { +void OrchestratorState::begin_scope(PTO2ScopeMode mode) { auto *orch = this; if (orch->fatal) { return; @@ -1384,7 +1384,7 @@ void PTO2OrchestratorState::begin_scope(PTO2ScopeMode mode) { } } -void PTO2OrchestratorState::end_scope() { +void OrchestratorState::end_scope() { auto *orch = this; if (orch->fatal) { return; @@ -1420,7 +1420,7 @@ void PTO2OrchestratorState::end_scope() { // SIMPLER_ERROR_TENSORMAP_OVERFLOW and bail rather than letting new_entry()'s hard // assert fire mid-registration. Returns false when the pool is exhausted or a // fatal is already latched by another party. -static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t needed) { +static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { PTO2TensorMap &tm = orch->tensor_map; if (tm.free_entries() >= needed) { return true; @@ -1449,7 +1449,7 @@ static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t neede // computation (explicit_deps + auto), output registration, slot init, and // Orch-side wiring/ready publication. static TaskOutputTensors submit_task_common( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, + OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id ) { CYCLE_COUNT_START(); @@ -1751,7 +1751,7 @@ void graph_reset_outer_payload(TaskPayload &payload) { } bool graph_submit_outer( - PTO2OrchestratorState *orch, GraphHostState *state, uint64_t full_key, uint64_t definition_hash, int32_t owned_heap, + OrchestratorState *orch, GraphHostState *state, uint64_t full_key, uint64_t definition_hash, int32_t owned_heap, bool defer_heap, const GraphTaskArgs &args, TaskId *submitted_id ) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit Graph outside a scope"); @@ -1896,7 +1896,7 @@ bool graph_submit_outer( } bool graph_submit_definition( - PTO2OrchestratorState *orch, GraphHostState *state, const GraphDefinition *definition, const GraphTaskArgs &args, + OrchestratorState *orch, GraphHostState *state, const GraphDefinition *definition, const GraphTaskArgs &args, TaskId *submitted_id ) { if (definition == nullptr || !graph_boundary_matches(*definition, args) || @@ -1913,13 +1913,12 @@ bool graph_submit_definition( } bool graph_submit_pending_definition( - PTO2OrchestratorState *orch, GraphHostState *state, uint64_t full_key, const GraphTaskArgs &args, - TaskId *submitted_id + OrchestratorState *orch, GraphHostState *state, uint64_t full_key, const GraphTaskArgs &args, TaskId *submitted_id ) { return graph_submit_outer(orch, state, full_key, 0, 0, true, args, submitted_id); } -bool graph_finalize_pending_submissions(PTO2OrchestratorState *orch, GraphHostState *state, uint64_t *failed_key) { +bool graph_finalize_pending_submissions(OrchestratorState *orch, GraphHostState *state, uint64_t *failed_key) { for (GraphPendingUpload &pending : state->pending_uploads) { if (!pending.deferred_heap) continue; auto definition_it = state->definitions.find(pending.full_key); @@ -1963,7 +1962,7 @@ bool graph_finalize_pending_submissions(PTO2OrchestratorState *orch, GraphHostSt // recording.nodes keeps those addresses valid because the inner buffer is // transferred, not copied. TaskOutputTensors graph_record_submit_node( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, + OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id ) { ORCH_PHASE_START(); @@ -2222,8 +2221,7 @@ TaskOutputTensors graph_record_submit_node( } // namespace -GraphScopeResult -PTO2OrchestratorState::graph_begin(uint64_t graph_key, const GraphTaskArgs &args, uint64_t callable_hash) { +GraphScopeResult OrchestratorState::graph_begin(uint64_t graph_key, const GraphTaskArgs &args, uint64_t callable_hash) { ORCH_PHASE_START_SPANNING(); const GraphScopeResult result = graph_begin_inner(graph_key, args, callable_hash); ORCH_PHASE_END_SPANNING(HostPhaseKind::OrchGraphBegin, graph_key); @@ -2231,7 +2229,7 @@ PTO2OrchestratorState::graph_begin(uint64_t graph_key, const GraphTaskArgs &args } GraphScopeResult -PTO2OrchestratorState::graph_begin_inner(uint64_t graph_key, const GraphTaskArgs &args, uint64_t callable_hash) { +OrchestratorState::graph_begin_inner(uint64_t graph_key, const GraphTaskArgs &args, uint64_t callable_hash) { auto *orch = this; GraphScopeResult result; GraphHostState *state = graph_state_from(orch); @@ -2348,7 +2346,7 @@ PTO2OrchestratorState::graph_begin_inner(uint64_t graph_key, const GraphTaskArgs return result; } -bool PTO2OrchestratorState::graph_prepare(void *recording_handle, const GraphTaskArgs &args) { +bool OrchestratorState::graph_prepare(void *recording_handle, const GraphTaskArgs &args) { GraphHostState *state = graph_state_from(this); if (state == nullptr || recording_handle == nullptr || g_active_graph_recording != nullptr) return false; auto *entry = static_cast(recording_handle); @@ -2389,7 +2387,7 @@ bool PTO2OrchestratorState::graph_prepare(void *recording_handle, const GraphTas return true; } -void PTO2OrchestratorState::graph_abort(void *recording_handle) { +void OrchestratorState::graph_abort(void *recording_handle) { GraphHostState *state = graph_state_from(this); auto *entry = static_cast(recording_handle); if (state == nullptr || entry == nullptr) return; @@ -2411,7 +2409,7 @@ void PTO2OrchestratorState::graph_abort(void *recording_handle) { // Finish the background recording and publish the Definition. The main // thread finalizes the already-submitted outer Graph tasks in graph_commit. -bool PTO2OrchestratorState::graph_end() { +bool OrchestratorState::graph_end() { GraphHostState *state = graph_state_from(this); GraphRecording *recording = active_graph_recording(this); GraphInflightRecording *entry = g_active_graph_entry; @@ -2471,13 +2469,13 @@ bool PTO2OrchestratorState::graph_end() { // Join every recording in flight and back-patch all deferred shells in submit // order. Orchestration completion is the only normal-path barrier. -void PTO2OrchestratorState::graph_commit() { +void OrchestratorState::graph_commit() { ORCH_PHASE_START_SPANNING(); graph_commit_inner(); ORCH_PHASE_END_SPANNING(HostPhaseKind::OrchGraphCommit, 0); } -void PTO2OrchestratorState::graph_commit_inner() { +void OrchestratorState::graph_commit_inner() { if (active_graph_recording(this) != nullptr) return; GraphHostState *state = graph_state_from(this); if (state == nullptr || state->inflight_count.load(std::memory_order_acquire) == 0) return; @@ -2517,7 +2515,7 @@ void PTO2OrchestratorState::graph_commit_inner() { } } -TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args) { auto *orch = this; // Orchestration API should short-circuit after fatal, but keep this entry @@ -2609,7 +2607,7 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k // AICore dispatch. Empty active_mask routes the slot to the DUMMY ready // bucket; dispatch loop short-circuits to completion. Accepts the same Arg // shape as submit_task; scalars are permitted but never consumed. -TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::submit_dummy_task(const CoreTaskArgs &args) { auto *orch = this; if (orch->fatal) { @@ -2645,7 +2643,7 @@ TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const CoreTaskArgs &a ); } -TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { auto *orch = this; // Orchestration API should short-circuit after fatal, but keep this entry // robust as a no-op in case a caller reaches it directly. @@ -2775,7 +2773,7 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) // Flow Control // ============================================================================= -void PTO2OrchestratorState::mark_done() { +void OrchestratorState::mark_done() { auto *orch = this; int32_t total_tasks = orch->task_allocator.active_count(); if (total_tasks > 0) { @@ -2790,8 +2788,8 @@ void PTO2OrchestratorState::mark_done() { } #if SIMPLER_ORCH_PROFILING -PTO2OrchProfilingData orchestrator_get_profiling() { - PTO2OrchProfilingData d; +OrchProfilingData orchestrator_get_profiling() { + OrchProfilingData d; d.alloc_cycle = g_orch_alloc_cycle; d.args_cycle = g_orch_args_cycle; d.lookup_cycle = g_orch_lookup_cycle; diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 4dac7bd3cd..ab38c79b4d 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -168,7 +168,7 @@ MAYBE_UNINITIALIZED_BEGIN static bool wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_for_consumers, const char *caller) { TaskId owner = tensor.owner_task_id; - PTO2OrchestratorState &orch = *rt->orchestrator; + OrchestratorState &orch = *rt->orchestrator; // Segmented wait: collect up to kSegmentCap producer slots, then flush by // spinning on each. When the segment fills, we wait for the accumulated diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_core.h b/src/a5/runtime/host_build_graph/runtime/runtime_core.h index 04cacd46af..22a739a68c 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_core.h @@ -145,7 +145,7 @@ struct RuntimeArenaLayout { // adjacent on the device and travel as one copy. // // The orchestrator is NOT here. It runs on the host, owns its own scratch, and - // no device code reads any of it — see PTO2OrchestratorState. + // no device code reads any of it — see OrchestratorState. size_t off_copied_begin{0}; size_t off_copied_end{0}; @@ -176,7 +176,7 @@ struct RuntimeContext { // the orchestrator runs on the host and owns non-trivial scratch. Null on the // device — bind drops it before the copied zone is uploaded, so no device code // may dereference it. - PTO2OrchestratorState *orchestrator; + OrchestratorState *orchestrator; // Device-only zone: the scheduler state holds no per-run content, so it is // addressed through the arena rather than carried inside this header. SchedulerState *scheduler; diff --git a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp index e98c0b1cf2..5b87b39445 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -197,11 +197,11 @@ void SchedulerState::destroy() { // Orchestrator // ============================================================================= -bool PTO2OrchestratorState::init( +bool OrchestratorState::init( void *sm_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size, SchedulerState *scheduler_arg ) { auto *orch = this; - *orch = PTO2OrchestratorState{}; + *orch = OrchestratorState{}; // A power-of-two window lets pto2_task_slot() mask instead of dividing. always_assert(task_window_size > 0 && (task_window_size & (task_window_size - 1)) == 0); @@ -243,7 +243,7 @@ bool PTO2OrchestratorState::init( return true; } -void PTO2OrchestratorState::set_scheduler(SchedulerState *scheduler) { this->scheduler = scheduler; } +void OrchestratorState::set_scheduler(SchedulerState *scheduler) { this->scheduler = scheduler; } // ============================================================================= // Top-level runtime arena diff --git a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index aa04f53b1f..6996a4d05b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -742,7 +742,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { // Print orchestrator profiling data #if SIMPLER_ORCH_PROFILING - PTO2OrchProfilingData p = orchestrator_get_profiling(); + OrchProfilingData p = orchestrator_get_profiling(); uint64_t total = p.sync_cycle + p.alloc_cycle + p.args_cycle + p.lookup_cycle + p.insert_cycle + p.fanin_cycle; if (total == 0) total = 1; // avoid div-by-zero diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index a93e319afc..04884ebea4 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -58,7 +58,7 @@ struct PTO2RingSet { }; ``` -### 4.2 PTO2OrchestratorState (modified) +### 4.2 OrchestratorState (modified) ```cpp // Before: single ring diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index e542cc12bb..43c8351376 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -345,7 +345,7 @@ This guarantees lookup only traverses valid entries — O(valid_entries_in_bucke **Layer 2 — Periodic Batch Cleanup** (`cleanup_retired`, per-task): -Every time the orchestrator submits a task (Step 0 of `PTO2OrchestratorState::submit_task`), it calls `PTO2TensorMap::sync_tensormap`. When `last_task_alive` has advanced by more than `PTO2_TENSORMAP_CLEANUP_INTERVAL` (default 64) tasks since the last cleanup, `PTO2TensorMap::cleanup_retired` runs: +Every time the orchestrator submits a task (Step 0 of `OrchestratorState::submit_task`), it calls `PTO2TensorMap::sync_tensormap`. When `last_task_alive` has advanced by more than `PTO2_TENSORMAP_CLEANUP_INTERVAL` (default 64) tasks since the last cleanup, `PTO2TensorMap::cleanup_retired` runs: This uses the **per-task entry chain** (`task_entry_head[task_slot]`) — each task's entries are doubly-linked together at insert time via `next_in_task`/`prev_in_task`. A slot's chain can hold more than one task's entries: a task at `local_id + N * window` reuses the slot and prepends to the chain already there, and cleanup can lag that reuse. Cleanup therefore walks the chain and frees only the entries whose `producer_task_id` matches the retiring task, unlinking each and leaving the rest linked — O(entries_in_slot), with no scan of the entire pool or all buckets. Freed entries are returned to `free_entry_list` for immediate reuse. @@ -367,7 +367,7 @@ In steady state, the number of valid TensorMap entries ≈ `active_tasks × avg_ ### 5.5 Dependency Discovery Flow -When `PTO2OrchestratorState::submit_task` processes parameters: +When `OrchestratorState::submit_task` processes parameters: 1. **INPUT/INOUT**: `PTO2TensorMap::lookup` searches for overlapping producers (with chain truncation) 2. For each producer found: `append_fanin_or_fail` adds the dependency @@ -425,7 +425,7 @@ In the scheduler's `task_state[]` array (`std::atomic`): ## 7. Orchestrator -### 7.1 PTO2OrchestratorState +### 7.1 OrchestratorState The orchestrator runs on AICPU Thread 3 and builds the task graph by calling the user-provided orchestration function. @@ -437,7 +437,7 @@ Key members: - `scheduler`: pointer to scheduler state (for Orch-side wiring helpers and ready queue access) - `gm_heap_base`, `gm_heap_size`: GM heap for output buffers -### 7.2 Task Submission Flow (`PTO2OrchestratorState::submit_task`) +### 7.2 Task Submission Flow (`OrchestratorState::submit_task`) | Step | Operation | | ---- | --------- | diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 1477edd4cd..af7ab556f9 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -160,7 +160,7 @@ static uint32_t g_orch_submit_idx = 0; #define CYCLE_COUNT_ORCH_SUBMIT_RECORD(tid) #endif -static int32_t orch_mark_fatal(PTO2OrchestratorState *orch, int32_t error_code) { +static int32_t orch_mark_fatal(OrchestratorState *orch, int32_t error_code) { always_assert(orch != nullptr); orch->fatal = true; if (error_code == SIMPLER_ERROR_NONE || orch->sm_header == nullptr) { @@ -176,7 +176,7 @@ static int32_t orch_mark_fatal(PTO2OrchestratorState *orch, int32_t error_code) } static void -orch_report_fatal_v(PTO2OrchestratorState *orch, int32_t error_code, const char *func, const char *fmt, va_list args) { +orch_report_fatal_v(OrchestratorState *orch, int32_t error_code, const char *func, const char *fmt, va_list args) { int32_t latched_code = orch_mark_fatal(orch, error_code); #if SIMPLER_DFX @@ -205,7 +205,7 @@ orch_report_fatal_v(PTO2OrchestratorState *orch, int32_t error_code, const char unified_log_error(func, "FATAL(code=%d): %s", error_code, message); } -void PTO2OrchestratorState::report_fatal(int32_t error_code, const char *func, const char *fmt, ...) { +void OrchestratorState::report_fatal(int32_t error_code, const char *func, const char *fmt, ...) { auto *orch = this; va_list args; va_start(args, fmt); @@ -213,7 +213,7 @@ void PTO2OrchestratorState::report_fatal(int32_t error_code, const char *func, c va_end(args); } -static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { +static uint32_t next_fanin_seen_epoch(OrchestratorState *orch) { uint32_t next = orch->fanin_seen_current_epoch + 1; if (next == 0) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -229,7 +229,7 @@ static uint32_t next_fanin_seen_epoch(PTO2OrchestratorState *orch) { } struct PTO2FaninBuilder { - PTO2FaninBuilder(PTO2OrchestratorState *orch, PTO2FaninPool &spill_pool, uint32_t seen_epoch) : + PTO2FaninBuilder(OrchestratorState *orch, PTO2FaninPool &spill_pool, uint32_t seen_epoch) : count(0), wait_count(0), spill_start(0), @@ -239,7 +239,7 @@ struct PTO2FaninBuilder { int32_t count{0}; // total fanin edges (all flag combinations) int32_t wait_count{0}; // edges carrying DEP_WAIT — sizes readiness accounting int32_t spill_start{0}; - PTO2OrchestratorState *orch{nullptr}; + OrchestratorState *orch{nullptr}; uint32_t seen_epoch{0}; PTO2FaninPool &spill_pool; PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP]; @@ -310,7 +310,7 @@ struct PTO2FaninBuilder { }; static bool append_fanin_or_fail( - PTO2OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, + OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, TaskId producer_task_id, PTO2FaninBuilder *fanin_builder, uint8_t ring_id, DepFlags kind ) { // Decide-and-claim under the producer's fanout_lock. Two conditions make this @@ -424,7 +424,7 @@ static bool all_claimed_fanin_allow_early_resolve(const PTO2FaninBuilder &fanin_ }); } -void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state) { +void OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state) { SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; slot_state.dep_pool_mark = rss.dep_pool.top; @@ -435,7 +435,7 @@ void PTO2OrchestratorState::mark_dep_pool_position(ChipTaskSlotState &slot_state #endif } -void PTO2OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin) { +void OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin) { SchedulerState *sched = scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; TaskPayload *payload = slot_state.payload; @@ -501,7 +501,7 @@ void PTO2OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32 } } -static ChipTaskSlotState *oldest_open_task_on_current_ring(const PTO2OrchestratorState *orch) { +static ChipTaskSlotState *oldest_open_task_on_current_ring(const OrchestratorState *orch) { // Scope depth maps directly to a ring until the deepest ring, where all // further nested scopes share that ring. Its shallowest open scope begin // therefore marks the oldest task pinned by any open scope on this ring. @@ -509,7 +509,7 @@ static ChipTaskSlotState *oldest_open_task_on_current_ring(const PTO2Orchestrato return begin < orch->scope_tasks_size ? orch->scope_tasks[begin] : nullptr; } -static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, ChipTaskSlotState &slot_state, int32_t wfanin) { +static bool orch_wire_live_fanin_task(OrchestratorState *orch, ChipTaskSlotState &slot_state, int32_t wfanin) { SchedulerState *sched = orch->scheduler; auto &rss = sched->ring_sched_states[slot_state.ring_id]; @@ -527,7 +527,7 @@ static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, ChipTaskSlotS return true; } -static void scope_tasks_push(PTO2OrchestratorState *orch, ChipTaskSlotState *task_slot_state); +static void scope_tasks_push(OrchestratorState *orch, ChipTaskSlotState *task_slot_state); struct PreparedTask { TaskId task_id = TaskId::invalid(); @@ -551,7 +551,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { return layout; } -static bool check_scope_can_accept_task(PTO2OrchestratorState *orch, PTO2TaskAllocator &allocator, uint8_t ring_id) { +static bool check_scope_can_accept_task(OrchestratorState *orch, PTO2TaskAllocator &allocator, uint8_t ring_id) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit task outside a scope"); int32_t scope_task_count = orch->scope_tasks_size - orch->scope_begins[orch->scope_stack_top]; @@ -585,7 +585,7 @@ static bool check_scope_can_accept_task(PTO2OrchestratorState *orch, PTO2TaskAll } static bool prepare_task( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, + OrchestratorState *orch, const CoreTaskArgs &args, int32_t total_output_size, ActiveMask active_mask, TaskAttrs task_attrs, PreparedTask *out ) { uint8_t ring_id = orch->current_ring_id(); @@ -656,7 +656,7 @@ static bool prepare_task( // Scope Management // ============================================================================= -static void scope_tasks_push(PTO2OrchestratorState *orch, ChipTaskSlotState *task_slot_state) { +static void scope_tasks_push(OrchestratorState *orch, ChipTaskSlotState *task_slot_state) { if (orch->scope_tasks_size >= orch->scope_tasks_capacity) { // scope_tasks lives in the per-Worker arena (single backing allocation), // so realloc is not legal. Capacity is the total in-flight slot budget @@ -672,7 +672,7 @@ static void scope_tasks_push(PTO2OrchestratorState *orch, ChipTaskSlotState *tas orch->scope_tasks[orch->scope_tasks_size++] = task_slot_state; } -void PTO2OrchestratorState::begin_scope(PTO2ScopeMode mode) { +void OrchestratorState::begin_scope(PTO2ScopeMode mode) { auto *orch = this; if (orch->fatal) { return; @@ -711,7 +711,7 @@ void PTO2OrchestratorState::begin_scope(PTO2ScopeMode mode) { #endif } -void PTO2OrchestratorState::end_scope() { +void OrchestratorState::end_scope() { auto *orch = this; if (orch->fatal) { return; @@ -777,7 +777,7 @@ void PTO2OrchestratorState::end_scope() { // (no entry freed) is a genuine deadlock: latch SIMPLER_ERROR_TENSORMAP_OVERFLOW // and bail. Returns false on deadlock or on a fatal already latched by another // party. Cold path — the fast path returns immediately when the pool has room. -static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t needed) { +static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { PTO2TensorMap &tm = orch->tensor_map; if (tm.free_entries() >= needed) { return true; @@ -884,7 +884,7 @@ static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t neede // computation (explicit_deps + auto), output registration, slot init, and // Orch-side wiring/ready publication. static TaskOutputTensors submit_task_common( - PTO2OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, + OrchestratorState *orch, const CoreTaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id ) { CYCLE_COUNT_START(); @@ -1145,7 +1145,7 @@ static TaskOutputTensors submit_task_common( return result; } -TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args) { auto *orch = this; // Orchestration API should short-circuit after fatal, but keep this entry @@ -1225,7 +1225,7 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k // AICore dispatch. Empty active_mask routes the slot to the DUMMY ready // bucket; dispatch loop short-circuits to completion. Accepts the same Arg // shape as submit_task; scalars are permitted but never consumed. -TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::submit_dummy_task(const CoreTaskArgs &args) { auto *orch = this; if (orch->fatal) { @@ -1255,7 +1255,7 @@ TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const CoreTaskArgs &a ); } -TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { +TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { auto *orch = this; // Orchestration API should short-circuit after fatal, but keep this entry // robust as a no-op in case a caller reaches it directly. @@ -1367,7 +1367,7 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const CoreTaskArgs &args) // Flow Control // ============================================================================= -void PTO2OrchestratorState::mark_done() { +void OrchestratorState::mark_done() { auto *orch = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { int32_t total_tasks = orch->rings[r].task_allocator.active_count(); @@ -1392,8 +1392,8 @@ void PTO2OrchestratorState::mark_done() { } #if SIMPLER_ORCH_PROFILING -PTO2OrchProfilingData orchestrator_get_profiling() { - PTO2OrchProfilingData d; +OrchProfilingData orchestrator_get_profiling() { + OrchProfilingData d; d.sync_cycle = g_orch_sync_cycle; d.alloc_cycle = g_orch_alloc_cycle; d.args_cycle = g_orch_args_cycle; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index aa4a6ceb00..9a8ecd5e88 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -38,11 +38,11 @@ #include "types.h" /** - * Layout descriptor produced by PTO2OrchestratorState::reserve_layout(). Holds + * Layout descriptor produced by OrchestratorState::reserve_layout(). Holds * arena offsets for every sub-region the orchestrator owns (per-ring fanin * pools, scope arrays, plus the nested PTO2TensorMap layout). */ -struct PTO2OrchestratorLayout { +struct OrchestratorLayout { size_t off_fanin_pool[CHIP_MAX_RING_DEPTH]; size_t off_fanin_seen_epoch[CHIP_MAX_RING_DEPTH]; size_t off_scope_tasks; @@ -62,7 +62,7 @@ struct PTO2OrchestratorLayout { * * Contains all state needed for task graph construction and buffer management. */ -struct PTO2OrchestratorState { +struct OrchestratorState { // === SHARED MEMORY ACCESS === SharedMemoryHeader *sm_header; @@ -139,11 +139,11 @@ struct PTO2OrchestratorState { // tensor_map sub-layout) on the supplied arena. task_window_sizes feeds // the nested tensor_map layout. Returned layout is consumed by // init_data_from_layout. - static PTO2OrchestratorLayout reserve_layout( + static OrchestratorLayout reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE ); - static PTO2OrchestratorLayout reserve_layout( + static OrchestratorLayout reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH] ); @@ -153,15 +153,15 @@ struct PTO2OrchestratorState { // task_window_size feeds the per-ring SM address arithmetic. Safe to call // on a host arena that holds the prebuilt image. bool init_data_from_layout( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, uint64_t heap_size, + const OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size ); bool init_data_from_layout( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, + const OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ); bool reset_for_reuse( - const PTO2OrchestratorLayout &layout, void *sm_dev_base, void *gm_heap, + const OrchestratorLayout &layout, void *sm_dev_base, void *gm_heap, const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ); @@ -169,7 +169,7 @@ struct PTO2OrchestratorState { // scope_begins, rings[].fanin_pool.base, tensor_map.{buckets,entry_pool, // free_entry_list,task_entry_heads}, scheduler reference). // Idempotent — host runs once on the image, AICPU runs once after attach. - void wire_arena_pointers(const PTO2OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler); + void wire_arena_pointers(const OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler); // Forget pointers; arena owns the backing buffers. void destroy(); @@ -190,7 +190,7 @@ struct PTO2OrchestratorState { // ============================================================================= #if SIMPLER_ORCH_PROFILING -struct PTO2OrchProfilingData { +struct OrchProfilingData { uint64_t sync_cycle; uint64_t alloc_cycle; // Combined task slot + heap allocation uint64_t args_cycle; @@ -208,5 +208,5 @@ struct PTO2OrchProfilingData { uint64_t scope_end_atomic_count; }; -PTO2OrchProfilingData orchestrator_get_profiling(); +OrchProfilingData orchestrator_get_profiling(); #endif diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 8793534fdf..8ddd8c3827 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -99,7 +99,7 @@ MAYBE_UNINITIALIZED_BEGIN static bool wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_for_consumers, const char *caller) { TaskId owner = tensor.owner_task_id; - PTO2OrchestratorState &orch = rt->orchestrator; + OrchestratorState &orch = rt->orchestrator; // Segmented wait: collect up to kSegmentCap producer slots, then flush by // spinning on each. When the segment fills, we wait for the accumulated diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index eff705d01b..94ec967711 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -122,7 +122,7 @@ struct ArenaSizingKey { */ struct ArenaOffsets { size_t off_sm_handle{0}; - PTO2OrchestratorLayout orch; + OrchestratorLayout orch; SchedulerLayout sched; size_t off_runtime{0}; size_t off_mailbox{0}; @@ -157,7 +157,7 @@ struct RuntimeContext { // Components SharedMemoryHandle *sm_handle; - PTO2OrchestratorState orchestrator; + OrchestratorState orchestrator; SchedulerState scheduler; AICoreCompletionMailbox *aicore_mailbox; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index afb7c44ec0..88c1319735 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -305,7 +305,7 @@ void SchedulerState::destroy() { // Orchestrator // ============================================================================= -PTO2OrchestratorLayout PTO2OrchestratorState::reserve_layout( +OrchestratorLayout OrchestratorState::reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], int32_t dep_pool_capacity ) { int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]; @@ -315,11 +315,11 @@ PTO2OrchestratorLayout PTO2OrchestratorState::reserve_layout( return reserve_layout(arena, task_window_sizes, dep_pool_capacities); } -PTO2OrchestratorLayout PTO2OrchestratorState::reserve_layout( +OrchestratorLayout OrchestratorState::reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH] ) { - PTO2OrchestratorLayout layout{}; + OrchestratorLayout layout{}; // scope_tasks holds every task in the open scope across all rings, so its cap // is the real in-flight budget = sum of the (runtime) per-ring windows. Using // the compile-time PTO2_SCOPE_TASKS_CAP instead under-sized the buffer when @@ -359,8 +359,8 @@ PTO2OrchestratorLayout PTO2OrchestratorState::reserve_layout( return layout; } -bool PTO2OrchestratorState::init_data_from_layout( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, uint64_t heap_size, +bool OrchestratorState::init_data_from_layout( + const OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, uint64_t heap_size, uint64_t task_window_size ) { uint64_t heap_sizes[CHIP_MAX_RING_DEPTH]; @@ -372,12 +372,12 @@ bool PTO2OrchestratorState::init_data_from_layout( return init_data_from_layout(layout, arena, sm_dev_base, gm_heap, heap_sizes, task_window_sizes); } -bool PTO2OrchestratorState::init_data_from_layout( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, +bool OrchestratorState::init_data_from_layout( + const OrchestratorLayout &layout, DeviceArena &arena, void *sm_dev_base, void *gm_heap, const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ) { auto *orch = this; - *orch = PTO2OrchestratorState{}; + *orch = OrchestratorState{}; orch->sm_header = reinterpret_cast(sm_dev_base); orch->gm_heap_base = gm_heap; @@ -431,9 +431,9 @@ bool PTO2OrchestratorState::init_data_from_layout( return true; } -bool PTO2OrchestratorState::reset_for_reuse( - const PTO2OrchestratorLayout &layout, void *sm_dev_base, void *gm_heap, - const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] +bool OrchestratorState::reset_for_reuse( + const OrchestratorLayout &layout, void *sm_dev_base, void *gm_heap, const uint64_t heap_sizes[CHIP_MAX_RING_DEPTH], + const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH] ) { auto *orch = this; orch->sm_header = reinterpret_cast(sm_dev_base); @@ -496,8 +496,8 @@ bool PTO2OrchestratorState::reset_for_reuse( return true; } -void PTO2OrchestratorState::wire_arena_pointers( - const PTO2OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler_arg +void OrchestratorState::wire_arena_pointers( + const OrchestratorLayout &layout, DeviceArena &arena, SchedulerState *scheduler_arg ) { auto *orch = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -510,7 +510,7 @@ void PTO2OrchestratorState::wire_arena_pointers( orch->set_scheduler(scheduler_arg); } -void PTO2OrchestratorState::destroy() { +void OrchestratorState::destroy() { auto *orch = this; orch->tensor_map.destroy(); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -521,7 +521,7 @@ void PTO2OrchestratorState::destroy() { orch->scope_begins = nullptr; } -void PTO2OrchestratorState::set_scheduler(SchedulerState *scheduler_arg) { +void OrchestratorState::set_scheduler(SchedulerState *scheduler_arg) { scheduler = scheduler_arg; if (scheduler == nullptr) return; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -567,7 +567,7 @@ RuntimeArenaLayout runtime_reserve_layout( for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { task_window_sizes_i32[r] = static_cast(task_window_sizes[r]); } - layout.offsets.orch = PTO2OrchestratorState::reserve_layout(arena, task_window_sizes_i32, dep_pool_capacities); + layout.offsets.orch = OrchestratorState::reserve_layout(arena, task_window_sizes_i32, dep_pool_capacities); layout.offsets.sched = SchedulerState::reserve_layout(arena, dep_pool_capacities); layout.offsets.off_runtime = arena.reserve(sizeof(RuntimeContext), PTO2_ALIGN_SIZE); layout.offsets.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); diff --git a/src/common/hierarchical/scope.h b/src/common/hierarchical/scope.h index f430c640b6..ac39c3092c 100644 --- a/src/common/hierarchical/scope.h +++ b/src/common/hierarchical/scope.h @@ -49,7 +49,7 @@ class Scope { // L2-style 0-based scope index: the innermost open scope, or 0 when // none is open. Used by Ring::alloc to choose a heap ring: // ring_idx = min(current_depth(), MAX_RING_DEPTH - 1) - // Matches `PTO2OrchestratorState::current_ring_id` semantics: the + // Matches `OrchestratorState::current_ring_id` semantics: the // first scope opened (depth()==1) maps to ring 0, the next nested // scope maps to ring 1, and so on. Returns 0 when no scope is open // so tasks submitted outside `Worker::run` still have a deterministic diff --git a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp index a6af42ee8f..80b89fc1d7 100644 --- a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp @@ -47,7 +47,7 @@ class HbgSubmitPoisonTest : public ::testing::Test { DeviceArena sm_arena; DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; SchedulerState sched{}; SchedulerLayout sched_layout{}; std::vector gm_heap; diff --git a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp index 30de4f6de5..61687a8f9f 100644 --- a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp @@ -24,9 +24,9 @@ class OrchestratorFaninTest : public ::testing::Test { DeviceArena sm_arena; DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; SchedulerState sched{}; - PTO2OrchestratorLayout orch_layout{}; + OrchestratorLayout orch_layout{}; SchedulerLayout sched_layout{}; std::vector gm_heap; @@ -40,7 +40,7 @@ class OrchestratorFaninTest : public ::testing::Test { task_window_sizes[r] = static_cast(PTO2_TASK_WINDOW_SIZE); } - orch_layout = PTO2OrchestratorState::reserve_layout(runtime_arena, task_window_sizes); + orch_layout = OrchestratorState::reserve_layout(runtime_arena, task_window_sizes); sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); @@ -344,7 +344,7 @@ TEST_F(OrchestratorFaninTest, ClosedChildHeadUsesTimeoutWithOpenParentOnSharedRi TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { auto cap_for = [](const int32_t windows[CHIP_MAX_RING_DEPTH]) { DeviceArena arena; - int32_t cap = PTO2OrchestratorState::reserve_layout(arena, windows).scope_tasks_cap; + int32_t cap = OrchestratorState::reserve_layout(arena, windows).scope_tasks_cap; arena.release(); return cap; }; diff --git a/tests/ut/cpp/a2a3/test_shared_memory.cpp b/tests/ut/cpp/a2a3/test_shared_memory.cpp index 22b3b73bd2..0456212146 100644 --- a/tests/ut/cpp/a2a3/test_shared_memory.cpp +++ b/tests/ut/cpp/a2a3/test_shared_memory.cpp @@ -282,7 +282,7 @@ TEST(RuntimeArenaLayout, RejectsOverflowingPerRingHeapSum) { char gm = 0; EXPECT_EQ(runtime_init_data_from_layout(runtime_arena, layout, PTO2_MODE_EXECUTE, &sm, 0, &gm, heaps), nullptr); - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; EXPECT_FALSE(orch.init_data_from_layout(layout.offsets.orch, runtime_arena, &sm, &gm, heaps, ws)); } diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index de09175de7..ea03dd7067 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -43,7 +43,7 @@ uint64_t get_test_reg_stub_base_addr(); class WiringTest : public ::testing::Test { protected: - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; SchedulerState sched{}; SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; diff --git a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp index a6af42ee8f..80b89fc1d7 100644 --- a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp @@ -47,7 +47,7 @@ class HbgSubmitPoisonTest : public ::testing::Test { DeviceArena sm_arena; DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; SchedulerState sched{}; SchedulerLayout sched_layout{}; std::vector gm_heap; diff --git a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp index 087791cdd8..b9eb2a7c36 100644 --- a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp @@ -24,9 +24,9 @@ class OrchestratorFaninTest : public ::testing::Test { DeviceArena sm_arena; DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; SchedulerState sched{}; - PTO2OrchestratorLayout orch_layout{}; + OrchestratorLayout orch_layout{}; SchedulerLayout sched_layout{}; std::vector gm_heap; @@ -40,7 +40,7 @@ class OrchestratorFaninTest : public ::testing::Test { task_window_sizes[r] = static_cast(PTO2_TASK_WINDOW_SIZE); } - orch_layout = PTO2OrchestratorState::reserve_layout(runtime_arena, task_window_sizes); + orch_layout = OrchestratorState::reserve_layout(runtime_arena, task_window_sizes); sched_layout = SchedulerState::reserve_layout(runtime_arena); ASSERT_NE(runtime_arena.commit(), nullptr); @@ -358,7 +358,7 @@ TEST_F(OrchestratorFaninTest, ClosedChildHeadUsesTimeoutWithOpenParentOnSharedRi TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { auto cap_for = [](const int32_t windows[CHIP_MAX_RING_DEPTH]) { DeviceArena arena; - int32_t cap = PTO2OrchestratorState::reserve_layout(arena, windows).scope_tasks_cap; + int32_t cap = OrchestratorState::reserve_layout(arena, windows).scope_tasks_cap; arena.release(); return cap; }; diff --git a/tests/ut/cpp/a5/test_shared_memory.cpp b/tests/ut/cpp/a5/test_shared_memory.cpp index 3c9cb32883..7b7696ee7f 100644 --- a/tests/ut/cpp/a5/test_shared_memory.cpp +++ b/tests/ut/cpp/a5/test_shared_memory.cpp @@ -329,7 +329,7 @@ TEST(RuntimeArenaLayout, RejectsOverflowingPerRingHeapSum) { char gm = 0; EXPECT_EQ(runtime_init_data_from_layout(runtime_arena, layout, PTO2_MODE_EXECUTE, &sm, 0, &gm, heaps), nullptr); - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; EXPECT_FALSE(orch.init_data_from_layout(layout.offsets.orch, runtime_arena, &sm, &gm, heaps, ws)); } diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index 83013bdb5f..00f24011c4 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -39,7 +39,7 @@ class WiringTest : public ::testing::Test { protected: - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; SchedulerState sched{}; SharedMemoryHandle *sm_handle = nullptr; DeviceArena sm_arena; diff --git a/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp b/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp index ceb6fdfe5a..90b1dee9ac 100644 --- a/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp @@ -42,7 +42,7 @@ class HbgGraphDefinitionArenaTest : public ::testing::Test { DeviceArena sm_arena; DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; SchedulerState sched{}; SchedulerLayout sched_layout{}; GraphHostStatePtr graph_state; diff --git a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp index f0566841e0..a70c08b9cd 100644 --- a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp @@ -34,7 +34,7 @@ class HbgGraphSubmitFailureTest : public ::testing::Test { DeviceArena sm_arena; DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; SchedulerState sched{}; SchedulerLayout sched_layout{}; GraphHostStatePtr graph_state; diff --git a/tests/ut/cpp/common/test_hbg_slot_claim.cpp b/tests/ut/cpp/common/test_hbg_slot_claim.cpp index e29e6268f7..61f78e5679 100644 --- a/tests/ut/cpp/common/test_hbg_slot_claim.cpp +++ b/tests/ut/cpp/common/test_hbg_slot_claim.cpp @@ -33,7 +33,7 @@ class HbgSlotClaimTest : public ::testing::Test { DeviceArena sm_arena; DeviceArena runtime_arena; SharedMemoryHandle *sm_handle = nullptr; - PTO2OrchestratorState orch{}; + OrchestratorState orch{}; SchedulerState sched{}; SchedulerLayout sched_layout{}; GraphHostStatePtr graph_state; From 0e8e4abb21fce9dab4b5deac4b4167f491edb787 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:19:55 -0700 Subject: [PATCH 06/22] Refactor: name the chip runtime's tensor map by its context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PTO2TensorMap` becomes `ChipTensorMap` — not `TensorMap`, because `src/common/hierarchical/tensormap.h:46` already declares a `class TensorMap`: the host orchestrator's producer lookup, which resolves `TensorKey` to a host `TaskSlot`. This one is the chip runtime's lazily-invalidated device-side map that the AICPU orchestrator walks to infer dependencies. Two maps, two contexts, so `.claude/rules/codestyle.md` rule 13 gives the bare name to the L3+ side and the `Chip` prefix to this one. Its satellites take the same prefix, so the cluster reads one way throughout: `ChipTensorMapEntry`, `ChipTensorMapLayout`, `ChipTensorMapProfilingData`, `chip_tensormap_get_profiling`, and the three capacity constants `CHIP_TENSORMAP_POOL_SIZE` / `CHIP_TENSORMAP_NUM_BUCKETS` / `CHIP_TENSORMAP_CLEANUP_INTERVAL`. `PTO2LookupResult` is a plain strip to `LookupResult`: it names no map of its own and nothing else claims the name. 445 occurrences across 61 files. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/rules/running-onboard.md | 2 +- docs/dfx/dep-gen.md | 4 +- ...-08-host-orch-phase-tail-is-page-faults.md | 2 +- .../device-error-codes/capacity.md | 2 +- .../device-error-codes/untested.md | 2 +- .../host/dep_gen_host_graph.cpp | 6 +- .../host_build_graph/runtime/dep_compute.h | 10 +- .../runtime/dep_gen_host_graph.h | 2 +- .../host_build_graph/runtime/orchestrator.h | 2 +- .../orchestrator_core/orchestrator.cpp | 14 +-- .../orchestrator_core/runtime_core.cpp | 2 +- .../host_build_graph/runtime/runtime_types.h | 4 +- .../runtime/shared/tensormap.cpp | 24 ++--- .../host_build_graph/runtime/tensormap.h | 72 +++++++-------- .../aicpu/aicpu_executor.cpp | 2 +- .../docs/MULTI_RING.md | 8 +- .../docs/RUNTIME_LOGIC.md | 12 +-- .../host/dep_gen_replay.cpp | 32 +++---- .../host/dep_gen_replay.h | 2 +- .../runtime/dep_compute.h | 6 +- .../runtime/orchestrator.cpp | 4 +- .../runtime/orchestrator.h | 6 +- .../runtime/runtime_core.cpp | 2 +- .../runtime/runtime_types.h | 6 +- .../runtime/shared/runtime_init.cpp | 2 +- .../runtime/shared/tensormap.cpp | 58 ++++++------ .../runtime/tensormap.h | 92 +++++++++---------- .../host/dep_gen_host_graph.cpp | 6 +- .../host_build_graph/runtime/dep_compute.h | 10 +- .../runtime/dep_gen_host_graph.h | 2 +- .../host_build_graph/runtime/orchestrator.h | 2 +- .../orchestrator_core/orchestrator.cpp | 14 +-- .../orchestrator_core/runtime_core.cpp | 2 +- .../host_build_graph/runtime/runtime_types.h | 4 +- .../runtime/shared/tensormap.cpp | 24 ++--- .../host_build_graph/runtime/tensormap.h | 72 +++++++-------- .../aicpu/aicpu_executor.cpp | 2 +- .../docs/MULTI_RING.md | 8 +- .../docs/RUNTIME_LOGIC.md | 12 +-- .../host/dep_gen_replay.cpp | 32 +++---- .../host/dep_gen_replay.h | 2 +- .../runtime/dep_compute.h | 6 +- .../runtime/orchestrator.cpp | 4 +- .../runtime/orchestrator.h | 6 +- .../runtime/runtime_core.cpp | 2 +- .../runtime/runtime_types.h | 6 +- .../runtime/shared/runtime_init.cpp | 2 +- .../runtime/shared/tensormap.cpp | 58 ++++++------ .../runtime/tensormap.h | 92 +++++++++---------- src/common/hierarchical/tensormap.h | 4 +- src/common/runtime_status/error_names.h | 2 +- src/common/task_interface/buffer.h | 2 +- src/common/task_interface/tensor.h | 2 +- .../dfx/dep_gen/test_dep_gen.py | 2 +- .../dfx/dep_gen/test_dep_gen.py | 2 +- .../dfx/dep_gen/test_dep_gen.py | 2 +- .../test_runtime_fatal_codes.py | 4 +- tests/ut/cpp/CMakeLists.txt | 2 +- tests/ut/cpp/a2a3/test_hbg_tensormap.cpp | 12 +-- tests/ut/cpp/a2a3/test_tensormap.cpp | 18 ++-- tests/ut/cpp/a5/test_tensormap.cpp | 18 ++-- 61 files changed, 409 insertions(+), 409 deletions(-) diff --git a/.claude/rules/running-onboard.md b/.claude/rules/running-onboard.md index 45e8593c40..f5ab090a32 100644 --- a/.claude/rules/running-onboard.md +++ b/.claude/rules/running-onboard.md @@ -175,7 +175,7 @@ for the signature that actually fired:** | `FATAL: Task Allocator Deadlock` / `FATAL: Dependency Pool Deadlock` / `FATAL: Fanin Spill Pool Deadlock` | an allocator or pool could not reclaim enough space | This header identifies the blocked resource, not the root cause. Classify it using the structural/timeout line that follows. | | `Provable head-of-line deadlock` | **proven open-scope structural deadlock** | TRB only: the reclaim head is the oldest task owned by an open scope on that ring. The blocked orchestrator cannot end that scope, so the head cannot become consumed. On A5, classification waits for at least 10 ms without reclaim progress and an exact-watermark publication acknowledgment. | | `No reclaim progress for ~500 ms` / `cannot reclaim space after ~500 ms` | allocator/pool **reclaim timeout** | TRB only — the 500ms backstop (`PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES`) lives in its reclaiming allocators. It proves prolonged lack of reclaim progress, not why progress stopped; check capacity, the dumped head, consumers and scheduler state. | -| `Task Window Exhausted` / `Fanin Capacity Exhausted` / `TensorMap Entry Pool Exhausted` | **capacity**, HBG only | HBG is whole-graph-resident and reclaims nothing mid-run, so this is a sizing verdict reached immediately, not a stall. The line carries used/capacity and the requested amount; raise `PTO2_RING_TASK_WINDOW` / `PTO2_TENSORMAP_POOL_SIZE` or shrink the graph. Inline fanin is hard-capped at `PTO2_MAX_FANIN=128`; HBG has no `PTO2_RING_DEP_POOL`. HBG's graph heap has no knob and cannot exhaust: it is committed after orchestration at the measured size, so a graph too large for the device fails host-side at that commit. | +| `Task Window Exhausted` / `Fanin Capacity Exhausted` / `TensorMap Entry Pool Exhausted` | **capacity**, HBG only | HBG is whole-graph-resident and reclaims nothing mid-run, so this is a sizing verdict reached immediately, not a stall. The line carries used/capacity and the requested amount; raise `PTO2_RING_TASK_WINDOW` / `CHIP_TENSORMAP_POOL_SIZE` or shrink the graph. Inline fanin is hard-capped at `PTO2_MAX_FANIN=128`; HBG has no `PTO2_RING_DEP_POOL`. HBG's graph heap has no knob and cannot exhaust: it is committed after orchestration at the measured size, so a graph too large for the device fails host-side at that commit. | | `Timeout (N cycles): producer/consumers ...` | **SPIN** wait on a specific producer/consumer | `runtime_core.cpp`. | | `HandleTaskTimeout` / `kill aicpu-sd` | **OS op-execute timeout** | STARS/tsdaemon, default 45s (`PLATFORM_OP_EXECUTE_TIMEOUT_US`). **A 45s kill ≠ deadlock** — the op was merely long or stalled. Raise this constant to measure true on-device duration. | | `log_stall_diagnostics` (cores idle + tasks `state=WAIT fanin 0/N` + `completed` frozen) | **forward-progress stall** | No dedicated detector — intermittent races (often contention-triggered) land here and are reaped only by the op-timeout above. | diff --git a/docs/dfx/dep-gen.md b/docs/dfx/dep-gen.md index 2aabe7348a..76a5a8e990 100644 --- a/docs/dfx/dep-gen.md +++ b/docs/dfx/dep-gen.md @@ -54,7 +54,7 @@ inputs to each submit are captured and the graph is reconstructed afterwards. - **Host replay (dual-pass, self-checking).** After `reconcile_counters()` confirms a clean trace (no drops, no leftovers), `dep_gen_replay_emit_deps_json` runs every record back through *two* - parallel host-resident `PTO2TensorMap` instances that evolve in lockstep: + parallel host-resident `ChipTensorMap` instances that evolve in lockstep: - **Oracle pass** drives the canonical `compute_task_fanin` template from `dep_compute.h` and collects the producer-id → `DepFlags` mapping the runtime would have emitted (flags OR-accumulated per @@ -398,7 +398,7 @@ list; only the dep_gen replay graph loses the tail. | AICPU writer | `src/{a2a3,a5}/platform/include/aicpu/dep_gen_collector_aicpu.h`, `src/common/platform/shared/aicpu/dep_gen_collector_aicpu.cpp` | Single-instance write path; weak-fallback exported to host build. Both platforms share the same writer implementation — the writer accesses its own device-side view of shared memory, independent of how host↔device transport is implemented. | | Host collector | `src/common/platform/include/host/dep_gen_collector.h`, `src/common/platform/shared/host/dep_gen_collector.cpp` | `ProfilerBase` — drains ring → `records_` vector. On non-SVM platforms it uses the base `alloc_paired_buffer`, which malloc's a host shadow + `copy_to_device`'s it and registers it via `add_malloc_shadow` so teardown can free it; `reconcile_counters` explicitly `copy_from_device`'s the BufferState before reading, and `finalize` lets `BufferPoolManager::clear_mappings()` release all shadows as the single source of truth. | | Capture call site (device-orch) | `src/{a2a3,a5}/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp` `submit_task_common` | One conditional block that snapshots inputs into the ring when `is_dep_gen_enabled()`; fires for both `submit_task` and `submit_dummy_task`. The schema carries `kernel_ids[3] = {aic, aiv0, aiv1}` so the swimlane post-processor can resolve `task_id → kernel` from `deps.json` at level=1 where the AICore record is the sole device-side identity source. Inactive subslots stay at `INVALID_KERNEL_ID = -1`. It also carries the SPMD logical block num (`block_num` on a2a3, `core_num` on a5's launch spec) as `tasks[].block_num`. | -| Replay | `src/{a2a3,a5}/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.{h,cpp}` | Pure CPU; runs dual-pass differential replay — `compute_task_fanin` (oracle) + inlined STEP A/B mirror (annotated) against two `PTO2TensorMap` instances. Emits `deps.json` when both passes agree per record. Platform-agnostic — a5 reuses the a2a3 source verbatim. | +| Replay | `src/{a2a3,a5}/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.{h,cpp}` | Pure CPU; runs dual-pass differential replay — `compute_task_fanin` (oracle) + inlined STEP A/B mirror (annotated) against two `ChipTensorMap` instances. Emits `deps.json` when both passes agree per record. Platform-agnostic — a5 reuses the a2a3 source verbatim. | | Host-direct capture (host-orch) | `src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h`, `src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp` | Task / tensor / edge tables filled from `submit_task_common` + `compute_task_fanin`'s `Annotate` hooks (`src/a2a3/runtime/host_build_graph/runtime/dep_compute.h`), reset per orchestration by `run_host_orchestration`, serialized by the same `deps.json` writer. The runtime translation unit carries weak no-op fallbacks so the AICPU build links without it. | | Device-runner hookup | `src/{a2a3,a5}/platform/{onboard,sim}/host/device_runner.cpp` | `dep_gen_host_graph_active()` picks the shape: host-orch calls `dep_gen_host_graph_emit(deps_path)` at teardown — reading the thread-local graph its own orchestration built on the same child progress thread — and skips collector init/start/reconcile entirely; device-orch calls `dep_gen_replay_emit_deps_json(records.data(), records.size(), deps_path)` post-`reconcile_counters`. The c_api latches the CallConfig before the bind so host capture is armed before the orchestration it records. | | Viewer | `simpler_setup/tools/deps_viewer.py` | `deps.json` → text (default) or pan/zoom HTML | diff --git a/docs/investigations/2026-08-host-orch-phase-tail-is-page-faults.md b/docs/investigations/2026-08-host-orch-phase-tail-is-page-faults.md index 2addac605b..e5dbc6c161 100644 --- a/docs/investigations/2026-08-host-orch-phase-tail-is-page-faults.md +++ b/docs/investigations/2026-08-host-orch-phase-tail-is-page-faults.md @@ -85,7 +85,7 @@ where the fault landed. Over the 447 faulting calls above 10 µs (867 faults, 19 leaves the per-node vector as the allocation: it is the single largest source. **But the structure that makes the others expensive is the hazard map.** -`PTO2TensorMap::init` takes four `new[]` allocations per recording — 4096×8 buckets, +`ChipTensorMap::init` takes four `new[]` allocations per recording — 4096×8 buckets, 16384×128 entries, 16384×8 free list, 1024×8 task heads = **2.17 MB**, of which the 2 MB entry pool is one block — and they are freed when the recording is destroyed at the end of the orchestration. Only ~115 KB is ever touched (the buckets and task heads `init` diff --git a/docs/troubleshooting/device-error-codes/capacity.md b/docs/troubleshooting/device-error-codes/capacity.md index 313c981754..a15619cd1e 100644 --- a/docs/troubleshooting/device-error-codes/capacity.md +++ b/docs/troubleshooting/device-error-codes/capacity.md @@ -44,7 +44,7 @@ that trips the code. | ------- | ------------------- | ---- | --- | | HBG | task window | 3 | raise `ring_task_window` (`PTO2_RING_TASK_WINDOW`), or shrink the graph | | HBG | inline fanin | 4 | reduce distinct producers to `PTO2_MAX_FANIN` (currently 128) or less; HBG has no `PTO2_RING_DEP_POOL` | -| HBG | TensorMap entries | 11 | increase `PTO2_TENSORMAP_POOL_SIZE`, or reduce registered outputs | +| HBG | TensorMap entries | 11 | increase `CHIP_TENSORMAP_POOL_SIZE`, or reduce registered outputs | | TRB | open-scope task window | 1 or 3 | raise `PTO2_RING_TASK_WINDOW`, split the scope, or diagnose stalled reclaim | | TRB | heap | 2 | raise `PTO2_RING_HEAP`, shrink allocations, or diagnose stalled reclaim | | TRB | dependency pool | 4 | raise `PTO2_RING_DEP_POOL`, cut fanin, or diagnose stalled reclaim | diff --git a/docs/troubleshooting/device-error-codes/untested.md b/docs/troubleshooting/device-error-codes/untested.md index 5f765507b9..bf456ed6ec 100644 --- a/docs/troubleshooting/device-error-codes/untested.md +++ b/docs/troubleshooting/device-error-codes/untested.md @@ -22,7 +22,7 @@ they cannot be provoked. Recording why, so the next person does not re-derive it end; hbg is whole-graph-resident and reclaims nothing, so overrunning the ring latches **2 HEAP_RING_DEADLOCK** or the ring's own exhaustion code instead. Reserved in hbg's table, raised only by `tensormap_and_ringbuffer`. -- **11 TENSORMAP_OVERFLOW** needs the 65536-entry pool (`PTO2_TENSORMAP_POOL_SIZE`, +- **11 TENSORMAP_OVERFLOW** needs the 65536-entry pool (`CHIP_TENSORMAP_POOL_SIZE`, compile-time, not tunable via `runtime_env`) flooded *and* a stalled producer holding entries. - **S4 / S5 / unknown**: not expressible through the public API. `set_dependencies()` diff --git a/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp b/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp index a11f05f3d5..9450b2cdaa 100644 --- a/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp +++ b/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp @@ -183,9 +183,9 @@ void fill_consumer(EdgeAnnot &e, const ChipTensor &t) { } } -// Copy a PTO2TensorMapEntry's slice description into an EdgeAnnot's producer_* +// Copy a ChipTensorMapEntry's slice description into an EdgeAnnot's producer_* // fields. Only called from the TENSORMAP path. -void fill_producer(EdgeAnnot &e, const PTO2TensorMapEntry &entry) { +void fill_producer(EdgeAnnot &e, const ChipTensorMapEntry &entry) { e.producer_ndims = entry.ndims; e.producer_start_offset = entry.start_offset; for (uint32_t i = 0; i < entry.ndims && i < MAX_TENSOR_DIMS; i++) { @@ -465,7 +465,7 @@ void dep_gen_host_graph_add_creator_edge(uint64_t producer_raw, int32_t arg_idx, } void dep_gen_host_graph_add_tensormap_edge( - uint64_t producer_raw, int32_t arg_idx, const ChipTensor &consumer, const PTO2TensorMapEntry &entry, + uint64_t producer_raw, int32_t arg_idx, const ChipTensor &consumer, const ChipTensorMapEntry &entry, OverlapStatus overlap ) { HostGraphState &s = state(); diff --git a/src/a2a3/runtime/host_build_graph/runtime/dep_compute.h b/src/a2a3/runtime/host_build_graph/runtime/dep_compute.h index 5c19e480e9..cbf050699c 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/dep_compute.h +++ b/src/a2a3/runtime/host_build_graph/runtime/dep_compute.h @@ -37,7 +37,7 @@ * The Annotate callback contract (dep_gen graph capture): * void annotate.creator(int32_t arg_idx, const ChipTensor &consumer, TaskId producer); * void annotate.tensormap(int32_t arg_idx, const ChipTensor &consumer, - * const PTO2TensorMapEntry &entry, OverlapStatus overlap); + * const ChipTensorMapEntry &entry, OverlapStatus overlap); * Both fire for exactly the producers `emit` saw, with the tensor-side context an * edge needs. `tensormap` runs before the INOUT/COVERED remove_entry(), so `entry` * is still live when it is read. NoDepAnnotate is the empty default and compiles @@ -80,7 +80,7 @@ struct DepInputs { */ struct NoDepAnnotate { void creator(int32_t, const ChipTensor &, TaskId) const {} - void tensormap(int32_t, const ChipTensor &, const PTO2TensorMapEntry &, OverlapStatus) const {} + void tensormap(int32_t, const ChipTensor &, const ChipTensorMapEntry &, OverlapStatus) const {} }; /** @@ -97,7 +97,7 @@ struct NoDepAnnotate { */ template [[nodiscard]] inline bool compute_task_fanin( - const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_manual_scope, Emit emit, Annotate annotate = Annotate{} + const DepInputs &inputs, ChipTensorMap &tensor_map, bool in_manual_scope, Emit emit, Annotate annotate = Annotate{} ) { if (in_manual_scope) { return true; @@ -131,7 +131,7 @@ template } bool fatal = false; - tensor_map.lookup(*tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus overlap_status) -> bool { + tensor_map.lookup(*tensor, [&](ChipTensorMapEntry &entry, OverlapStatus overlap_status) -> bool { if (!emit(entry.producer_task_id)) { fatal = true; return false; // stop iteration @@ -158,7 +158,7 @@ template * No-op when in_manual_scope. */ inline void -register_task_outputs(const DepInputs &inputs, TaskId task_id, PTO2TensorMap &tensor_map, bool in_manual_scope) { +register_task_outputs(const DepInputs &inputs, TaskId task_id, ChipTensorMap &tensor_map, bool in_manual_scope) { if (in_manual_scope) { return; } diff --git a/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h b/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h index ecf5037a6d..97c4178a4f 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h +++ b/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h @@ -104,7 +104,7 @@ void dep_gen_host_graph_add_creator_edge(uint64_t producer_raw, int32_t arg_idx, /** STEP 3 Step B: a tensormap producer whose written slice this task reads. */ void dep_gen_host_graph_add_tensormap_edge( - uint64_t producer_raw, int32_t arg_idx, const ChipTensor &consumer, const PTO2TensorMapEntry &entry, + uint64_t producer_raw, int32_t arg_idx, const ChipTensor &consumer, const ChipTensorMapEntry &entry, OverlapStatus overlap ); diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h index faa174a66b..245dc41ffb 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h @@ -67,7 +67,7 @@ struct OrchestratorState { uint32_t fanin_seen_current_epoch{1}; // === TENSOR MAP (Private) === - PTO2TensorMap tensor_map; // Producer lookup + ChipTensorMap tensor_map; // Producer lookup // === SCOPE STACK (Private) === // Depth only. A scope decides whether a submit takes its fanin from TensorMap diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index a5a0bede66..dbb7a56567 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -81,7 +81,7 @@ struct DepGraphAnnotate { dep_gen_host_graph_add_creator_edge(producer.raw, arg_idx, consumer); } void tensormap( - int32_t arg_idx, const ChipTensor &consumer, const PTO2TensorMapEntry &entry, OverlapStatus overlap + int32_t arg_idx, const ChipTensor &consumer, const ChipTensorMapEntry &entry, OverlapStatus overlap ) const { dep_gen_host_graph_add_tensormap_edge(entry.producer_task_id.raw, arg_idx, consumer, entry, overlap); } @@ -90,7 +90,7 @@ __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_add_explicit __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_add_creator_edge(uint64_t, int32_t, const ChipTensor &) {} __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_add_tensormap_edge( - uint64_t, int32_t, const ChipTensor &, const PTO2TensorMapEntry &, OverlapStatus + uint64_t, int32_t, const ChipTensor &, const ChipTensorMapEntry &, OverlapStatus ) {} // AICore register accessor (aicpu/platform_regs.h). The host orchestrator's @@ -442,7 +442,7 @@ struct GraphRecording { // (the shape every generated orchestration uses) would therefore record a // node with no edge to its actual producer, and the Definition would replay // a DAG the same body never had when submitted task by task. - PTO2TensorMap tensor_map{}; + ChipTensorMap tensor_map{}; bool tensor_map_ready{false}; // Scope depth as the body sees it. begin_scope/end_scope leave the real // orchestrator stack untouched while recording (a Graph replays flat), but @@ -649,7 +649,7 @@ graph_classify_scalar(const GraphRecording &recording, const ArgT &args, int32_t // Entry capacity for one recorded body's hazard map. A Definition is capped at // GRAPH_MAX_NODES nodes and each node registers at most its INOUT/OUTPUT_EXISTING // args, so this bounds the worst realistic body while staying a small fraction of -// the ring path's whole-orchestration pool (PTO2_TENSORMAP_POOL_SIZE). Exhausting +// the ring path's whole-orchestration pool (CHIP_TENSORMAP_POOL_SIZE). Exhausting // it marks the recording unsupported, which graph_commit reports as // SIMPLER_ERROR_INVALID_ARGS -- the outer shell is already submitted by then, so there // is no ordinary-path fallback left to take. @@ -659,7 +659,7 @@ constexpr int32_t GRAPH_RECORD_TENSORMAP_POOL_SIZE = 16384; // reported to the caller, which abandons the recording rather than producing a // Definition with inferred edges missing. bool graph_recording_init_tensor_map(GraphRecording &recording) { - if (!recording.tensor_map.init(PTO2_TENSORMAP_NUM_BUCKETS, GRAPH_RECORD_TENSORMAP_POOL_SIZE, GRAPH_MAX_NODES)) { + if (!recording.tensor_map.init(CHIP_TENSORMAP_NUM_BUCKETS, GRAPH_RECORD_TENSORMAP_POOL_SIZE, GRAPH_MAX_NODES)) { return false; } recording.tensor_map_ready = true; @@ -1421,7 +1421,7 @@ void OrchestratorState::end_scope() { // assert fire mid-registration. Returns false when the pool is exhausted or a // fatal is already latched by another party. static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { - PTO2TensorMap &tm = orch->tensor_map; + ChipTensorMap &tm = orch->tensor_map; if (tm.free_entries() >= needed) { return true; } @@ -1437,7 +1437,7 @@ static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { LOG_ERROR(" - Free: %d entries", tm.free_entries()); LOG_ERROR(" - Needed: %d entries", needed); LOG_ERROR("Solution:"); - LOG_ERROR(" Increase PTO2_TENSORMAP_POOL_SIZE (current: %d).", tm.pool_capacity()); + LOG_ERROR(" Increase CHIP_TENSORMAP_POOL_SIZE (current: %d).", tm.pool_capacity()); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_TENSORMAP_OVERFLOW); return false; diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index ab38c79b4d..0c934b1325 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -301,7 +301,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo } // Step B: modifier writer lookup (OverlapMap), direct callback - orch.tensor_map.lookup(tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus) -> bool { + orch.tensor_map.lookup(tensor, [&](ChipTensorMapEntry &entry, OverlapStatus) -> bool { TaskId pid = entry.producer_task_id; const auto *slot = resolve_producer(pid); if (slot == nullptr) return false; diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index 57a868583a..f429363d09 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -84,8 +84,8 @@ // bind_callable_to_runtime_impl). // Memory pools (total = value, single ring) -#define PTO2_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool -#define PTO2_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) +#define CHIP_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool +#define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) // Three address classes coexist during orchestration, in windows the two constants // below keep disjoint: real device addresses stay below HEAP_VIRTUAL_BASE, since diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/tensormap.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/tensormap.cpp index cc0f50a7df..82799bff20 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/tensormap.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/tensormap.cpp @@ -61,17 +61,17 @@ uint64_t g_insert_count = 0; * exists to avoid, and nothrow keeps the failure reportable to a caller that * still has an alternative path. */ -bool PTO2TensorMap::init(int32_t new_num_buckets, int32_t new_pool_size, int32_t new_task_window_size) { +bool ChipTensorMap::init(int32_t new_num_buckets, int32_t new_pool_size, int32_t new_task_window_size) { // num_buckets must be a power of two for the hash truncation to work, and // task_window_size for the task-chain slot mask. always_assert(new_num_buckets > 0 && (new_num_buckets & (new_num_buckets - 1)) == 0); always_assert(new_task_window_size > 0 && (new_task_window_size & (new_task_window_size - 1)) == 0); always_assert(new_pool_size > 0); - buckets.reset(new (std::nothrow) PTO2TensorMapEntry *[new_num_buckets]); - entry_pool.reset(new (std::nothrow) PTO2TensorMapEntry[new_pool_size]); - free_entry_list.reset(new (std::nothrow) PTO2TensorMapEntry *[new_pool_size]); - task_entry_heads.reset(new (std::nothrow) PTO2TensorMapEntry *[new_task_window_size]); + buckets.reset(new (std::nothrow) ChipTensorMapEntry *[new_num_buckets]); + entry_pool.reset(new (std::nothrow) ChipTensorMapEntry[new_pool_size]); + free_entry_list.reset(new (std::nothrow) ChipTensorMapEntry *[new_pool_size]); + task_entry_heads.reset(new (std::nothrow) ChipTensorMapEntry *[new_task_window_size]); if (buckets == nullptr || entry_pool == nullptr || free_entry_list == nullptr || task_entry_heads == nullptr) { LOG_ERROR( "TensorMap init failed (buckets=%d, pool=%d, window=%d)", new_num_buckets, new_pool_size, @@ -105,7 +105,7 @@ bool PTO2TensorMap::init(int32_t new_num_buckets, int32_t new_pool_size, int32_t return true; } -void PTO2TensorMap::reset() { +void ChipTensorMap::reset() { always_assert(buckets != nullptr && entry_pool != nullptr && task_entry_heads != nullptr); for (int32_t i = 0; i < num_buckets; i++) { @@ -121,15 +121,15 @@ void PTO2TensorMap::reset() { free_num = 0; } -bool PTO2TensorMap::init_default(int32_t new_task_window_size) { - return init(PTO2_TENSORMAP_NUM_BUCKETS, PTO2_TENSORMAP_POOL_SIZE, new_task_window_size); +bool ChipTensorMap::init_default(int32_t new_task_window_size) { + return init(CHIP_TENSORMAP_NUM_BUCKETS, CHIP_TENSORMAP_POOL_SIZE, new_task_window_size); } // ============================================================================= // Debug Utilities // ============================================================================= -void PTO2TensorMap::print_stats() { +void ChipTensorMap::print_stats() { int32_t valid = 0; int32_t empty_buckets = 0; int32_t max_chain = 0; @@ -178,7 +178,7 @@ void PTO2TensorMap::print_stats() { LOG_DEBUG("============================"); } -int32_t PTO2TensorMap::valid_count() { +int32_t ChipTensorMap::valid_count() { int32_t count = 0; // Init-on-write: only [0, next_entry_idx) slots have ever been allocated and @@ -196,8 +196,8 @@ int32_t PTO2TensorMap::valid_count() { // TensorMap Lookup Profiling // ============================================================================= #if SIMPLER_TENSORMAP_PROFILING -PTO2TensorMapProfilingData pto2_tensormap_get_profiling() { - PTO2TensorMapProfilingData d; +ChipTensorMapProfilingData chip_tensormap_get_profiling() { + ChipTensorMapProfilingData d; d.lookup_chain_total = g_lookup_chain_total; d.lookup_count = g_lookup_count; d.lookup_chain_max = g_lookup_chain_max; diff --git a/src/a2a3/runtime/host_build_graph/runtime/tensormap.h b/src/a2a3/runtime/host_build_graph/runtime/tensormap.h index 158681b33e..0deee432df 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/tensormap.h +++ b/src/a2a3/runtime/host_build_graph/runtime/tensormap.h @@ -111,10 +111,10 @@ extern uint64_t g_insert_count; * * Entry size: 128B (2 cache lines), matches ChipTensor. */ -struct alignas(64) PTO2TensorMapEntry { +struct alignas(64) ChipTensorMapEntry { // === Cache line 1 (64B) — lookup hot path; mirrors ChipTensor line 1 from byte 16 === uint64_t buffer_addr; // 8B [0, 8): tensor base address (hash key, mirrors ChipTensor::buffer.addr) - PTO2TensorMapEntry + ChipTensorMapEntry *next_in_bucket; // 8B [8, 16): next entry in hash bucket chain (overlays ChipTensor::buffer.size) TaskId producer_task_id; // 8B [16,24): mirrors ChipTensor::owner_task_id slot uint64_t start_offset; // 8B [24,32): mirrors ChipTensor::start_offset (element offset) @@ -127,9 +127,9 @@ struct alignas(64) PTO2TensorMapEntry { uint32_t shapes[MAX_TENSOR_DIMS]; // 20B [44,64): mirrors ChipTensor::shapes // === Cache line 2 (64B) — chain manipulation + non-contiguous overlap data === - PTO2TensorMapEntry *prev_in_bucket; // 8B [64, 72) - PTO2TensorMapEntry *next_in_task; // 8B [72, 80) - PTO2TensorMapEntry *prev_in_task; // 8B [80, 88) + ChipTensorMapEntry *prev_in_bucket; // 8B [64, 72) + ChipTensorMapEntry *next_in_task; // 8B [72, 80) + ChipTensorMapEntry *prev_in_task; // 8B [80, 88) int32_t bucket_index; // 4B [88, 92): -1 when unlinked uint32_t __padding2__; // 4B [92, 96) uint64_t extent_elem_cache; // 8B [96,104): non-contiguous extent (mirrors ChipTensor) @@ -323,18 +323,18 @@ struct alignas(64) PTO2TensorMapEntry { } }; -static_assert(sizeof(PTO2TensorMapEntry) == 128, "TensorMapEntry must be exactly 2 cache lines (128 bytes)"); -static_assert(offsetof(PTO2TensorMapEntry, buffer_addr) == offsetof(ChipTensor, buffer.addr)); -static_assert(offsetof(PTO2TensorMapEntry, producer_task_id) == offsetof(ChipTensor, owner_task_id)); -static_assert(offsetof(PTO2TensorMapEntry, start_offset) == offsetof(ChipTensor, start_offset)); -static_assert(offsetof(PTO2TensorMapEntry, version) == offsetof(ChipTensor, version)); -static_assert(offsetof(PTO2TensorMapEntry, ndims) == offsetof(ChipTensor, ndims)); -static_assert(offsetof(PTO2TensorMapEntry, dtype) == offsetof(ChipTensor, dtype)); -static_assert(offsetof(PTO2TensorMapEntry, manual_dep) == offsetof(ChipTensor, manual_dep)); -static_assert(offsetof(PTO2TensorMapEntry, is_contiguous) == offsetof(ChipTensor, is_contiguous)); -static_assert(offsetof(PTO2TensorMapEntry, shapes) == offsetof(ChipTensor, shapes)); +static_assert(sizeof(ChipTensorMapEntry) == 128, "TensorMapEntry must be exactly 2 cache lines (128 bytes)"); +static_assert(offsetof(ChipTensorMapEntry, buffer_addr) == offsetof(ChipTensor, buffer.addr)); +static_assert(offsetof(ChipTensorMapEntry, producer_task_id) == offsetof(ChipTensor, owner_task_id)); +static_assert(offsetof(ChipTensorMapEntry, start_offset) == offsetof(ChipTensor, start_offset)); +static_assert(offsetof(ChipTensorMapEntry, version) == offsetof(ChipTensor, version)); +static_assert(offsetof(ChipTensorMapEntry, ndims) == offsetof(ChipTensor, ndims)); +static_assert(offsetof(ChipTensorMapEntry, dtype) == offsetof(ChipTensor, dtype)); +static_assert(offsetof(ChipTensorMapEntry, manual_dep) == offsetof(ChipTensor, manual_dep)); +static_assert(offsetof(ChipTensorMapEntry, is_contiguous) == offsetof(ChipTensor, is_contiguous)); +static_assert(offsetof(ChipTensorMapEntry, shapes) == offsetof(ChipTensor, shapes)); static_assert( - offsetof(PTO2TensorMapEntry, prev_in_bucket) == 64, "TensorMapEntry must be exactly 2 cache lines (128 bytes)" + offsetof(ChipTensorMapEntry, prev_in_bucket) == 64, "TensorMapEntry must be exactly 2 cache lines (128 bytes)" ); // ============================================================================= @@ -347,23 +347,23 @@ static_assert( * Hash table with a fixed-capacity entry pool and no watermark invalidation. * Owns its four arrays; init() sizes them and leaves the map empty. */ -struct PTO2TensorMap { +struct ChipTensorMap { // Hash table buckets (fixed size, power of 2). An empty bucket is nullptr. - std::unique_ptr buckets; + std::unique_ptr buckets; int32_t num_buckets{0}; // Must be power of 2 for fast modulo // Entry pool: bump allocation plus reuse of explicitly removed entries. A // linked entry is reached by pointer, so the pool is allocated once at // pool_size and never resized. - std::unique_ptr entry_pool; - std::unique_ptr free_entry_list; + std::unique_ptr entry_pool; + std::unique_ptr free_entry_list; int32_t pool_size{0}; // Total pool capacity int32_t next_entry_idx{0}; // id when next entry insert int32_t free_num{0}; // free entry number in entry pool // Per-task entry tracking for O(1) unlinking of covered producers. // Indexed by [local_id & (task_window_size - 1)] - std::unique_ptr task_entry_heads; + std::unique_ptr task_entry_heads; int32_t task_window_size{0}; // Task window size (for slot masking) uint32_t get_task_local_id_slot(uint32_t task_local_id) const { return task_local_id & (task_window_size - 1); } @@ -377,14 +377,14 @@ struct PTO2TensorMap { // new_entry allocates a slot and initializes only its linkage (bucket_index // and the four link pointers) to the clean unlinked state; insert() assigns // the tensor attributes and producer_task_id. - PTO2TensorMapEntry *new_entry() { + ChipTensorMapEntry *new_entry() { if (free_num > 0) { - PTO2TensorMapEntry *res = free_entry_list[--free_num]; + ChipTensorMapEntry *res = free_entry_list[--free_num]; debug_assert(res->bucket_index == -1); return res; } always_assert(next_entry_idx < pool_size); - PTO2TensorMapEntry *res = &entry_pool[next_entry_idx++]; + ChipTensorMapEntry *res = &entry_pool[next_entry_idx++]; // Init-on-write: the pool is not pre-zeroed (init() skips // the O(pool_size) memset), so put this fresh slot into the same clean // unlinked state free_entry() leaves recycled slots in. The insert path @@ -398,7 +398,7 @@ struct PTO2TensorMap { return res; } - void free_entry(PTO2TensorMapEntry &entry) { + void free_entry(ChipTensorMapEntry &entry) { always_assert(entry.bucket_index != -1); // must still be in a bucket // Update predecessor's next pointer (O(1) via prev_in_bucket) @@ -459,8 +459,8 @@ struct PTO2TensorMap { void reset(); /** - * Same as init() with default sizes (PTO2_TENSORMAP_NUM_BUCKETS, - * PTO2_TENSORMAP_POOL_SIZE). + * Same as init() with default sizes (CHIP_TENSORMAP_NUM_BUCKETS, + * CHIP_TENSORMAP_POOL_SIZE). */ bool init_default(int32_t new_task_window_size); @@ -470,7 +470,7 @@ struct PTO2TensorMap { * Searches the hash table for matching regions and invokes the callback * for each overlapping entry. * - * The callback receives (PTO2TensorMapEntry &, OverlapStatus) and should + * The callback receives (ChipTensorMapEntry &, OverlapStatus) and should * return true to continue iteration, false to stop early. It is safe for * the callback to call remove_entry() on the current entry: next_in_bucket * is latched before invocation. @@ -481,7 +481,7 @@ struct PTO2TensorMap { template void lookup(const ChipTensor &tensor, Fn &&on_match) { uint32_t bucket_index = hash(tensor.buffer.addr); - PTO2TensorMapEntry *cur_entry = buckets[bucket_index]; + ChipTensorMapEntry *cur_entry = buckets[bucket_index]; #if SIMPLER_TENSORMAP_PROFILING g_lookup_count++; @@ -489,7 +489,7 @@ struct PTO2TensorMap { #endif while (cur_entry != nullptr) { - PTO2TensorMapEntry *next_entry = cur_entry->next_in_bucket; + ChipTensorMapEntry *next_entry = cur_entry->next_in_bucket; #if SIMPLER_TENSORMAP_PROFILING chain_len++; @@ -536,7 +536,7 @@ struct PTO2TensorMap { * @param producer_task_id Task ID of producer */ void insert(const ChipTensor &tensor, TaskId producer_task_id) { - PTO2TensorMapEntry *entry = new_entry(); + ChipTensorMapEntry *entry = new_entry(); entry->copy_from_tensor(tensor); link_entry(entry, tensor.buffer.addr, producer_task_id); } @@ -561,7 +561,7 @@ struct PTO2TensorMap { /** * Link an initialized entry into bucket and task chains. */ - void link_entry(PTO2TensorMapEntry *entry, uint64_t addr, TaskId producer_task_id) { + void link_entry(ChipTensorMapEntry *entry, uint64_t addr, TaskId producer_task_id) { #if SIMPLER_TENSORMAP_PROFILING g_insert_count++; #endif @@ -589,7 +589,7 @@ struct PTO2TensorMap { task_entry_heads[task_slot] = entry; } - void remove_entry(PTO2TensorMapEntry &entry) { + void remove_entry(ChipTensorMapEntry &entry) { remove_from_task(entry); free_entry(entry); } @@ -599,7 +599,7 @@ struct PTO2TensorMap { * computation calls this before freeing a producer made redundant by a * covering input. */ - void remove_from_task(PTO2TensorMapEntry &entry) { + void remove_from_task(ChipTensorMapEntry &entry) { always_assert(entry.bucket_index != -1); // must still be in a bucket // Update predecessor's next pointer (O(1) via prev_in_task) if (entry.prev_in_task == nullptr) { @@ -636,7 +636,7 @@ struct PTO2TensorMap { }; #if SIMPLER_TENSORMAP_PROFILING -struct PTO2TensorMapProfilingData { +struct ChipTensorMapProfilingData { uint64_t lookup_chain_total; uint64_t lookup_count; int32_t lookup_chain_max; @@ -645,5 +645,5 @@ struct PTO2TensorMapProfilingData { uint64_t insert_count; }; -PTO2TensorMapProfilingData pto2_tensormap_get_profiling(); +ChipTensorMapProfilingData chip_tensormap_get_profiling(); #endif diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index e567347f76..12ba8680e2 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -778,7 +778,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { ); #if SIMPLER_TENSORMAP_PROFILING - PTO2TensorMapProfilingData tp = pto2_tensormap_get_profiling(); + ChipTensorMapProfilingData tp = chip_tensormap_get_profiling(); LOG_INFO("Thread %d: === TensorMap Lookup Stats ===", thread_idx); LOG_INFO( "Thread %d: lookups : %" PRIu64 ", inserts: %" PRIu64 "", thread_idx, diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 75588a7a44..daa94eae31 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -41,7 +41,7 @@ Type changes: | Field | Before | After | | ----- | ------ | ----- | | `TaskDescriptor.task_id` | `int32_t` | `TaskId` | -| `PTO2TensorMapEntry.producer_task_id` | `int32_t` | `TaskId` | +| `ChipTensorMapEntry.producer_task_id` | `int32_t` | `TaskId` | | `ChipTaskSlotState.ring_id` | N/A | `uint8_t` (new, denormalized for fast access) | ## 4. Data Structures @@ -144,17 +144,17 @@ RingSchedState ring_sched_states[CHIP_MAX_RING_DEPTH]; `slot_states`, `task_window_size`, and `task_window_mask` are no longer duplicated — callers access them via `ring->get_slot_state_by_*()` and other ring header accessors. The ring pointer shares cache line 0 with `last_task_alive` and `advance_lock`. -### 4.6 PTO2TensorMap (modified) +### 4.6 ChipTensorMap (modified) ```cpp -PTO2TensorMapEntry** task_entry_heads[CHIP_MAX_RING_DEPTH]; +ChipTensorMapEntry** task_entry_heads[CHIP_MAX_RING_DEPTH]; int64_t last_task_alives[CHIP_MAX_RING_DEPTH]; ``` Entry validity checks and `cleanup_retired` operate per-ring: ```cpp -bool entry_valid(const PTO2TensorMapEntry& e) { +bool entry_valid(const ChipTensorMapEntry& e) { int32_t ring = e.producer_task_id.ring(); int32_t local = e.producer_task_id.local(); return local >= last_task_alives[ring]; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 22de58bf83..7351701b4e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -316,7 +316,7 @@ Unlike the Task Ring and Heap Ring, TensorMap entries are **not** managed by a r 2. **Bump allocation**: if free list is empty, `entry_pool[next_entry_idx++]` allocates from the end of the pool. 3. **Blocking reclaim**: if the pool is short of the inserts a task needs, the orchestrator's `ensure_tensormap_capacity` reads the latest `last_task_alive` for every ring and calls `reclaim_retired_all` (`cleanup_retired` per ring) to batch-free entries belonging to retired tasks, returning them to the free list, before the inserts proceed. -This design avoids the complexity of ring-based wrapping while still being bounded by `PTO2_TENSORMAP_POOL_SIZE` (default 65536 entries). +This design avoids the complexity of ring-based wrapping while still being bounded by `CHIP_TENSORMAP_POOL_SIZE` (default 65536 entries). ### 5.4 Stale Entry Cleanup: Three-Layer Defense @@ -329,13 +329,13 @@ Three complementary mechanisms achieve this: **Layer 1 — Chain Truncation during Lookup** (lazy, per-bucket): -Since `insert` always prepends to the bucket head, entries in each bucket chain are in **descending task_id order**. When `PTO2TensorMap::lookup` encounters the first stale entry (`producer_task_id < last_task_alive`), all subsequent entries in the chain are guaranteed stale too. The entire tail is truncated in one operation using `prev_in_bucket` pointers for O(1) unlinking. +Since `insert` always prepends to the bucket head, entries in each bucket chain are in **descending task_id order**. When `ChipTensorMap::lookup` encounters the first stale entry (`producer_task_id < last_task_alive`), all subsequent entries in the chain are guaranteed stale too. The entire tail is truncated in one operation using `prev_in_bucket` pointers for O(1) unlinking. This guarantees lookup only traverses valid entries — O(valid_entries_in_bucket), not O(total_entries). **Layer 2 — Periodic Batch Cleanup** (`cleanup_retired`, per-task): -Every time the orchestrator submits a task (Step 0 of `OrchestratorState::submit_task`), it calls `PTO2TensorMap::sync_tensormap`. When `last_task_alive` has advanced by more than `PTO2_TENSORMAP_CLEANUP_INTERVAL` (default 64) tasks since the last cleanup, `PTO2TensorMap::cleanup_retired` runs: +Every time the orchestrator submits a task (Step 0 of `OrchestratorState::submit_task`), it calls `ChipTensorMap::sync_tensormap`. When `last_task_alive` has advanced by more than `CHIP_TENSORMAP_CLEANUP_INTERVAL` (default 64) tasks since the last cleanup, `ChipTensorMap::cleanup_retired` runs: This uses the **per-task entry chain** (`task_entry_head[task_slot]`) — each task's entries are doubly-linked together at insert time via `next_in_task`/`prev_in_task`. A slot's chain can hold more than one task's entries: a task at `local_id + N * window` reuses the slot and prepends to the chain already there, and cleanup can lag that reuse. Cleanup therefore walks the chain and frees only the entries whose `producer_task_id` matches the retiring task, unlinking each and leaving the rest linked — O(entries_in_slot), with no scan of the entire pool or all buckets. Freed entries are returned to `free_entry_list` for immediate reuse. @@ -359,9 +359,9 @@ In steady state, the number of valid TensorMap entries ≈ `active_tasks × avg_ When `OrchestratorState::submit_task` processes parameters: -1. **INPUT/INOUT**: `PTO2TensorMap::lookup` searches for overlapping producers (with chain truncation) +1. **INPUT/INOUT**: `ChipTensorMap::lookup` searches for overlapping producers (with chain truncation) 2. For each producer found: `append_fanin_or_fail` adds the dependency -3. **OUTPUT/INOUT**: `PTO2TensorMap::insert` registers the current task as the new producer at bucket head +3. **OUTPUT/INOUT**: `ChipTensorMap::insert` registers the current task as the new producer at bucket head 4. Stale entries are pruned lazily during lookup (Layer 1) and periodically by cleanup (Layer 2) --- @@ -431,7 +431,7 @@ Key members: | Step | Operation | | ---- | --------- | -| 0 | `PTO2TensorMap::sync_tensormap` — prune stale TensorMap entries | +| 0 | `ChipTensorMap::sync_tensormap` — prune stale TensorMap entries | | 1 | `PTO2TaskAllocator::alloc` — allocate task slot (may block on flow control) | | 2 | Initialize task descriptor + slot state, copy parameters | | 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `PTO2FaninBuilder` | diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp index 1ea1f35b67..0e3ce5513f 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp @@ -12,10 +12,10 @@ /** * @file dep_gen_replay.cpp * @brief Replay in-memory DepGenRecord stream → deps.json (strided tensor - * representation, tensor-annotated) via a host-resident PTO2TensorMap, + * representation, tensor-annotated) via a host-resident ChipTensorMap, * with a differential check against the runtime template `compute_task_fanin`. * - * Two passes run per record against two parallel PTO2TensorMap instances that + * Two passes run per record against two parallel ChipTensorMap instances that * evolve in lockstep: * * ORACLE pass (read-only contract): @@ -29,7 +29,7 @@ * ANNOT pass (this file's feature): * Inlines the same STEP A (creator retention) + STEP B (tensormap lookup) * against `tm_annot`, but the callback fires with the full - * `PTO2TensorMapEntry&` + the consumer ChipTensor* + the arg index, so the + * `ChipTensorMapEntry&` + the consumer ChipTensor* + the arg index, so the * replay can record per-edge tensor metadata (producer/consumer * shape/offset, dtype, version). * @@ -295,9 +295,9 @@ void fill_consumer(EdgeAnnot &e, const ChipTensor &t) { } } -// Copy a PTO2TensorMapEntry's slice description into an EdgeAnnot's producer_* +// Copy a ChipTensorMapEntry's slice description into an EdgeAnnot's producer_* // fields. Only called from the TENSORMAP emit path. -void fill_producer(EdgeAnnot &e, const PTO2TensorMapEntry &entry) { +void fill_producer(EdgeAnnot &e, const ChipTensorMapEntry &entry) { e.producer_ndims = entry.ndims; e.producer_start_offset = entry.start_offset; for (uint32_t i = 0; i < entry.ndims && i < MAX_TENSOR_DIMS; i++) { @@ -425,7 +425,7 @@ bool write_deps_json( template void annot_pass( - const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_manual_scope, EmitCreator emit_creator, + const DepInputs &inputs, ChipTensorMap &tensor_map, bool in_manual_scope, EmitCreator emit_creator, EmitTM emit_tensormap ) { if (in_manual_scope) { @@ -452,7 +452,7 @@ void annot_pass( continue; } - tensor_map.lookup(*tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus overlap_status) -> bool { + tensor_map.lookup(*tensor, [&](ChipTensorMapEntry &entry, OverlapStatus overlap_status) -> bool { emit_tensormap(entry.producer_task_id, i, *tensor, entry, overlap_status); if (ptype == TensorArgType::INOUT && overlap_status == OverlapStatus::COVERED) { tensor_map.remove_entry(entry); @@ -498,12 +498,12 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c int32_t output_count = count_outputs(records, num_records); int32_t pool_size = output_count + (output_count / 10) + 64; - if (pool_size < PTO2_TENSORMAP_POOL_SIZE) { - pool_size = PTO2_TENSORMAP_POOL_SIZE; + if (pool_size < CHIP_TENSORMAP_POOL_SIZE) { + pool_size = CHIP_TENSORMAP_POOL_SIZE; } - PTO2TensorMap tm_oracle; - PTO2TensorMap tm_annot; + ChipTensorMap tm_oracle; + ChipTensorMap tm_annot; std::memset(&tm_oracle, 0, sizeof(tm_oracle)); std::memset(&tm_annot, 0, sizeof(tm_annot)); @@ -512,12 +512,12 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c DeviceArena replay_arena; auto oracle_layout = - PTO2TensorMap::reserve_layout(replay_arena, PTO2_TENSORMAP_NUM_BUCKETS, pool_size, task_window_sizes); + ChipTensorMap::reserve_layout(replay_arena, CHIP_TENSORMAP_NUM_BUCKETS, pool_size, task_window_sizes); auto annot_layout = - PTO2TensorMap::reserve_layout(replay_arena, PTO2_TENSORMAP_NUM_BUCKETS, pool_size, task_window_sizes); + ChipTensorMap::reserve_layout(replay_arena, CHIP_TENSORMAP_NUM_BUCKETS, pool_size, task_window_sizes); if (replay_arena.commit() == nullptr || !tm_oracle.init_data_from_layout(oracle_layout, replay_arena) || !tm_annot.init_data_from_layout(annot_layout, replay_arena)) { - LOG_ERROR("dep_gen replay: tensormap.init failed (buckets=%d, pool=%d)", PTO2_TENSORMAP_NUM_BUCKETS, pool_size); + LOG_ERROR("dep_gen replay: tensormap.init failed (buckets=%d, pool=%d)", CHIP_TENSORMAP_NUM_BUCKETS, pool_size); return -3; } // Replay tensormaps live entirely on host; only arena-internal pointer @@ -658,7 +658,7 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c // Register tasks[] entry (with per-arg slot info) and any unseen // tensors[] entries up-front. ChipTensors are registered from the // consumer-side blob so raw_shapes / dtype are populated (the - // producer-side PTO2TensorMapEntry drops raw_shapes to fit in two + // producer-side ChipTensorMapEntry drops raw_shapes to fit in two // cache lines). TaskTableEntry task_entry; task_entry.task_id = rec.task_id; @@ -753,7 +753,7 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c annot_edges.push_back(e); }, // emit_tensormap(producer, arg_idx, consumer_tensor, entry, status) - [&](TaskId producer, int32_t arg_idx, const ChipTensor &consumer, const PTO2TensorMapEntry &entry, + [&](TaskId producer, int32_t arg_idx, const ChipTensor &consumer, const ChipTensorMapEntry &entry, OverlapStatus status) { // Per-(succ, arg_idx, producer_buffer_addr, producer_version) // dedup gives us "the same producer slice fired twice for the diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h b/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h index a9860cc053..3bb89ee315 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h @@ -15,7 +15,7 @@ * * Takes the records the host collector drained from the device ring buffer * (``DepGenCollector::records()``) and runs them back through a host-resident - * PTO2TensorMap using the same ``compute_task_fanin`` / ``register_task_outputs`` + * ChipTensorMap using the same ``compute_task_fanin`` / ``register_task_outputs`` * primitives the device orchestrator uses, emitting the full * predecessor → successor edge list to deps.json. * diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dep_compute.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dep_compute.h index 0bb98f7f6c..201b654cbb 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dep_compute.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dep_compute.h @@ -82,7 +82,7 @@ struct DepInputs { */ template [[nodiscard]] inline bool -compute_task_fanin(const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_manual_scope, Emit emit) { +compute_task_fanin(const DepInputs &inputs, ChipTensorMap &tensor_map, bool in_manual_scope, Emit emit) { if (in_manual_scope) { return true; } @@ -115,7 +115,7 @@ compute_task_fanin(const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_m } bool fatal = false; - tensor_map.lookup(*tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus overlap_status) -> bool { + tensor_map.lookup(*tensor, [&](ChipTensorMapEntry &entry, OverlapStatus overlap_status) -> bool { // Ordering-only (DEP_WAIT): a modifier only rewrote a buffer someone // else allocated, so its lifetime rides that allocator's creator edge, // not this modifier edge. Retention-safety invariant that makes this @@ -151,7 +151,7 @@ compute_task_fanin(const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_m * No-op when in_manual_scope. */ inline void -register_task_outputs(const DepInputs &inputs, TaskId task_id, PTO2TensorMap &tensor_map, bool in_manual_scope) { +register_task_outputs(const DepInputs &inputs, TaskId task_id, ChipTensorMap &tensor_map, bool in_manual_scope) { if (in_manual_scope) { return; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 3e5d39019c..89eab4b76f 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -790,7 +790,7 @@ void OrchestratorState::end_scope() { // and bail. Returns false on deadlock or on a fatal already latched by another // party. Cold path — the fast path returns immediately when the pool has room. static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { - PTO2TensorMap &tm = orch->tensor_map; + ChipTensorMap &tm = orch->tensor_map; if (tm.free_entries() >= needed) { return true; } @@ -865,7 +865,7 @@ static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { LOG_ERROR(" still move on rings with no registered outputs). Check TaskRing"); LOG_ERROR(" diagnostics for the stalled producer."); LOG_ERROR("Solution:"); - LOG_ERROR(" Increase PTO2_TENSORMAP_POOL_SIZE (current: %d).", tm.pool_capacity()); + LOG_ERROR(" Increase CHIP_TENSORMAP_POOL_SIZE (current: %d).", tm.pool_capacity()); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_TENSORMAP_OVERFLOW); return false; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 9a8ecd5e88..0b5949dab4 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -40,14 +40,14 @@ /** * Layout descriptor produced by OrchestratorState::reserve_layout(). Holds * arena offsets for every sub-region the orchestrator owns (per-ring fanin - * pools, scope arrays, plus the nested PTO2TensorMap layout). + * pools, scope arrays, plus the nested ChipTensorMap layout). */ struct OrchestratorLayout { size_t off_fanin_pool[CHIP_MAX_RING_DEPTH]; size_t off_fanin_seen_epoch[CHIP_MAX_RING_DEPTH]; size_t off_scope_tasks; size_t off_scope_begins; - PTO2TensorMapLayout tensor_map; + ChipTensorMapLayout tensor_map; int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]; int32_t scope_tasks_cap; uint64_t scope_stack_capacity; @@ -72,7 +72,7 @@ struct OrchestratorState { uint32_t fanin_seen_current_epoch{1}; // === TENSOR MAP (Private) === - PTO2TensorMap tensor_map; // Producer lookup + ChipTensorMap tensor_map; // Producer lookup // === SCOPE STACK (Private) === // Single contiguous buffer of task IDs, partitioned by scope level. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 8ddd8c3827..83f3671398 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -196,7 +196,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo } // Step B: modifier writer lookup (OverlapMap), direct callback - orch.tensor_map.lookup(tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus) -> bool { + orch.tensor_map.lookup(tensor, [&](ChipTensorMapEntry &entry, OverlapStatus) -> bool { TaskId pid = entry.producer_task_id; auto &s = orch.sm_header->rings[pid.ring()].get_slot_state_by_task_id(pid.local()); try_push(s); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 241281744b..a16995a418 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -71,8 +71,8 @@ // Memory pools (per-ring defaults; total = value × CHIP_MAX_RING_DEPTH) #define PTO2_HEAP_SIZE (256 * 1024 * 1024) // 256MB per ring (1GB total) #define PTO2_DEP_LIST_POOL_SIZE 16384 // Per-ring dependency list pool entries -#define PTO2_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool -#define PTO2_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) +#define CHIP_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool +#define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) // Scope management #define PTO2_MAX_SCOPE_DEPTH 64 // Maximum nesting depth @@ -102,7 +102,7 @@ #define PTO2_DEP_DEGREE_DEBUG_THRESHOLD 16 // TensorMap cleanup interval -#define PTO2_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks +#define CHIP_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks #define PTO2_DEP_POOL_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index f515786fcb..8a01aa0f03 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -338,7 +338,7 @@ OrchestratorLayout OrchestratorState::reserve_layout( arena.reserve(static_cast(layout.scope_tasks_cap) * sizeof(uintptr_t), alignof(ChipTaskSlotState *)); layout.off_scope_begins = arena.reserve(static_cast(layout.scope_stack_capacity) * sizeof(int32_t), alignof(int32_t)); - layout.tensor_map = PTO2TensorMap::reserve_layout_default(arena, task_window_sizes); + layout.tensor_map = ChipTensorMap::reserve_layout_default(arena, task_window_sizes); return layout; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/tensormap.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/tensormap.cpp index 09ca4a5dd2..dfa35caace 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/tensormap.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/tensormap.cpp @@ -47,14 +47,14 @@ uint64_t g_insert_count = 0; // Initialization and Destruction // ============================================================================= -PTO2TensorMapLayout PTO2TensorMap::reserve_layout( +ChipTensorMapLayout ChipTensorMap::reserve_layout( DeviceArena &arena, int32_t new_num_buckets, int32_t new_pool_size, const int32_t new_task_window_sizes[CHIP_MAX_RING_DEPTH] ) { // num_buckets must be a power of two for the hash truncation to work. always_assert((new_num_buckets & (new_num_buckets - 1)) == 0); - PTO2TensorMapLayout layout{}; + ChipTensorMapLayout layout{}; layout.num_buckets = new_num_buckets; layout.pool_size = new_pool_size; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -62,17 +62,17 @@ PTO2TensorMapLayout PTO2TensorMap::reserve_layout( } layout.off_buckets = arena.reserve( - static_cast(new_num_buckets) * sizeof(PTO2TensorMapEntry *), alignof(PTO2TensorMapEntry *) + static_cast(new_num_buckets) * sizeof(ChipTensorMapEntry *), alignof(ChipTensorMapEntry *) ); layout.off_bucket_epochs = arena.reserve(static_cast(new_num_buckets) * sizeof(uint32_t), alignof(uint32_t)); layout.off_entry_pool = - arena.reserve(static_cast(new_pool_size) * sizeof(PTO2TensorMapEntry), alignof(PTO2TensorMapEntry)); + arena.reserve(static_cast(new_pool_size) * sizeof(ChipTensorMapEntry), alignof(ChipTensorMapEntry)); layout.off_free_entry_list = - arena.reserve(static_cast(new_pool_size) * sizeof(PTO2TensorMapEntry *), alignof(PTO2TensorMapEntry *)); + arena.reserve(static_cast(new_pool_size) * sizeof(ChipTensorMapEntry *), alignof(ChipTensorMapEntry *)); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { layout.off_task_entry_heads[r] = arena.reserve( - static_cast(new_task_window_sizes[r]) * sizeof(PTO2TensorMapEntry *), alignof(PTO2TensorMapEntry *) + static_cast(new_task_window_sizes[r]) * sizeof(ChipTensorMapEntry *), alignof(ChipTensorMapEntry *) ); layout.off_task_entry_head_epochs[r] = arena.reserve(static_cast(new_task_window_sizes[r]) * sizeof(uint32_t), alignof(uint32_t)); @@ -80,21 +80,21 @@ PTO2TensorMapLayout PTO2TensorMap::reserve_layout( return layout; } -PTO2TensorMapLayout -PTO2TensorMap::reserve_layout_default(DeviceArena &arena, const int32_t new_task_window_sizes[CHIP_MAX_RING_DEPTH]) { - return reserve_layout(arena, PTO2_TENSORMAP_NUM_BUCKETS, PTO2_TENSORMAP_POOL_SIZE, new_task_window_sizes); +ChipTensorMapLayout +ChipTensorMap::reserve_layout_default(DeviceArena &arena, const int32_t new_task_window_sizes[CHIP_MAX_RING_DEPTH]) { + return reserve_layout(arena, CHIP_TENSORMAP_NUM_BUCKETS, CHIP_TENSORMAP_POOL_SIZE, new_task_window_sizes); } -bool PTO2TensorMap::init_data_from_layout(const PTO2TensorMapLayout &layout, DeviceArena &arena) { +bool ChipTensorMap::init_data_from_layout(const ChipTensorMapLayout &layout, DeviceArena &arena) { num_buckets = layout.num_buckets; pool_size = layout.pool_size; // Address arena regions for data writes; do not store these in struct // fields (wire_arena_pointers does that). - auto *buckets_arena = static_cast(arena.region_ptr(layout.off_buckets)); + auto *buckets_arena = static_cast(arena.region_ptr(layout.off_buckets)); auto *bucket_epochs_arena = static_cast(arena.region_ptr(layout.off_bucket_epochs)); - auto *entry_pool_arena = static_cast(arena.region_ptr(layout.off_entry_pool)); - auto *free_list_arena = static_cast(arena.region_ptr(layout.off_free_entry_list)); + auto *entry_pool_arena = static_cast(arena.region_ptr(layout.off_entry_pool)); + auto *free_list_arena = static_cast(arena.region_ptr(layout.off_free_entry_list)); // buckets[]: empty == nullptr. for (int32_t i = 0; i < num_buckets; i++) { @@ -105,7 +105,7 @@ bool PTO2TensorMap::init_data_from_layout(const PTO2TensorMapLayout &layout, Dev // entry_pool: zero-init equivalent to the previous calloc(entry_pool, ...). // The pool's persistent invariant after init is "bucket_index == -1 means // not linked", set explicitly below. - memset(entry_pool_arena, 0, static_cast(pool_size) * sizeof(PTO2TensorMapEntry)); + memset(entry_pool_arena, 0, static_cast(pool_size) * sizeof(ChipTensorMapEntry)); for (int32_t i = 0; i < pool_size; i++) { entry_pool_arena[i].bucket_index = -1; entry_pool_arena[i].next_in_bucket = nullptr; @@ -117,13 +117,13 @@ bool PTO2TensorMap::init_data_from_layout(const PTO2TensorMapLayout &layout, Dev // free_entry_list: zeroed (was calloc'd before); contents become meaningful // only after entries are freed back, so the body of the array stays as 0. - memset(free_list_arena, 0, static_cast(pool_size) * sizeof(PTO2TensorMapEntry *)); + memset(free_list_arena, 0, static_cast(pool_size) * sizeof(ChipTensorMapEntry *)); next_entry_idx = 0; free_num = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - auto *heads_arena = static_cast(arena.region_ptr(layout.off_task_entry_heads[r])); + auto *heads_arena = static_cast(arena.region_ptr(layout.off_task_entry_heads[r])); auto *head_epochs_arena = static_cast(arena.region_ptr(layout.off_task_entry_head_epochs[r])); for (int32_t i = 0; i < layout.task_window_sizes[r]; i++) { heads_arena[i] = nullptr; @@ -137,7 +137,7 @@ bool PTO2TensorMap::init_data_from_layout(const PTO2TensorMapLayout &layout, Dev return true; } -void PTO2TensorMap::reset_for_reuse(const PTO2TensorMapLayout &layout) { +void ChipTensorMap::reset_for_reuse(const ChipTensorMapLayout &layout) { num_buckets = layout.num_buckets; pool_size = layout.pool_size; next_entry_idx = 0; @@ -158,18 +158,18 @@ void PTO2TensorMap::reset_for_reuse(const PTO2TensorMapLayout &layout) { } } -void PTO2TensorMap::wire_arena_pointers(const PTO2TensorMapLayout &layout, DeviceArena &arena) { - buckets = static_cast(arena.region_ptr(layout.off_buckets)); +void ChipTensorMap::wire_arena_pointers(const ChipTensorMapLayout &layout, DeviceArena &arena) { + buckets = static_cast(arena.region_ptr(layout.off_buckets)); bucket_epochs = static_cast(arena.region_ptr(layout.off_bucket_epochs)); - entry_pool = static_cast(arena.region_ptr(layout.off_entry_pool)); - free_entry_list = static_cast(arena.region_ptr(layout.off_free_entry_list)); + entry_pool = static_cast(arena.region_ptr(layout.off_entry_pool)); + free_entry_list = static_cast(arena.region_ptr(layout.off_free_entry_list)); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - task_entry_heads[r] = static_cast(arena.region_ptr(layout.off_task_entry_heads[r])); + task_entry_heads[r] = static_cast(arena.region_ptr(layout.off_task_entry_heads[r])); task_entry_head_epochs[r] = static_cast(arena.region_ptr(layout.off_task_entry_head_epochs[r])); } } -void PTO2TensorMap::destroy() { +void ChipTensorMap::destroy() { // Arena owns the backing memory; here we only forget our pointers so any // stray post-destroy access trips a nullptr dereference instead of reading // a recycled allocation. @@ -187,7 +187,7 @@ void PTO2TensorMap::destroy() { // Debug Utilities // ============================================================================= -void PTO2TensorMap::print_stats() { +void ChipTensorMap::print_stats() { int32_t valid = 0; int32_t stale = 0; int32_t empty_buckets = 0; @@ -243,7 +243,7 @@ void PTO2TensorMap::print_stats() { LOG_DEBUG("============================"); } -int32_t PTO2TensorMap::valid_count() { +int32_t ChipTensorMap::valid_count() { int32_t count = 0; for (int32_t i = 0; i < pool_size; i++) { @@ -255,7 +255,7 @@ int32_t PTO2TensorMap::valid_count() { return count; } -void PTO2TensorMap::sync_tensormap(TaskId task_id, int32_t sm_last_task_alive) { +void ChipTensorMap::sync_tensormap(TaskId task_id, int32_t sm_last_task_alive) { auto ring_id = task_id.ring(); auto local_id = task_id.local(); sync_validity(ring_id, sm_last_task_alive); @@ -263,7 +263,7 @@ void PTO2TensorMap::sync_tensormap(TaskId task_id, int32_t sm_last_task_alive) { // Only attempt cleanup when last_task_alive has actually advanced; // otherwise cleanup_retired would empty-loop and we'd spin forever. auto overlap = get_task_local_id_slot(ring_id, local_id) == get_task_local_id_slot(ring_id, last_cleanup[ring_id]); - if (sm_last_task_alive - last_cleanup[ring_id] >= PTO2_TENSORMAP_CLEANUP_INTERVAL || overlap) { + if (sm_last_task_alive - last_cleanup[ring_id] >= CHIP_TENSORMAP_CLEANUP_INTERVAL || overlap) { cleanup_retired(ring_id, last_cleanup[ring_id], sm_last_task_alive); last_cleanup[ring_id] = sm_last_task_alive; } @@ -273,8 +273,8 @@ void PTO2TensorMap::sync_tensormap(TaskId task_id, int32_t sm_last_task_alive) { // TensorMap Lookup Profiling // ============================================================================= #if SIMPLER_TENSORMAP_PROFILING -PTO2TensorMapProfilingData pto2_tensormap_get_profiling() { - PTO2TensorMapProfilingData d; +ChipTensorMapProfilingData chip_tensormap_get_profiling() { + ChipTensorMapProfilingData d; d.lookup_chain_total = g_lookup_chain_total; d.lookup_count = g_lookup_count; d.lookup_chain_max = g_lookup_chain_max; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/tensormap.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/tensormap.h index fcfd2a143f..b9fbf156c4 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/tensormap.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/tensormap.h @@ -66,13 +66,13 @@ struct Segment { }; /** - * Layout descriptor produced by PTO2TensorMap::reserve_layout(). Stores the + * Layout descriptor produced by ChipTensorMap::reserve_layout(). Stores the * region offsets returned by DeviceArena::reserve() so init_data_from_layout() * can fetch the matching pointers after the arena is committed. * * All offsets are relative to the arena's base. */ -struct PTO2TensorMapLayout { +struct ChipTensorMapLayout { size_t off_buckets; size_t off_bucket_epochs; size_t off_entry_pool; @@ -124,10 +124,10 @@ extern uint64_t g_insert_count; * * Entry size: 128B (2 cache lines), matches ChipTensor. */ -struct alignas(64) PTO2TensorMapEntry { +struct alignas(64) ChipTensorMapEntry { // === Cache line 1 (64B) — lookup hot path; mirrors ChipTensor line 1 from byte 16 === uint64_t buffer_addr; // 8B [0, 8): tensor base address (hash key, mirrors ChipTensor::buffer.addr) - PTO2TensorMapEntry + ChipTensorMapEntry *next_in_bucket; // 8B [8, 16): next entry in hash bucket chain (overlays ChipTensor::buffer.size) TaskId producer_task_id; // 8B [16,24): mirrors ChipTensor::owner_task_id slot uint64_t start_offset; // 8B [24,32): mirrors ChipTensor::start_offset (element offset) @@ -140,9 +140,9 @@ struct alignas(64) PTO2TensorMapEntry { uint32_t shapes[MAX_TENSOR_DIMS]; // 20B [44,64): mirrors ChipTensor::shapes // === Cache line 2 (64B) — chain manipulation + non-contiguous overlap data === - PTO2TensorMapEntry *prev_in_bucket; // 8B [64, 72) - PTO2TensorMapEntry *next_in_task; // 8B [72, 80) - PTO2TensorMapEntry *prev_in_task; // 8B [80, 88) + ChipTensorMapEntry *prev_in_bucket; // 8B [64, 72) + ChipTensorMapEntry *next_in_task; // 8B [72, 80) + ChipTensorMapEntry *prev_in_task; // 8B [80, 88) int32_t bucket_index; // 4B [88, 92): -1 when unlinked uint32_t __padding2__; // 4B [92, 96) uint64_t extent_elem_cache; // 8B [96,104): non-contiguous extent (mirrors ChipTensor) @@ -336,18 +336,18 @@ struct alignas(64) PTO2TensorMapEntry { } }; -static_assert(sizeof(PTO2TensorMapEntry) == 128, "TensorMapEntry must be exactly 2 cache lines (128 bytes)"); -static_assert(offsetof(PTO2TensorMapEntry, buffer_addr) == offsetof(ChipTensor, buffer.addr)); -static_assert(offsetof(PTO2TensorMapEntry, producer_task_id) == offsetof(ChipTensor, owner_task_id)); -static_assert(offsetof(PTO2TensorMapEntry, start_offset) == offsetof(ChipTensor, start_offset)); -static_assert(offsetof(PTO2TensorMapEntry, version) == offsetof(ChipTensor, version)); -static_assert(offsetof(PTO2TensorMapEntry, ndims) == offsetof(ChipTensor, ndims)); -static_assert(offsetof(PTO2TensorMapEntry, dtype) == offsetof(ChipTensor, dtype)); -static_assert(offsetof(PTO2TensorMapEntry, manual_dep) == offsetof(ChipTensor, manual_dep)); -static_assert(offsetof(PTO2TensorMapEntry, is_contiguous) == offsetof(ChipTensor, is_contiguous)); -static_assert(offsetof(PTO2TensorMapEntry, shapes) == offsetof(ChipTensor, shapes)); +static_assert(sizeof(ChipTensorMapEntry) == 128, "TensorMapEntry must be exactly 2 cache lines (128 bytes)"); +static_assert(offsetof(ChipTensorMapEntry, buffer_addr) == offsetof(ChipTensor, buffer.addr)); +static_assert(offsetof(ChipTensorMapEntry, producer_task_id) == offsetof(ChipTensor, owner_task_id)); +static_assert(offsetof(ChipTensorMapEntry, start_offset) == offsetof(ChipTensor, start_offset)); +static_assert(offsetof(ChipTensorMapEntry, version) == offsetof(ChipTensor, version)); +static_assert(offsetof(ChipTensorMapEntry, ndims) == offsetof(ChipTensor, ndims)); +static_assert(offsetof(ChipTensorMapEntry, dtype) == offsetof(ChipTensor, dtype)); +static_assert(offsetof(ChipTensorMapEntry, manual_dep) == offsetof(ChipTensor, manual_dep)); +static_assert(offsetof(ChipTensorMapEntry, is_contiguous) == offsetof(ChipTensor, is_contiguous)); +static_assert(offsetof(ChipTensorMapEntry, shapes) == offsetof(ChipTensor, shapes)); static_assert( - offsetof(PTO2TensorMapEntry, prev_in_bucket) == 64, "TensorMapEntry must be exactly 2 cache lines (128 bytes)" + offsetof(ChipTensorMapEntry, prev_in_bucket) == 64, "TensorMapEntry must be exactly 2 cache lines (128 bytes)" ); // ============================================================================= @@ -359,22 +359,22 @@ static_assert( * * Hash table with ring buffer entry pool and lazy invalidation. */ -struct PTO2TensorMap { +struct ChipTensorMap { // Hash table buckets (fixed size, power of 2) - PTO2TensorMapEntry **buckets; // Array of offsets into entry_pool (-1 = empty) + ChipTensorMapEntry **buckets; // Array of offsets into entry_pool (-1 = empty) uint32_t *bucket_epochs; int32_t num_buckets; // Must be power of 2 for fast modulo // Entry pool as ring buffer - PTO2TensorMapEntry *entry_pool; // Ring buffer of entries - PTO2TensorMapEntry **free_entry_list; // free entry ids + ChipTensorMapEntry *entry_pool; // Ring buffer of entries + ChipTensorMapEntry **free_entry_list; // free entry ids int32_t pool_size; // Total pool capacity int32_t next_entry_idx; // id when next entry insert int32_t free_num; // free entry number in entry pool // Per-ring per-task entry tracking (for efficient bucket cleanup) // Indexed by [ring_id][local_id & (task_window_sizes[ring_id] - 1)] - PTO2TensorMapEntry **task_entry_heads[CHIP_MAX_RING_DEPTH]; + ChipTensorMapEntry **task_entry_heads[CHIP_MAX_RING_DEPTH]; uint32_t *task_entry_head_epochs[CHIP_MAX_RING_DEPTH]; int32_t task_window_sizes[CHIP_MAX_RING_DEPTH]; // Per-ring task window size (for slot masking) uint32_t current_epoch{1}; @@ -419,18 +419,18 @@ struct PTO2TensorMap { } // new_entry only allocates memory, does not assign attributes - PTO2TensorMapEntry *new_entry() { + ChipTensorMapEntry *new_entry() { if (free_num > 0) { - PTO2TensorMapEntry *res = free_entry_list[--free_num]; + ChipTensorMapEntry *res = free_entry_list[--free_num]; debug_assert(res->bucket_index == -1); return res; } always_assert(next_entry_idx < pool_size); - PTO2TensorMapEntry *res = &entry_pool[next_entry_idx++]; + ChipTensorMapEntry *res = &entry_pool[next_entry_idx++]; return res; } - void free_entry(PTO2TensorMapEntry &entry) { + void free_entry(ChipTensorMapEntry &entry) { always_assert(entry.bucket_index != -1); // must still be in a bucket // Update predecessor's next pointer (O(1) via prev_in_bucket) @@ -465,15 +465,15 @@ struct PTO2TensorMap { * the returned layout descriptor. Must be called before the arena is * committed. */ - static PTO2TensorMapLayout reserve_layout( + static ChipTensorMapLayout reserve_layout( DeviceArena &arena, int32_t num_buckets, int32_t pool_size, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH] ); /** - * Same as reserve_layout() with default sizes (PTO2_TENSORMAP_NUM_BUCKETS, - * PTO2_TENSORMAP_POOL_SIZE). + * Same as reserve_layout() with default sizes (CHIP_TENSORMAP_NUM_BUCKETS, + * CHIP_TENSORMAP_POOL_SIZE). */ - static PTO2TensorMapLayout + static ChipTensorMapLayout reserve_layout_default(DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH]); /** @@ -483,14 +483,14 @@ struct PTO2TensorMap { * but does not store those addresses in struct fields. Safe to call on * a host arena that holds the prebuilt image. */ - bool init_data_from_layout(const PTO2TensorMapLayout &layout, DeviceArena &arena); - void reset_for_reuse(const PTO2TensorMapLayout &layout); + bool init_data_from_layout(const ChipTensorMapLayout &layout, DeviceArena &arena); + void reset_for_reuse(const ChipTensorMapLayout &layout); /** * Phase 3b: write the arena-internal pointer fields. Idempotent; * called once on the host arena and once on the AICPU after attach. */ - void wire_arena_pointers(const PTO2TensorMapLayout &layout, DeviceArena &arena); + void wire_arena_pointers(const ChipTensorMapLayout &layout, DeviceArena &arena); /** * Tear down state. Does not free memory — the arena owns the backing @@ -513,7 +513,7 @@ struct PTO2TensorMap { * for each overlapping valid entry. * Stale entries from different rings are skipped (not truncated). * - * The callback receives (PTO2TensorMapEntry &, OverlapStatus) and should + * The callback receives (ChipTensorMapEntry &, OverlapStatus) and should * return true to continue iteration, false to stop early. It is safe for * the callback to call remove_entry() on the current entry: next_in_bucket * is latched before invocation. @@ -527,7 +527,7 @@ struct PTO2TensorMap { if (bucket_epochs[bucket_index] != current_epoch) { return; } - PTO2TensorMapEntry *cur_entry = buckets[bucket_index]; + ChipTensorMapEntry *cur_entry = buckets[bucket_index]; #if SIMPLER_TENSORMAP_PROFILING g_lookup_count++; @@ -535,7 +535,7 @@ struct PTO2TensorMap { #endif while (cur_entry != nullptr) { - PTO2TensorMapEntry *next_entry = cur_entry->next_in_bucket; + ChipTensorMapEntry *next_entry = cur_entry->next_in_bucket; #if SIMPLER_TENSORMAP_PROFILING chain_len++; @@ -589,7 +589,7 @@ struct PTO2TensorMap { * @param producer_task_id Task ID of producer */ void insert(const ChipTensor &tensor, TaskId producer_task_id) { - PTO2TensorMapEntry *entry = new_entry(); + ChipTensorMapEntry *entry = new_entry(); entry->copy_from_tensor(tensor); link_entry(entry, tensor.buffer.addr, producer_task_id); } @@ -615,9 +615,9 @@ struct PTO2TensorMap { // Free only entries produced by the retiring local_id, unlinking // each from the chain; entries from other tasks stay linked. TaskId retired_task = TaskId::make(static_cast(ring_id), static_cast(local_id)); - PTO2TensorMapEntry *cur_entry = task_entry_heads[ring_id][task_slot]; + ChipTensorMapEntry *cur_entry = task_entry_heads[ring_id][task_slot]; while (cur_entry != nullptr) { - PTO2TensorMapEntry *next_entry = cur_entry->next_in_task; // free_entry clears it + ChipTensorMapEntry *next_entry = cur_entry->next_in_task; // free_entry clears it if (cur_entry->producer_task_id == retired_task) { if (cur_entry->prev_in_task != nullptr) { cur_entry->prev_in_task->next_in_task = next_entry; @@ -654,7 +654,7 @@ struct PTO2TensorMap { /** * Link an initialized entry into bucket and task chains. */ - void link_entry(PTO2TensorMapEntry *entry, uint64_t addr, TaskId producer_task_id) { + void link_entry(ChipTensorMapEntry *entry, uint64_t addr, TaskId producer_task_id) { #if SIMPLER_TENSORMAP_PROFILING g_insert_count++; #endif @@ -694,11 +694,11 @@ struct PTO2TensorMap { /** * Check if entry is valid (producer has not retired) */ - bool entry_valid(const PTO2TensorMapEntry &entry) const { + bool entry_valid(const ChipTensorMapEntry &entry) const { return static_cast(entry.producer_task_id.local()) >= last_task_alives[entry.producer_task_id.ring()]; } - void remove_entry(PTO2TensorMapEntry &entry) { + void remove_entry(ChipTensorMapEntry &entry) { remove_from_task(entry); free_entry(entry); } @@ -707,7 +707,7 @@ struct PTO2TensorMap { * Remove entry from its task chain (O(1) with prev pointer) * Called during pool wrap-around to unlink reused entries. */ - void remove_from_task(PTO2TensorMapEntry &entry) { + void remove_from_task(ChipTensorMapEntry &entry) { always_assert(entry.bucket_index != -1); // must still be in a bucket // Update predecessor's next pointer (O(1) via prev_in_task) if (entry.prev_in_task == nullptr) { @@ -757,7 +757,7 @@ struct PTO2TensorMap { }; #if SIMPLER_TENSORMAP_PROFILING -struct PTO2TensorMapProfilingData { +struct ChipTensorMapProfilingData { uint64_t lookup_chain_total; uint64_t lookup_count; int32_t lookup_chain_max; @@ -766,5 +766,5 @@ struct PTO2TensorMapProfilingData { uint64_t insert_count; }; -PTO2TensorMapProfilingData pto2_tensormap_get_profiling(); +ChipTensorMapProfilingData chip_tensormap_get_profiling(); #endif diff --git a/src/a5/runtime/host_build_graph/host/dep_gen_host_graph.cpp b/src/a5/runtime/host_build_graph/host/dep_gen_host_graph.cpp index a11f05f3d5..9450b2cdaa 100644 --- a/src/a5/runtime/host_build_graph/host/dep_gen_host_graph.cpp +++ b/src/a5/runtime/host_build_graph/host/dep_gen_host_graph.cpp @@ -183,9 +183,9 @@ void fill_consumer(EdgeAnnot &e, const ChipTensor &t) { } } -// Copy a PTO2TensorMapEntry's slice description into an EdgeAnnot's producer_* +// Copy a ChipTensorMapEntry's slice description into an EdgeAnnot's producer_* // fields. Only called from the TENSORMAP path. -void fill_producer(EdgeAnnot &e, const PTO2TensorMapEntry &entry) { +void fill_producer(EdgeAnnot &e, const ChipTensorMapEntry &entry) { e.producer_ndims = entry.ndims; e.producer_start_offset = entry.start_offset; for (uint32_t i = 0; i < entry.ndims && i < MAX_TENSOR_DIMS; i++) { @@ -465,7 +465,7 @@ void dep_gen_host_graph_add_creator_edge(uint64_t producer_raw, int32_t arg_idx, } void dep_gen_host_graph_add_tensormap_edge( - uint64_t producer_raw, int32_t arg_idx, const ChipTensor &consumer, const PTO2TensorMapEntry &entry, + uint64_t producer_raw, int32_t arg_idx, const ChipTensor &consumer, const ChipTensorMapEntry &entry, OverlapStatus overlap ) { HostGraphState &s = state(); diff --git a/src/a5/runtime/host_build_graph/runtime/dep_compute.h b/src/a5/runtime/host_build_graph/runtime/dep_compute.h index 814a1156c7..6a7dc2d84a 100644 --- a/src/a5/runtime/host_build_graph/runtime/dep_compute.h +++ b/src/a5/runtime/host_build_graph/runtime/dep_compute.h @@ -37,7 +37,7 @@ * The Annotate callback contract (dep_gen graph capture): * void annotate.creator(int32_t arg_idx, const ChipTensor &consumer, TaskId producer); * void annotate.tensormap(int32_t arg_idx, const ChipTensor &consumer, - * const PTO2TensorMapEntry &entry, OverlapStatus overlap); + * const ChipTensorMapEntry &entry, OverlapStatus overlap); * Both fire for exactly the producers `emit` saw, with the tensor-side context an * edge needs. `tensormap` runs before the INOUT/COVERED remove_entry(), so `entry` * is still live when it is read. NoDepAnnotate is the empty default and compiles @@ -79,7 +79,7 @@ struct DepInputs { */ struct NoDepAnnotate { void creator(int32_t, const ChipTensor &, TaskId) const {} - void tensormap(int32_t, const ChipTensor &, const PTO2TensorMapEntry &, OverlapStatus) const {} + void tensormap(int32_t, const ChipTensor &, const ChipTensorMapEntry &, OverlapStatus) const {} }; /** @@ -96,7 +96,7 @@ struct NoDepAnnotate { */ template [[nodiscard]] inline bool compute_task_fanin( - const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_manual_scope, Emit emit, Annotate annotate = Annotate{} + const DepInputs &inputs, ChipTensorMap &tensor_map, bool in_manual_scope, Emit emit, Annotate annotate = Annotate{} ) { if (in_manual_scope) { return true; @@ -130,7 +130,7 @@ template } bool fatal = false; - tensor_map.lookup(*tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus overlap_status) -> bool { + tensor_map.lookup(*tensor, [&](ChipTensorMapEntry &entry, OverlapStatus overlap_status) -> bool { if (!emit(entry.producer_task_id)) { fatal = true; return false; // stop iteration @@ -157,7 +157,7 @@ template * No-op when in_manual_scope. */ inline void -register_task_outputs(const DepInputs &inputs, TaskId task_id, PTO2TensorMap &tensor_map, bool in_manual_scope) { +register_task_outputs(const DepInputs &inputs, TaskId task_id, ChipTensorMap &tensor_map, bool in_manual_scope) { if (in_manual_scope) { return; } diff --git a/src/a5/runtime/host_build_graph/runtime/dep_gen_host_graph.h b/src/a5/runtime/host_build_graph/runtime/dep_gen_host_graph.h index c56f7f7c40..ea840b0610 100644 --- a/src/a5/runtime/host_build_graph/runtime/dep_gen_host_graph.h +++ b/src/a5/runtime/host_build_graph/runtime/dep_gen_host_graph.h @@ -103,7 +103,7 @@ void dep_gen_host_graph_add_creator_edge(uint64_t producer_raw, int32_t arg_idx, /** STEP 3 Step B: a tensormap producer whose written slice this task reads. */ void dep_gen_host_graph_add_tensormap_edge( - uint64_t producer_raw, int32_t arg_idx, const ChipTensor &consumer, const PTO2TensorMapEntry &entry, + uint64_t producer_raw, int32_t arg_idx, const ChipTensor &consumer, const ChipTensorMapEntry &entry, OverlapStatus overlap ); diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator.h b/src/a5/runtime/host_build_graph/runtime/orchestrator.h index c469578ca4..ccf5d15098 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator.h @@ -67,7 +67,7 @@ struct OrchestratorState { uint32_t fanin_seen_current_epoch{1}; // === TENSOR MAP (Private) === - PTO2TensorMap tensor_map; // Producer lookup + ChipTensorMap tensor_map; // Producer lookup // === SCOPE STACK (Private) === // Depth only. A scope decides whether a submit takes its fanin from TensorMap diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index a5a0bede66..dbb7a56567 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -81,7 +81,7 @@ struct DepGraphAnnotate { dep_gen_host_graph_add_creator_edge(producer.raw, arg_idx, consumer); } void tensormap( - int32_t arg_idx, const ChipTensor &consumer, const PTO2TensorMapEntry &entry, OverlapStatus overlap + int32_t arg_idx, const ChipTensor &consumer, const ChipTensorMapEntry &entry, OverlapStatus overlap ) const { dep_gen_host_graph_add_tensormap_edge(entry.producer_task_id.raw, arg_idx, consumer, entry, overlap); } @@ -90,7 +90,7 @@ __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_add_explicit __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_add_creator_edge(uint64_t, int32_t, const ChipTensor &) {} __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_add_tensormap_edge( - uint64_t, int32_t, const ChipTensor &, const PTO2TensorMapEntry &, OverlapStatus + uint64_t, int32_t, const ChipTensor &, const ChipTensorMapEntry &, OverlapStatus ) {} // AICore register accessor (aicpu/platform_regs.h). The host orchestrator's @@ -442,7 +442,7 @@ struct GraphRecording { // (the shape every generated orchestration uses) would therefore record a // node with no edge to its actual producer, and the Definition would replay // a DAG the same body never had when submitted task by task. - PTO2TensorMap tensor_map{}; + ChipTensorMap tensor_map{}; bool tensor_map_ready{false}; // Scope depth as the body sees it. begin_scope/end_scope leave the real // orchestrator stack untouched while recording (a Graph replays flat), but @@ -649,7 +649,7 @@ graph_classify_scalar(const GraphRecording &recording, const ArgT &args, int32_t // Entry capacity for one recorded body's hazard map. A Definition is capped at // GRAPH_MAX_NODES nodes and each node registers at most its INOUT/OUTPUT_EXISTING // args, so this bounds the worst realistic body while staying a small fraction of -// the ring path's whole-orchestration pool (PTO2_TENSORMAP_POOL_SIZE). Exhausting +// the ring path's whole-orchestration pool (CHIP_TENSORMAP_POOL_SIZE). Exhausting // it marks the recording unsupported, which graph_commit reports as // SIMPLER_ERROR_INVALID_ARGS -- the outer shell is already submitted by then, so there // is no ordinary-path fallback left to take. @@ -659,7 +659,7 @@ constexpr int32_t GRAPH_RECORD_TENSORMAP_POOL_SIZE = 16384; // reported to the caller, which abandons the recording rather than producing a // Definition with inferred edges missing. bool graph_recording_init_tensor_map(GraphRecording &recording) { - if (!recording.tensor_map.init(PTO2_TENSORMAP_NUM_BUCKETS, GRAPH_RECORD_TENSORMAP_POOL_SIZE, GRAPH_MAX_NODES)) { + if (!recording.tensor_map.init(CHIP_TENSORMAP_NUM_BUCKETS, GRAPH_RECORD_TENSORMAP_POOL_SIZE, GRAPH_MAX_NODES)) { return false; } recording.tensor_map_ready = true; @@ -1421,7 +1421,7 @@ void OrchestratorState::end_scope() { // assert fire mid-registration. Returns false when the pool is exhausted or a // fatal is already latched by another party. static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { - PTO2TensorMap &tm = orch->tensor_map; + ChipTensorMap &tm = orch->tensor_map; if (tm.free_entries() >= needed) { return true; } @@ -1437,7 +1437,7 @@ static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { LOG_ERROR(" - Free: %d entries", tm.free_entries()); LOG_ERROR(" - Needed: %d entries", needed); LOG_ERROR("Solution:"); - LOG_ERROR(" Increase PTO2_TENSORMAP_POOL_SIZE (current: %d).", tm.pool_capacity()); + LOG_ERROR(" Increase CHIP_TENSORMAP_POOL_SIZE (current: %d).", tm.pool_capacity()); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_TENSORMAP_OVERFLOW); return false; diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index ab38c79b4d..0c934b1325 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -301,7 +301,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo } // Step B: modifier writer lookup (OverlapMap), direct callback - orch.tensor_map.lookup(tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus) -> bool { + orch.tensor_map.lookup(tensor, [&](ChipTensorMapEntry &entry, OverlapStatus) -> bool { TaskId pid = entry.producer_task_id; const auto *slot = resolve_producer(pid); if (slot == nullptr) return false; diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 366baa3575..2c7306630c 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -85,8 +85,8 @@ // bind_callable_to_runtime_impl). // Memory pools (total = value, single ring) -#define PTO2_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool -#define PTO2_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) +#define CHIP_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool +#define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) // Three address classes coexist during orchestration, in windows the two constants // below keep disjoint: real device addresses stay below HEAP_VIRTUAL_BASE, since diff --git a/src/a5/runtime/host_build_graph/runtime/shared/tensormap.cpp b/src/a5/runtime/host_build_graph/runtime/shared/tensormap.cpp index cc0f50a7df..82799bff20 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/tensormap.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/tensormap.cpp @@ -61,17 +61,17 @@ uint64_t g_insert_count = 0; * exists to avoid, and nothrow keeps the failure reportable to a caller that * still has an alternative path. */ -bool PTO2TensorMap::init(int32_t new_num_buckets, int32_t new_pool_size, int32_t new_task_window_size) { +bool ChipTensorMap::init(int32_t new_num_buckets, int32_t new_pool_size, int32_t new_task_window_size) { // num_buckets must be a power of two for the hash truncation to work, and // task_window_size for the task-chain slot mask. always_assert(new_num_buckets > 0 && (new_num_buckets & (new_num_buckets - 1)) == 0); always_assert(new_task_window_size > 0 && (new_task_window_size & (new_task_window_size - 1)) == 0); always_assert(new_pool_size > 0); - buckets.reset(new (std::nothrow) PTO2TensorMapEntry *[new_num_buckets]); - entry_pool.reset(new (std::nothrow) PTO2TensorMapEntry[new_pool_size]); - free_entry_list.reset(new (std::nothrow) PTO2TensorMapEntry *[new_pool_size]); - task_entry_heads.reset(new (std::nothrow) PTO2TensorMapEntry *[new_task_window_size]); + buckets.reset(new (std::nothrow) ChipTensorMapEntry *[new_num_buckets]); + entry_pool.reset(new (std::nothrow) ChipTensorMapEntry[new_pool_size]); + free_entry_list.reset(new (std::nothrow) ChipTensorMapEntry *[new_pool_size]); + task_entry_heads.reset(new (std::nothrow) ChipTensorMapEntry *[new_task_window_size]); if (buckets == nullptr || entry_pool == nullptr || free_entry_list == nullptr || task_entry_heads == nullptr) { LOG_ERROR( "TensorMap init failed (buckets=%d, pool=%d, window=%d)", new_num_buckets, new_pool_size, @@ -105,7 +105,7 @@ bool PTO2TensorMap::init(int32_t new_num_buckets, int32_t new_pool_size, int32_t return true; } -void PTO2TensorMap::reset() { +void ChipTensorMap::reset() { always_assert(buckets != nullptr && entry_pool != nullptr && task_entry_heads != nullptr); for (int32_t i = 0; i < num_buckets; i++) { @@ -121,15 +121,15 @@ void PTO2TensorMap::reset() { free_num = 0; } -bool PTO2TensorMap::init_default(int32_t new_task_window_size) { - return init(PTO2_TENSORMAP_NUM_BUCKETS, PTO2_TENSORMAP_POOL_SIZE, new_task_window_size); +bool ChipTensorMap::init_default(int32_t new_task_window_size) { + return init(CHIP_TENSORMAP_NUM_BUCKETS, CHIP_TENSORMAP_POOL_SIZE, new_task_window_size); } // ============================================================================= // Debug Utilities // ============================================================================= -void PTO2TensorMap::print_stats() { +void ChipTensorMap::print_stats() { int32_t valid = 0; int32_t empty_buckets = 0; int32_t max_chain = 0; @@ -178,7 +178,7 @@ void PTO2TensorMap::print_stats() { LOG_DEBUG("============================"); } -int32_t PTO2TensorMap::valid_count() { +int32_t ChipTensorMap::valid_count() { int32_t count = 0; // Init-on-write: only [0, next_entry_idx) slots have ever been allocated and @@ -196,8 +196,8 @@ int32_t PTO2TensorMap::valid_count() { // TensorMap Lookup Profiling // ============================================================================= #if SIMPLER_TENSORMAP_PROFILING -PTO2TensorMapProfilingData pto2_tensormap_get_profiling() { - PTO2TensorMapProfilingData d; +ChipTensorMapProfilingData chip_tensormap_get_profiling() { + ChipTensorMapProfilingData d; d.lookup_chain_total = g_lookup_chain_total; d.lookup_count = g_lookup_count; d.lookup_chain_max = g_lookup_chain_max; diff --git a/src/a5/runtime/host_build_graph/runtime/tensormap.h b/src/a5/runtime/host_build_graph/runtime/tensormap.h index 158681b33e..0deee432df 100644 --- a/src/a5/runtime/host_build_graph/runtime/tensormap.h +++ b/src/a5/runtime/host_build_graph/runtime/tensormap.h @@ -111,10 +111,10 @@ extern uint64_t g_insert_count; * * Entry size: 128B (2 cache lines), matches ChipTensor. */ -struct alignas(64) PTO2TensorMapEntry { +struct alignas(64) ChipTensorMapEntry { // === Cache line 1 (64B) — lookup hot path; mirrors ChipTensor line 1 from byte 16 === uint64_t buffer_addr; // 8B [0, 8): tensor base address (hash key, mirrors ChipTensor::buffer.addr) - PTO2TensorMapEntry + ChipTensorMapEntry *next_in_bucket; // 8B [8, 16): next entry in hash bucket chain (overlays ChipTensor::buffer.size) TaskId producer_task_id; // 8B [16,24): mirrors ChipTensor::owner_task_id slot uint64_t start_offset; // 8B [24,32): mirrors ChipTensor::start_offset (element offset) @@ -127,9 +127,9 @@ struct alignas(64) PTO2TensorMapEntry { uint32_t shapes[MAX_TENSOR_DIMS]; // 20B [44,64): mirrors ChipTensor::shapes // === Cache line 2 (64B) — chain manipulation + non-contiguous overlap data === - PTO2TensorMapEntry *prev_in_bucket; // 8B [64, 72) - PTO2TensorMapEntry *next_in_task; // 8B [72, 80) - PTO2TensorMapEntry *prev_in_task; // 8B [80, 88) + ChipTensorMapEntry *prev_in_bucket; // 8B [64, 72) + ChipTensorMapEntry *next_in_task; // 8B [72, 80) + ChipTensorMapEntry *prev_in_task; // 8B [80, 88) int32_t bucket_index; // 4B [88, 92): -1 when unlinked uint32_t __padding2__; // 4B [92, 96) uint64_t extent_elem_cache; // 8B [96,104): non-contiguous extent (mirrors ChipTensor) @@ -323,18 +323,18 @@ struct alignas(64) PTO2TensorMapEntry { } }; -static_assert(sizeof(PTO2TensorMapEntry) == 128, "TensorMapEntry must be exactly 2 cache lines (128 bytes)"); -static_assert(offsetof(PTO2TensorMapEntry, buffer_addr) == offsetof(ChipTensor, buffer.addr)); -static_assert(offsetof(PTO2TensorMapEntry, producer_task_id) == offsetof(ChipTensor, owner_task_id)); -static_assert(offsetof(PTO2TensorMapEntry, start_offset) == offsetof(ChipTensor, start_offset)); -static_assert(offsetof(PTO2TensorMapEntry, version) == offsetof(ChipTensor, version)); -static_assert(offsetof(PTO2TensorMapEntry, ndims) == offsetof(ChipTensor, ndims)); -static_assert(offsetof(PTO2TensorMapEntry, dtype) == offsetof(ChipTensor, dtype)); -static_assert(offsetof(PTO2TensorMapEntry, manual_dep) == offsetof(ChipTensor, manual_dep)); -static_assert(offsetof(PTO2TensorMapEntry, is_contiguous) == offsetof(ChipTensor, is_contiguous)); -static_assert(offsetof(PTO2TensorMapEntry, shapes) == offsetof(ChipTensor, shapes)); +static_assert(sizeof(ChipTensorMapEntry) == 128, "TensorMapEntry must be exactly 2 cache lines (128 bytes)"); +static_assert(offsetof(ChipTensorMapEntry, buffer_addr) == offsetof(ChipTensor, buffer.addr)); +static_assert(offsetof(ChipTensorMapEntry, producer_task_id) == offsetof(ChipTensor, owner_task_id)); +static_assert(offsetof(ChipTensorMapEntry, start_offset) == offsetof(ChipTensor, start_offset)); +static_assert(offsetof(ChipTensorMapEntry, version) == offsetof(ChipTensor, version)); +static_assert(offsetof(ChipTensorMapEntry, ndims) == offsetof(ChipTensor, ndims)); +static_assert(offsetof(ChipTensorMapEntry, dtype) == offsetof(ChipTensor, dtype)); +static_assert(offsetof(ChipTensorMapEntry, manual_dep) == offsetof(ChipTensor, manual_dep)); +static_assert(offsetof(ChipTensorMapEntry, is_contiguous) == offsetof(ChipTensor, is_contiguous)); +static_assert(offsetof(ChipTensorMapEntry, shapes) == offsetof(ChipTensor, shapes)); static_assert( - offsetof(PTO2TensorMapEntry, prev_in_bucket) == 64, "TensorMapEntry must be exactly 2 cache lines (128 bytes)" + offsetof(ChipTensorMapEntry, prev_in_bucket) == 64, "TensorMapEntry must be exactly 2 cache lines (128 bytes)" ); // ============================================================================= @@ -347,23 +347,23 @@ static_assert( * Hash table with a fixed-capacity entry pool and no watermark invalidation. * Owns its four arrays; init() sizes them and leaves the map empty. */ -struct PTO2TensorMap { +struct ChipTensorMap { // Hash table buckets (fixed size, power of 2). An empty bucket is nullptr. - std::unique_ptr buckets; + std::unique_ptr buckets; int32_t num_buckets{0}; // Must be power of 2 for fast modulo // Entry pool: bump allocation plus reuse of explicitly removed entries. A // linked entry is reached by pointer, so the pool is allocated once at // pool_size and never resized. - std::unique_ptr entry_pool; - std::unique_ptr free_entry_list; + std::unique_ptr entry_pool; + std::unique_ptr free_entry_list; int32_t pool_size{0}; // Total pool capacity int32_t next_entry_idx{0}; // id when next entry insert int32_t free_num{0}; // free entry number in entry pool // Per-task entry tracking for O(1) unlinking of covered producers. // Indexed by [local_id & (task_window_size - 1)] - std::unique_ptr task_entry_heads; + std::unique_ptr task_entry_heads; int32_t task_window_size{0}; // Task window size (for slot masking) uint32_t get_task_local_id_slot(uint32_t task_local_id) const { return task_local_id & (task_window_size - 1); } @@ -377,14 +377,14 @@ struct PTO2TensorMap { // new_entry allocates a slot and initializes only its linkage (bucket_index // and the four link pointers) to the clean unlinked state; insert() assigns // the tensor attributes and producer_task_id. - PTO2TensorMapEntry *new_entry() { + ChipTensorMapEntry *new_entry() { if (free_num > 0) { - PTO2TensorMapEntry *res = free_entry_list[--free_num]; + ChipTensorMapEntry *res = free_entry_list[--free_num]; debug_assert(res->bucket_index == -1); return res; } always_assert(next_entry_idx < pool_size); - PTO2TensorMapEntry *res = &entry_pool[next_entry_idx++]; + ChipTensorMapEntry *res = &entry_pool[next_entry_idx++]; // Init-on-write: the pool is not pre-zeroed (init() skips // the O(pool_size) memset), so put this fresh slot into the same clean // unlinked state free_entry() leaves recycled slots in. The insert path @@ -398,7 +398,7 @@ struct PTO2TensorMap { return res; } - void free_entry(PTO2TensorMapEntry &entry) { + void free_entry(ChipTensorMapEntry &entry) { always_assert(entry.bucket_index != -1); // must still be in a bucket // Update predecessor's next pointer (O(1) via prev_in_bucket) @@ -459,8 +459,8 @@ struct PTO2TensorMap { void reset(); /** - * Same as init() with default sizes (PTO2_TENSORMAP_NUM_BUCKETS, - * PTO2_TENSORMAP_POOL_SIZE). + * Same as init() with default sizes (CHIP_TENSORMAP_NUM_BUCKETS, + * CHIP_TENSORMAP_POOL_SIZE). */ bool init_default(int32_t new_task_window_size); @@ -470,7 +470,7 @@ struct PTO2TensorMap { * Searches the hash table for matching regions and invokes the callback * for each overlapping entry. * - * The callback receives (PTO2TensorMapEntry &, OverlapStatus) and should + * The callback receives (ChipTensorMapEntry &, OverlapStatus) and should * return true to continue iteration, false to stop early. It is safe for * the callback to call remove_entry() on the current entry: next_in_bucket * is latched before invocation. @@ -481,7 +481,7 @@ struct PTO2TensorMap { template void lookup(const ChipTensor &tensor, Fn &&on_match) { uint32_t bucket_index = hash(tensor.buffer.addr); - PTO2TensorMapEntry *cur_entry = buckets[bucket_index]; + ChipTensorMapEntry *cur_entry = buckets[bucket_index]; #if SIMPLER_TENSORMAP_PROFILING g_lookup_count++; @@ -489,7 +489,7 @@ struct PTO2TensorMap { #endif while (cur_entry != nullptr) { - PTO2TensorMapEntry *next_entry = cur_entry->next_in_bucket; + ChipTensorMapEntry *next_entry = cur_entry->next_in_bucket; #if SIMPLER_TENSORMAP_PROFILING chain_len++; @@ -536,7 +536,7 @@ struct PTO2TensorMap { * @param producer_task_id Task ID of producer */ void insert(const ChipTensor &tensor, TaskId producer_task_id) { - PTO2TensorMapEntry *entry = new_entry(); + ChipTensorMapEntry *entry = new_entry(); entry->copy_from_tensor(tensor); link_entry(entry, tensor.buffer.addr, producer_task_id); } @@ -561,7 +561,7 @@ struct PTO2TensorMap { /** * Link an initialized entry into bucket and task chains. */ - void link_entry(PTO2TensorMapEntry *entry, uint64_t addr, TaskId producer_task_id) { + void link_entry(ChipTensorMapEntry *entry, uint64_t addr, TaskId producer_task_id) { #if SIMPLER_TENSORMAP_PROFILING g_insert_count++; #endif @@ -589,7 +589,7 @@ struct PTO2TensorMap { task_entry_heads[task_slot] = entry; } - void remove_entry(PTO2TensorMapEntry &entry) { + void remove_entry(ChipTensorMapEntry &entry) { remove_from_task(entry); free_entry(entry); } @@ -599,7 +599,7 @@ struct PTO2TensorMap { * computation calls this before freeing a producer made redundant by a * covering input. */ - void remove_from_task(PTO2TensorMapEntry &entry) { + void remove_from_task(ChipTensorMapEntry &entry) { always_assert(entry.bucket_index != -1); // must still be in a bucket // Update predecessor's next pointer (O(1) via prev_in_task) if (entry.prev_in_task == nullptr) { @@ -636,7 +636,7 @@ struct PTO2TensorMap { }; #if SIMPLER_TENSORMAP_PROFILING -struct PTO2TensorMapProfilingData { +struct ChipTensorMapProfilingData { uint64_t lookup_chain_total; uint64_t lookup_count; int32_t lookup_chain_max; @@ -645,5 +645,5 @@ struct PTO2TensorMapProfilingData { uint64_t insert_count; }; -PTO2TensorMapProfilingData pto2_tensormap_get_profiling(); +ChipTensorMapProfilingData chip_tensormap_get_profiling(); #endif diff --git a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index 6996a4d05b..1d105fd88e 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -783,7 +783,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { ); #if SIMPLER_TENSORMAP_PROFILING - PTO2TensorMapProfilingData tp = pto2_tensormap_get_profiling(); + ChipTensorMapProfilingData tp = chip_tensormap_get_profiling(); LOG_INFO("Thread %d: === TensorMap Lookup Stats ===", thread_idx); LOG_INFO( "Thread %d: lookups : %" PRIu64 ", inserts: %" PRIu64 "", thread_idx, diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 04884ebea4..2e0471fd0c 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -41,7 +41,7 @@ Type changes: | Field | Before | After | | ----- | ------ | ----- | | `TaskDescriptor.task_id` | `int32_t` | `TaskId` | -| `PTO2TensorMapEntry.producer_task_id` | `int32_t` | `TaskId` | +| `ChipTensorMapEntry.producer_task_id` | `int32_t` | `TaskId` | | `ChipTaskSlotState.ring_id` | N/A | `uint8_t` (new, denormalized for fast access) | ## 4. Data Structures @@ -144,17 +144,17 @@ RingSchedState ring_sched_states[CHIP_MAX_RING_DEPTH]; `slot_states`, `task_window_size`, and `task_window_mask` are no longer duplicated — callers access them via `ring->get_slot_state_by_*()` and other ring header accessors. The ring pointer shares cache line 0 with `last_task_alive` and `advance_lock`. -### 4.6 PTO2TensorMap (modified) +### 4.6 ChipTensorMap (modified) ```cpp -PTO2TensorMapEntry** task_entry_heads[CHIP_MAX_RING_DEPTH]; +ChipTensorMapEntry** task_entry_heads[CHIP_MAX_RING_DEPTH]; int64_t last_task_alives[CHIP_MAX_RING_DEPTH]; ``` Entry validity checks and `cleanup_retired` operate per-ring: ```cpp -bool entry_valid(const PTO2TensorMapEntry& e) { +bool entry_valid(const ChipTensorMapEntry& e) { int32_t ring = e.producer_task_id.ring(); int32_t local = e.producer_task_id.local(); return local >= last_task_alives[ring]; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 43c8351376..db35f3ff70 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -326,7 +326,7 @@ Unlike the Task Ring and Heap Ring, TensorMap entries are **not** managed by a r 2. **Bump allocation**: if free list is empty, `entry_pool[next_entry_idx++]` allocates from the end of the pool. 3. **Blocking reclaim**: if the pool is short of the inserts a task needs, the orchestrator's `ensure_tensormap_capacity` reads the latest `last_task_alive` for every ring and calls `reclaim_retired_all` (`cleanup_retired` per ring) to batch-free entries belonging to retired tasks, returning them to the free list, before the inserts proceed. -This design avoids the complexity of ring-based wrapping while still being bounded by `PTO2_TENSORMAP_POOL_SIZE` (default 65536 entries). +This design avoids the complexity of ring-based wrapping while still being bounded by `CHIP_TENSORMAP_POOL_SIZE` (default 65536 entries). ### 5.4 Stale Entry Cleanup: Three-Layer Defense @@ -339,13 +339,13 @@ Three complementary mechanisms achieve this: **Layer 1 — Chain Truncation during Lookup** (lazy, per-bucket): -Since `insert` always prepends to the bucket head, entries in each bucket chain are in **descending task_id order**. When `PTO2TensorMap::lookup` encounters the first stale entry (`producer_task_id < last_task_alive`), all subsequent entries in the chain are guaranteed stale too. The entire tail is truncated in one operation using `prev_in_bucket` pointers for O(1) unlinking. +Since `insert` always prepends to the bucket head, entries in each bucket chain are in **descending task_id order**. When `ChipTensorMap::lookup` encounters the first stale entry (`producer_task_id < last_task_alive`), all subsequent entries in the chain are guaranteed stale too. The entire tail is truncated in one operation using `prev_in_bucket` pointers for O(1) unlinking. This guarantees lookup only traverses valid entries — O(valid_entries_in_bucket), not O(total_entries). **Layer 2 — Periodic Batch Cleanup** (`cleanup_retired`, per-task): -Every time the orchestrator submits a task (Step 0 of `OrchestratorState::submit_task`), it calls `PTO2TensorMap::sync_tensormap`. When `last_task_alive` has advanced by more than `PTO2_TENSORMAP_CLEANUP_INTERVAL` (default 64) tasks since the last cleanup, `PTO2TensorMap::cleanup_retired` runs: +Every time the orchestrator submits a task (Step 0 of `OrchestratorState::submit_task`), it calls `ChipTensorMap::sync_tensormap`. When `last_task_alive` has advanced by more than `CHIP_TENSORMAP_CLEANUP_INTERVAL` (default 64) tasks since the last cleanup, `ChipTensorMap::cleanup_retired` runs: This uses the **per-task entry chain** (`task_entry_head[task_slot]`) — each task's entries are doubly-linked together at insert time via `next_in_task`/`prev_in_task`. A slot's chain can hold more than one task's entries: a task at `local_id + N * window` reuses the slot and prepends to the chain already there, and cleanup can lag that reuse. Cleanup therefore walks the chain and frees only the entries whose `producer_task_id` matches the retiring task, unlinking each and leaving the rest linked — O(entries_in_slot), with no scan of the entire pool or all buckets. Freed entries are returned to `free_entry_list` for immediate reuse. @@ -369,9 +369,9 @@ In steady state, the number of valid TensorMap entries ≈ `active_tasks × avg_ When `OrchestratorState::submit_task` processes parameters: -1. **INPUT/INOUT**: `PTO2TensorMap::lookup` searches for overlapping producers (with chain truncation) +1. **INPUT/INOUT**: `ChipTensorMap::lookup` searches for overlapping producers (with chain truncation) 2. For each producer found: `append_fanin_or_fail` adds the dependency -3. **OUTPUT/INOUT**: `PTO2TensorMap::insert` registers the current task as the new producer at bucket head +3. **OUTPUT/INOUT**: `ChipTensorMap::insert` registers the current task as the new producer at bucket head 4. Stale entries are pruned lazily during lookup (Layer 1) and periodically by cleanup (Layer 2) --- @@ -441,7 +441,7 @@ Key members: | Step | Operation | | ---- | --------- | -| 0 | `PTO2TensorMap::sync_tensormap` — prune stale TensorMap entries | +| 0 | `ChipTensorMap::sync_tensormap` — prune stale TensorMap entries | | 1 | `PTO2TaskAllocator::alloc` — allocate task slot (may block on flow control) | | 2 | Initialize task descriptor + slot state, copy parameters | | 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `PTO2FaninBuilder` | diff --git a/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp b/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp index 1ea1f35b67..0e3ce5513f 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp @@ -12,10 +12,10 @@ /** * @file dep_gen_replay.cpp * @brief Replay in-memory DepGenRecord stream → deps.json (strided tensor - * representation, tensor-annotated) via a host-resident PTO2TensorMap, + * representation, tensor-annotated) via a host-resident ChipTensorMap, * with a differential check against the runtime template `compute_task_fanin`. * - * Two passes run per record against two parallel PTO2TensorMap instances that + * Two passes run per record against two parallel ChipTensorMap instances that * evolve in lockstep: * * ORACLE pass (read-only contract): @@ -29,7 +29,7 @@ * ANNOT pass (this file's feature): * Inlines the same STEP A (creator retention) + STEP B (tensormap lookup) * against `tm_annot`, but the callback fires with the full - * `PTO2TensorMapEntry&` + the consumer ChipTensor* + the arg index, so the + * `ChipTensorMapEntry&` + the consumer ChipTensor* + the arg index, so the * replay can record per-edge tensor metadata (producer/consumer * shape/offset, dtype, version). * @@ -295,9 +295,9 @@ void fill_consumer(EdgeAnnot &e, const ChipTensor &t) { } } -// Copy a PTO2TensorMapEntry's slice description into an EdgeAnnot's producer_* +// Copy a ChipTensorMapEntry's slice description into an EdgeAnnot's producer_* // fields. Only called from the TENSORMAP emit path. -void fill_producer(EdgeAnnot &e, const PTO2TensorMapEntry &entry) { +void fill_producer(EdgeAnnot &e, const ChipTensorMapEntry &entry) { e.producer_ndims = entry.ndims; e.producer_start_offset = entry.start_offset; for (uint32_t i = 0; i < entry.ndims && i < MAX_TENSOR_DIMS; i++) { @@ -425,7 +425,7 @@ bool write_deps_json( template void annot_pass( - const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_manual_scope, EmitCreator emit_creator, + const DepInputs &inputs, ChipTensorMap &tensor_map, bool in_manual_scope, EmitCreator emit_creator, EmitTM emit_tensormap ) { if (in_manual_scope) { @@ -452,7 +452,7 @@ void annot_pass( continue; } - tensor_map.lookup(*tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus overlap_status) -> bool { + tensor_map.lookup(*tensor, [&](ChipTensorMapEntry &entry, OverlapStatus overlap_status) -> bool { emit_tensormap(entry.producer_task_id, i, *tensor, entry, overlap_status); if (ptype == TensorArgType::INOUT && overlap_status == OverlapStatus::COVERED) { tensor_map.remove_entry(entry); @@ -498,12 +498,12 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c int32_t output_count = count_outputs(records, num_records); int32_t pool_size = output_count + (output_count / 10) + 64; - if (pool_size < PTO2_TENSORMAP_POOL_SIZE) { - pool_size = PTO2_TENSORMAP_POOL_SIZE; + if (pool_size < CHIP_TENSORMAP_POOL_SIZE) { + pool_size = CHIP_TENSORMAP_POOL_SIZE; } - PTO2TensorMap tm_oracle; - PTO2TensorMap tm_annot; + ChipTensorMap tm_oracle; + ChipTensorMap tm_annot; std::memset(&tm_oracle, 0, sizeof(tm_oracle)); std::memset(&tm_annot, 0, sizeof(tm_annot)); @@ -512,12 +512,12 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c DeviceArena replay_arena; auto oracle_layout = - PTO2TensorMap::reserve_layout(replay_arena, PTO2_TENSORMAP_NUM_BUCKETS, pool_size, task_window_sizes); + ChipTensorMap::reserve_layout(replay_arena, CHIP_TENSORMAP_NUM_BUCKETS, pool_size, task_window_sizes); auto annot_layout = - PTO2TensorMap::reserve_layout(replay_arena, PTO2_TENSORMAP_NUM_BUCKETS, pool_size, task_window_sizes); + ChipTensorMap::reserve_layout(replay_arena, CHIP_TENSORMAP_NUM_BUCKETS, pool_size, task_window_sizes); if (replay_arena.commit() == nullptr || !tm_oracle.init_data_from_layout(oracle_layout, replay_arena) || !tm_annot.init_data_from_layout(annot_layout, replay_arena)) { - LOG_ERROR("dep_gen replay: tensormap.init failed (buckets=%d, pool=%d)", PTO2_TENSORMAP_NUM_BUCKETS, pool_size); + LOG_ERROR("dep_gen replay: tensormap.init failed (buckets=%d, pool=%d)", CHIP_TENSORMAP_NUM_BUCKETS, pool_size); return -3; } // Replay tensormaps live entirely on host; only arena-internal pointer @@ -658,7 +658,7 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c // Register tasks[] entry (with per-arg slot info) and any unseen // tensors[] entries up-front. ChipTensors are registered from the // consumer-side blob so raw_shapes / dtype are populated (the - // producer-side PTO2TensorMapEntry drops raw_shapes to fit in two + // producer-side ChipTensorMapEntry drops raw_shapes to fit in two // cache lines). TaskTableEntry task_entry; task_entry.task_id = rec.task_id; @@ -753,7 +753,7 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c annot_edges.push_back(e); }, // emit_tensormap(producer, arg_idx, consumer_tensor, entry, status) - [&](TaskId producer, int32_t arg_idx, const ChipTensor &consumer, const PTO2TensorMapEntry &entry, + [&](TaskId producer, int32_t arg_idx, const ChipTensor &consumer, const ChipTensorMapEntry &entry, OverlapStatus status) { // Per-(succ, arg_idx, producer_buffer_addr, producer_version) // dedup gives us "the same producer slice fired twice for the diff --git a/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h b/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h index 0ee62ec026..02ac7feb48 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h @@ -15,7 +15,7 @@ * * Takes the records the host collector drained from the device ring buffer * (``DepGenCollector::records()``) and runs them back through a host-resident - * PTO2TensorMap using the same ``compute_task_fanin`` / ``register_task_outputs`` + * ChipTensorMap using the same ``compute_task_fanin`` / ``register_task_outputs`` * primitives the device orchestrator uses, emitting the full * predecessor → successor edge list to deps.json. * diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dep_compute.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dep_compute.h index 0bb98f7f6c..201b654cbb 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dep_compute.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dep_compute.h @@ -82,7 +82,7 @@ struct DepInputs { */ template [[nodiscard]] inline bool -compute_task_fanin(const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_manual_scope, Emit emit) { +compute_task_fanin(const DepInputs &inputs, ChipTensorMap &tensor_map, bool in_manual_scope, Emit emit) { if (in_manual_scope) { return true; } @@ -115,7 +115,7 @@ compute_task_fanin(const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_m } bool fatal = false; - tensor_map.lookup(*tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus overlap_status) -> bool { + tensor_map.lookup(*tensor, [&](ChipTensorMapEntry &entry, OverlapStatus overlap_status) -> bool { // Ordering-only (DEP_WAIT): a modifier only rewrote a buffer someone // else allocated, so its lifetime rides that allocator's creator edge, // not this modifier edge. Retention-safety invariant that makes this @@ -151,7 +151,7 @@ compute_task_fanin(const DepInputs &inputs, PTO2TensorMap &tensor_map, bool in_m * No-op when in_manual_scope. */ inline void -register_task_outputs(const DepInputs &inputs, TaskId task_id, PTO2TensorMap &tensor_map, bool in_manual_scope) { +register_task_outputs(const DepInputs &inputs, TaskId task_id, ChipTensorMap &tensor_map, bool in_manual_scope) { if (in_manual_scope) { return; } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index af7ab556f9..4f1359ad28 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -778,7 +778,7 @@ void OrchestratorState::end_scope() { // and bail. Returns false on deadlock or on a fatal already latched by another // party. Cold path — the fast path returns immediately when the pool has room. static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { - PTO2TensorMap &tm = orch->tensor_map; + ChipTensorMap &tm = orch->tensor_map; if (tm.free_entries() >= needed) { return true; } @@ -867,7 +867,7 @@ static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { LOG_ERROR(" still move on rings with no registered outputs). Check TaskRing"); LOG_ERROR(" diagnostics for the stalled producer."); LOG_ERROR("Solution:"); - LOG_ERROR(" Increase PTO2_TENSORMAP_POOL_SIZE (current: %d).", tm.pool_capacity()); + LOG_ERROR(" Increase CHIP_TENSORMAP_POOL_SIZE (current: %d).", tm.pool_capacity()); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_TENSORMAP_OVERFLOW); return false; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 9a8ecd5e88..0b5949dab4 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -40,14 +40,14 @@ /** * Layout descriptor produced by OrchestratorState::reserve_layout(). Holds * arena offsets for every sub-region the orchestrator owns (per-ring fanin - * pools, scope arrays, plus the nested PTO2TensorMap layout). + * pools, scope arrays, plus the nested ChipTensorMap layout). */ struct OrchestratorLayout { size_t off_fanin_pool[CHIP_MAX_RING_DEPTH]; size_t off_fanin_seen_epoch[CHIP_MAX_RING_DEPTH]; size_t off_scope_tasks; size_t off_scope_begins; - PTO2TensorMapLayout tensor_map; + ChipTensorMapLayout tensor_map; int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]; int32_t scope_tasks_cap; uint64_t scope_stack_capacity; @@ -72,7 +72,7 @@ struct OrchestratorState { uint32_t fanin_seen_current_epoch{1}; // === TENSOR MAP (Private) === - PTO2TensorMap tensor_map; // Producer lookup + ChipTensorMap tensor_map; // Producer lookup // === SCOPE STACK (Private) === // Single contiguous buffer of task IDs, partitioned by scope level. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 8ddd8c3827..83f3671398 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -196,7 +196,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo } // Step B: modifier writer lookup (OverlapMap), direct callback - orch.tensor_map.lookup(tensor, [&](PTO2TensorMapEntry &entry, OverlapStatus) -> bool { + orch.tensor_map.lookup(tensor, [&](ChipTensorMapEntry &entry, OverlapStatus) -> bool { TaskId pid = entry.producer_task_id; auto &s = orch.sm_header->rings[pid.ring()].get_slot_state_by_task_id(pid.local()); try_push(s); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index f20d90e8ba..8f3246bfa1 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -71,8 +71,8 @@ // Memory pools (per-ring defaults; total = value × CHIP_MAX_RING_DEPTH) #define PTO2_HEAP_SIZE (256 * 1024 * 1024) // 256MB per ring (1GB total) #define PTO2_DEP_LIST_POOL_SIZE 16384 // Per-ring dependency list pool entries -#define PTO2_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool -#define PTO2_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) +#define CHIP_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool +#define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) // Scope management #define PTO2_MAX_SCOPE_DEPTH 64 // Maximum nesting depth @@ -102,7 +102,7 @@ #define PTO2_DEP_DEGREE_DEBUG_THRESHOLD 16 // TensorMap cleanup interval -#define PTO2_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks +#define CHIP_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks #define PTO2_DEP_POOL_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 88c1319735..9738a6dea0 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -355,7 +355,7 @@ OrchestratorLayout OrchestratorState::reserve_layout( arena.reserve(static_cast(layout.scope_tasks_cap) * sizeof(uintptr_t), alignof(ChipTaskSlotState *)); layout.off_scope_begins = arena.reserve(static_cast(layout.scope_stack_capacity) * sizeof(int32_t), alignof(int32_t)); - layout.tensor_map = PTO2TensorMap::reserve_layout_default(arena, task_window_sizes); + layout.tensor_map = ChipTensorMap::reserve_layout_default(arena, task_window_sizes); return layout; } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/tensormap.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/tensormap.cpp index 09ca4a5dd2..dfa35caace 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/tensormap.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/tensormap.cpp @@ -47,14 +47,14 @@ uint64_t g_insert_count = 0; // Initialization and Destruction // ============================================================================= -PTO2TensorMapLayout PTO2TensorMap::reserve_layout( +ChipTensorMapLayout ChipTensorMap::reserve_layout( DeviceArena &arena, int32_t new_num_buckets, int32_t new_pool_size, const int32_t new_task_window_sizes[CHIP_MAX_RING_DEPTH] ) { // num_buckets must be a power of two for the hash truncation to work. always_assert((new_num_buckets & (new_num_buckets - 1)) == 0); - PTO2TensorMapLayout layout{}; + ChipTensorMapLayout layout{}; layout.num_buckets = new_num_buckets; layout.pool_size = new_pool_size; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { @@ -62,17 +62,17 @@ PTO2TensorMapLayout PTO2TensorMap::reserve_layout( } layout.off_buckets = arena.reserve( - static_cast(new_num_buckets) * sizeof(PTO2TensorMapEntry *), alignof(PTO2TensorMapEntry *) + static_cast(new_num_buckets) * sizeof(ChipTensorMapEntry *), alignof(ChipTensorMapEntry *) ); layout.off_bucket_epochs = arena.reserve(static_cast(new_num_buckets) * sizeof(uint32_t), alignof(uint32_t)); layout.off_entry_pool = - arena.reserve(static_cast(new_pool_size) * sizeof(PTO2TensorMapEntry), alignof(PTO2TensorMapEntry)); + arena.reserve(static_cast(new_pool_size) * sizeof(ChipTensorMapEntry), alignof(ChipTensorMapEntry)); layout.off_free_entry_list = - arena.reserve(static_cast(new_pool_size) * sizeof(PTO2TensorMapEntry *), alignof(PTO2TensorMapEntry *)); + arena.reserve(static_cast(new_pool_size) * sizeof(ChipTensorMapEntry *), alignof(ChipTensorMapEntry *)); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { layout.off_task_entry_heads[r] = arena.reserve( - static_cast(new_task_window_sizes[r]) * sizeof(PTO2TensorMapEntry *), alignof(PTO2TensorMapEntry *) + static_cast(new_task_window_sizes[r]) * sizeof(ChipTensorMapEntry *), alignof(ChipTensorMapEntry *) ); layout.off_task_entry_head_epochs[r] = arena.reserve(static_cast(new_task_window_sizes[r]) * sizeof(uint32_t), alignof(uint32_t)); @@ -80,21 +80,21 @@ PTO2TensorMapLayout PTO2TensorMap::reserve_layout( return layout; } -PTO2TensorMapLayout -PTO2TensorMap::reserve_layout_default(DeviceArena &arena, const int32_t new_task_window_sizes[CHIP_MAX_RING_DEPTH]) { - return reserve_layout(arena, PTO2_TENSORMAP_NUM_BUCKETS, PTO2_TENSORMAP_POOL_SIZE, new_task_window_sizes); +ChipTensorMapLayout +ChipTensorMap::reserve_layout_default(DeviceArena &arena, const int32_t new_task_window_sizes[CHIP_MAX_RING_DEPTH]) { + return reserve_layout(arena, CHIP_TENSORMAP_NUM_BUCKETS, CHIP_TENSORMAP_POOL_SIZE, new_task_window_sizes); } -bool PTO2TensorMap::init_data_from_layout(const PTO2TensorMapLayout &layout, DeviceArena &arena) { +bool ChipTensorMap::init_data_from_layout(const ChipTensorMapLayout &layout, DeviceArena &arena) { num_buckets = layout.num_buckets; pool_size = layout.pool_size; // Address arena regions for data writes; do not store these in struct // fields (wire_arena_pointers does that). - auto *buckets_arena = static_cast(arena.region_ptr(layout.off_buckets)); + auto *buckets_arena = static_cast(arena.region_ptr(layout.off_buckets)); auto *bucket_epochs_arena = static_cast(arena.region_ptr(layout.off_bucket_epochs)); - auto *entry_pool_arena = static_cast(arena.region_ptr(layout.off_entry_pool)); - auto *free_list_arena = static_cast(arena.region_ptr(layout.off_free_entry_list)); + auto *entry_pool_arena = static_cast(arena.region_ptr(layout.off_entry_pool)); + auto *free_list_arena = static_cast(arena.region_ptr(layout.off_free_entry_list)); // buckets[]: empty == nullptr. for (int32_t i = 0; i < num_buckets; i++) { @@ -105,7 +105,7 @@ bool PTO2TensorMap::init_data_from_layout(const PTO2TensorMapLayout &layout, Dev // entry_pool: zero-init equivalent to the previous calloc(entry_pool, ...). // The pool's persistent invariant after init is "bucket_index == -1 means // not linked", set explicitly below. - memset(entry_pool_arena, 0, static_cast(pool_size) * sizeof(PTO2TensorMapEntry)); + memset(entry_pool_arena, 0, static_cast(pool_size) * sizeof(ChipTensorMapEntry)); for (int32_t i = 0; i < pool_size; i++) { entry_pool_arena[i].bucket_index = -1; entry_pool_arena[i].next_in_bucket = nullptr; @@ -117,13 +117,13 @@ bool PTO2TensorMap::init_data_from_layout(const PTO2TensorMapLayout &layout, Dev // free_entry_list: zeroed (was calloc'd before); contents become meaningful // only after entries are freed back, so the body of the array stays as 0. - memset(free_list_arena, 0, static_cast(pool_size) * sizeof(PTO2TensorMapEntry *)); + memset(free_list_arena, 0, static_cast(pool_size) * sizeof(ChipTensorMapEntry *)); next_entry_idx = 0; free_num = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - auto *heads_arena = static_cast(arena.region_ptr(layout.off_task_entry_heads[r])); + auto *heads_arena = static_cast(arena.region_ptr(layout.off_task_entry_heads[r])); auto *head_epochs_arena = static_cast(arena.region_ptr(layout.off_task_entry_head_epochs[r])); for (int32_t i = 0; i < layout.task_window_sizes[r]; i++) { heads_arena[i] = nullptr; @@ -137,7 +137,7 @@ bool PTO2TensorMap::init_data_from_layout(const PTO2TensorMapLayout &layout, Dev return true; } -void PTO2TensorMap::reset_for_reuse(const PTO2TensorMapLayout &layout) { +void ChipTensorMap::reset_for_reuse(const ChipTensorMapLayout &layout) { num_buckets = layout.num_buckets; pool_size = layout.pool_size; next_entry_idx = 0; @@ -158,18 +158,18 @@ void PTO2TensorMap::reset_for_reuse(const PTO2TensorMapLayout &layout) { } } -void PTO2TensorMap::wire_arena_pointers(const PTO2TensorMapLayout &layout, DeviceArena &arena) { - buckets = static_cast(arena.region_ptr(layout.off_buckets)); +void ChipTensorMap::wire_arena_pointers(const ChipTensorMapLayout &layout, DeviceArena &arena) { + buckets = static_cast(arena.region_ptr(layout.off_buckets)); bucket_epochs = static_cast(arena.region_ptr(layout.off_bucket_epochs)); - entry_pool = static_cast(arena.region_ptr(layout.off_entry_pool)); - free_entry_list = static_cast(arena.region_ptr(layout.off_free_entry_list)); + entry_pool = static_cast(arena.region_ptr(layout.off_entry_pool)); + free_entry_list = static_cast(arena.region_ptr(layout.off_free_entry_list)); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - task_entry_heads[r] = static_cast(arena.region_ptr(layout.off_task_entry_heads[r])); + task_entry_heads[r] = static_cast(arena.region_ptr(layout.off_task_entry_heads[r])); task_entry_head_epochs[r] = static_cast(arena.region_ptr(layout.off_task_entry_head_epochs[r])); } } -void PTO2TensorMap::destroy() { +void ChipTensorMap::destroy() { // Arena owns the backing memory; here we only forget our pointers so any // stray post-destroy access trips a nullptr dereference instead of reading // a recycled allocation. @@ -187,7 +187,7 @@ void PTO2TensorMap::destroy() { // Debug Utilities // ============================================================================= -void PTO2TensorMap::print_stats() { +void ChipTensorMap::print_stats() { int32_t valid = 0; int32_t stale = 0; int32_t empty_buckets = 0; @@ -243,7 +243,7 @@ void PTO2TensorMap::print_stats() { LOG_DEBUG("============================"); } -int32_t PTO2TensorMap::valid_count() { +int32_t ChipTensorMap::valid_count() { int32_t count = 0; for (int32_t i = 0; i < pool_size; i++) { @@ -255,7 +255,7 @@ int32_t PTO2TensorMap::valid_count() { return count; } -void PTO2TensorMap::sync_tensormap(TaskId task_id, int32_t sm_last_task_alive) { +void ChipTensorMap::sync_tensormap(TaskId task_id, int32_t sm_last_task_alive) { auto ring_id = task_id.ring(); auto local_id = task_id.local(); sync_validity(ring_id, sm_last_task_alive); @@ -263,7 +263,7 @@ void PTO2TensorMap::sync_tensormap(TaskId task_id, int32_t sm_last_task_alive) { // Only attempt cleanup when last_task_alive has actually advanced; // otherwise cleanup_retired would empty-loop and we'd spin forever. auto overlap = get_task_local_id_slot(ring_id, local_id) == get_task_local_id_slot(ring_id, last_cleanup[ring_id]); - if (sm_last_task_alive - last_cleanup[ring_id] >= PTO2_TENSORMAP_CLEANUP_INTERVAL || overlap) { + if (sm_last_task_alive - last_cleanup[ring_id] >= CHIP_TENSORMAP_CLEANUP_INTERVAL || overlap) { cleanup_retired(ring_id, last_cleanup[ring_id], sm_last_task_alive); last_cleanup[ring_id] = sm_last_task_alive; } @@ -273,8 +273,8 @@ void PTO2TensorMap::sync_tensormap(TaskId task_id, int32_t sm_last_task_alive) { // TensorMap Lookup Profiling // ============================================================================= #if SIMPLER_TENSORMAP_PROFILING -PTO2TensorMapProfilingData pto2_tensormap_get_profiling() { - PTO2TensorMapProfilingData d; +ChipTensorMapProfilingData chip_tensormap_get_profiling() { + ChipTensorMapProfilingData d; d.lookup_chain_total = g_lookup_chain_total; d.lookup_count = g_lookup_count; d.lookup_chain_max = g_lookup_chain_max; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/tensormap.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/tensormap.h index fcfd2a143f..b9fbf156c4 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/tensormap.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/tensormap.h @@ -66,13 +66,13 @@ struct Segment { }; /** - * Layout descriptor produced by PTO2TensorMap::reserve_layout(). Stores the + * Layout descriptor produced by ChipTensorMap::reserve_layout(). Stores the * region offsets returned by DeviceArena::reserve() so init_data_from_layout() * can fetch the matching pointers after the arena is committed. * * All offsets are relative to the arena's base. */ -struct PTO2TensorMapLayout { +struct ChipTensorMapLayout { size_t off_buckets; size_t off_bucket_epochs; size_t off_entry_pool; @@ -124,10 +124,10 @@ extern uint64_t g_insert_count; * * Entry size: 128B (2 cache lines), matches ChipTensor. */ -struct alignas(64) PTO2TensorMapEntry { +struct alignas(64) ChipTensorMapEntry { // === Cache line 1 (64B) — lookup hot path; mirrors ChipTensor line 1 from byte 16 === uint64_t buffer_addr; // 8B [0, 8): tensor base address (hash key, mirrors ChipTensor::buffer.addr) - PTO2TensorMapEntry + ChipTensorMapEntry *next_in_bucket; // 8B [8, 16): next entry in hash bucket chain (overlays ChipTensor::buffer.size) TaskId producer_task_id; // 8B [16,24): mirrors ChipTensor::owner_task_id slot uint64_t start_offset; // 8B [24,32): mirrors ChipTensor::start_offset (element offset) @@ -140,9 +140,9 @@ struct alignas(64) PTO2TensorMapEntry { uint32_t shapes[MAX_TENSOR_DIMS]; // 20B [44,64): mirrors ChipTensor::shapes // === Cache line 2 (64B) — chain manipulation + non-contiguous overlap data === - PTO2TensorMapEntry *prev_in_bucket; // 8B [64, 72) - PTO2TensorMapEntry *next_in_task; // 8B [72, 80) - PTO2TensorMapEntry *prev_in_task; // 8B [80, 88) + ChipTensorMapEntry *prev_in_bucket; // 8B [64, 72) + ChipTensorMapEntry *next_in_task; // 8B [72, 80) + ChipTensorMapEntry *prev_in_task; // 8B [80, 88) int32_t bucket_index; // 4B [88, 92): -1 when unlinked uint32_t __padding2__; // 4B [92, 96) uint64_t extent_elem_cache; // 8B [96,104): non-contiguous extent (mirrors ChipTensor) @@ -336,18 +336,18 @@ struct alignas(64) PTO2TensorMapEntry { } }; -static_assert(sizeof(PTO2TensorMapEntry) == 128, "TensorMapEntry must be exactly 2 cache lines (128 bytes)"); -static_assert(offsetof(PTO2TensorMapEntry, buffer_addr) == offsetof(ChipTensor, buffer.addr)); -static_assert(offsetof(PTO2TensorMapEntry, producer_task_id) == offsetof(ChipTensor, owner_task_id)); -static_assert(offsetof(PTO2TensorMapEntry, start_offset) == offsetof(ChipTensor, start_offset)); -static_assert(offsetof(PTO2TensorMapEntry, version) == offsetof(ChipTensor, version)); -static_assert(offsetof(PTO2TensorMapEntry, ndims) == offsetof(ChipTensor, ndims)); -static_assert(offsetof(PTO2TensorMapEntry, dtype) == offsetof(ChipTensor, dtype)); -static_assert(offsetof(PTO2TensorMapEntry, manual_dep) == offsetof(ChipTensor, manual_dep)); -static_assert(offsetof(PTO2TensorMapEntry, is_contiguous) == offsetof(ChipTensor, is_contiguous)); -static_assert(offsetof(PTO2TensorMapEntry, shapes) == offsetof(ChipTensor, shapes)); +static_assert(sizeof(ChipTensorMapEntry) == 128, "TensorMapEntry must be exactly 2 cache lines (128 bytes)"); +static_assert(offsetof(ChipTensorMapEntry, buffer_addr) == offsetof(ChipTensor, buffer.addr)); +static_assert(offsetof(ChipTensorMapEntry, producer_task_id) == offsetof(ChipTensor, owner_task_id)); +static_assert(offsetof(ChipTensorMapEntry, start_offset) == offsetof(ChipTensor, start_offset)); +static_assert(offsetof(ChipTensorMapEntry, version) == offsetof(ChipTensor, version)); +static_assert(offsetof(ChipTensorMapEntry, ndims) == offsetof(ChipTensor, ndims)); +static_assert(offsetof(ChipTensorMapEntry, dtype) == offsetof(ChipTensor, dtype)); +static_assert(offsetof(ChipTensorMapEntry, manual_dep) == offsetof(ChipTensor, manual_dep)); +static_assert(offsetof(ChipTensorMapEntry, is_contiguous) == offsetof(ChipTensor, is_contiguous)); +static_assert(offsetof(ChipTensorMapEntry, shapes) == offsetof(ChipTensor, shapes)); static_assert( - offsetof(PTO2TensorMapEntry, prev_in_bucket) == 64, "TensorMapEntry must be exactly 2 cache lines (128 bytes)" + offsetof(ChipTensorMapEntry, prev_in_bucket) == 64, "TensorMapEntry must be exactly 2 cache lines (128 bytes)" ); // ============================================================================= @@ -359,22 +359,22 @@ static_assert( * * Hash table with ring buffer entry pool and lazy invalidation. */ -struct PTO2TensorMap { +struct ChipTensorMap { // Hash table buckets (fixed size, power of 2) - PTO2TensorMapEntry **buckets; // Array of offsets into entry_pool (-1 = empty) + ChipTensorMapEntry **buckets; // Array of offsets into entry_pool (-1 = empty) uint32_t *bucket_epochs; int32_t num_buckets; // Must be power of 2 for fast modulo // Entry pool as ring buffer - PTO2TensorMapEntry *entry_pool; // Ring buffer of entries - PTO2TensorMapEntry **free_entry_list; // free entry ids + ChipTensorMapEntry *entry_pool; // Ring buffer of entries + ChipTensorMapEntry **free_entry_list; // free entry ids int32_t pool_size; // Total pool capacity int32_t next_entry_idx; // id when next entry insert int32_t free_num; // free entry number in entry pool // Per-ring per-task entry tracking (for efficient bucket cleanup) // Indexed by [ring_id][local_id & (task_window_sizes[ring_id] - 1)] - PTO2TensorMapEntry **task_entry_heads[CHIP_MAX_RING_DEPTH]; + ChipTensorMapEntry **task_entry_heads[CHIP_MAX_RING_DEPTH]; uint32_t *task_entry_head_epochs[CHIP_MAX_RING_DEPTH]; int32_t task_window_sizes[CHIP_MAX_RING_DEPTH]; // Per-ring task window size (for slot masking) uint32_t current_epoch{1}; @@ -419,18 +419,18 @@ struct PTO2TensorMap { } // new_entry only allocates memory, does not assign attributes - PTO2TensorMapEntry *new_entry() { + ChipTensorMapEntry *new_entry() { if (free_num > 0) { - PTO2TensorMapEntry *res = free_entry_list[--free_num]; + ChipTensorMapEntry *res = free_entry_list[--free_num]; debug_assert(res->bucket_index == -1); return res; } always_assert(next_entry_idx < pool_size); - PTO2TensorMapEntry *res = &entry_pool[next_entry_idx++]; + ChipTensorMapEntry *res = &entry_pool[next_entry_idx++]; return res; } - void free_entry(PTO2TensorMapEntry &entry) { + void free_entry(ChipTensorMapEntry &entry) { always_assert(entry.bucket_index != -1); // must still be in a bucket // Update predecessor's next pointer (O(1) via prev_in_bucket) @@ -465,15 +465,15 @@ struct PTO2TensorMap { * the returned layout descriptor. Must be called before the arena is * committed. */ - static PTO2TensorMapLayout reserve_layout( + static ChipTensorMapLayout reserve_layout( DeviceArena &arena, int32_t num_buckets, int32_t pool_size, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH] ); /** - * Same as reserve_layout() with default sizes (PTO2_TENSORMAP_NUM_BUCKETS, - * PTO2_TENSORMAP_POOL_SIZE). + * Same as reserve_layout() with default sizes (CHIP_TENSORMAP_NUM_BUCKETS, + * CHIP_TENSORMAP_POOL_SIZE). */ - static PTO2TensorMapLayout + static ChipTensorMapLayout reserve_layout_default(DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH]); /** @@ -483,14 +483,14 @@ struct PTO2TensorMap { * but does not store those addresses in struct fields. Safe to call on * a host arena that holds the prebuilt image. */ - bool init_data_from_layout(const PTO2TensorMapLayout &layout, DeviceArena &arena); - void reset_for_reuse(const PTO2TensorMapLayout &layout); + bool init_data_from_layout(const ChipTensorMapLayout &layout, DeviceArena &arena); + void reset_for_reuse(const ChipTensorMapLayout &layout); /** * Phase 3b: write the arena-internal pointer fields. Idempotent; * called once on the host arena and once on the AICPU after attach. */ - void wire_arena_pointers(const PTO2TensorMapLayout &layout, DeviceArena &arena); + void wire_arena_pointers(const ChipTensorMapLayout &layout, DeviceArena &arena); /** * Tear down state. Does not free memory — the arena owns the backing @@ -513,7 +513,7 @@ struct PTO2TensorMap { * for each overlapping valid entry. * Stale entries from different rings are skipped (not truncated). * - * The callback receives (PTO2TensorMapEntry &, OverlapStatus) and should + * The callback receives (ChipTensorMapEntry &, OverlapStatus) and should * return true to continue iteration, false to stop early. It is safe for * the callback to call remove_entry() on the current entry: next_in_bucket * is latched before invocation. @@ -527,7 +527,7 @@ struct PTO2TensorMap { if (bucket_epochs[bucket_index] != current_epoch) { return; } - PTO2TensorMapEntry *cur_entry = buckets[bucket_index]; + ChipTensorMapEntry *cur_entry = buckets[bucket_index]; #if SIMPLER_TENSORMAP_PROFILING g_lookup_count++; @@ -535,7 +535,7 @@ struct PTO2TensorMap { #endif while (cur_entry != nullptr) { - PTO2TensorMapEntry *next_entry = cur_entry->next_in_bucket; + ChipTensorMapEntry *next_entry = cur_entry->next_in_bucket; #if SIMPLER_TENSORMAP_PROFILING chain_len++; @@ -589,7 +589,7 @@ struct PTO2TensorMap { * @param producer_task_id Task ID of producer */ void insert(const ChipTensor &tensor, TaskId producer_task_id) { - PTO2TensorMapEntry *entry = new_entry(); + ChipTensorMapEntry *entry = new_entry(); entry->copy_from_tensor(tensor); link_entry(entry, tensor.buffer.addr, producer_task_id); } @@ -615,9 +615,9 @@ struct PTO2TensorMap { // Free only entries produced by the retiring local_id, unlinking // each from the chain; entries from other tasks stay linked. TaskId retired_task = TaskId::make(static_cast(ring_id), static_cast(local_id)); - PTO2TensorMapEntry *cur_entry = task_entry_heads[ring_id][task_slot]; + ChipTensorMapEntry *cur_entry = task_entry_heads[ring_id][task_slot]; while (cur_entry != nullptr) { - PTO2TensorMapEntry *next_entry = cur_entry->next_in_task; // free_entry clears it + ChipTensorMapEntry *next_entry = cur_entry->next_in_task; // free_entry clears it if (cur_entry->producer_task_id == retired_task) { if (cur_entry->prev_in_task != nullptr) { cur_entry->prev_in_task->next_in_task = next_entry; @@ -654,7 +654,7 @@ struct PTO2TensorMap { /** * Link an initialized entry into bucket and task chains. */ - void link_entry(PTO2TensorMapEntry *entry, uint64_t addr, TaskId producer_task_id) { + void link_entry(ChipTensorMapEntry *entry, uint64_t addr, TaskId producer_task_id) { #if SIMPLER_TENSORMAP_PROFILING g_insert_count++; #endif @@ -694,11 +694,11 @@ struct PTO2TensorMap { /** * Check if entry is valid (producer has not retired) */ - bool entry_valid(const PTO2TensorMapEntry &entry) const { + bool entry_valid(const ChipTensorMapEntry &entry) const { return static_cast(entry.producer_task_id.local()) >= last_task_alives[entry.producer_task_id.ring()]; } - void remove_entry(PTO2TensorMapEntry &entry) { + void remove_entry(ChipTensorMapEntry &entry) { remove_from_task(entry); free_entry(entry); } @@ -707,7 +707,7 @@ struct PTO2TensorMap { * Remove entry from its task chain (O(1) with prev pointer) * Called during pool wrap-around to unlink reused entries. */ - void remove_from_task(PTO2TensorMapEntry &entry) { + void remove_from_task(ChipTensorMapEntry &entry) { always_assert(entry.bucket_index != -1); // must still be in a bucket // Update predecessor's next pointer (O(1) via prev_in_task) if (entry.prev_in_task == nullptr) { @@ -757,7 +757,7 @@ struct PTO2TensorMap { }; #if SIMPLER_TENSORMAP_PROFILING -struct PTO2TensorMapProfilingData { +struct ChipTensorMapProfilingData { uint64_t lookup_chain_total; uint64_t lookup_count; int32_t lookup_chain_max; @@ -766,5 +766,5 @@ struct PTO2TensorMapProfilingData { uint64_t insert_count; }; -PTO2TensorMapProfilingData pto2_tensormap_get_profiling(); +ChipTensorMapProfilingData chip_tensormap_get_profiling(); #endif diff --git a/src/common/hierarchical/tensormap.h b/src/common/hierarchical/tensormap.h index e9f4b7c9b4..b2017c88e5 100644 --- a/src/common/hierarchical/tensormap.h +++ b/src/common/hierarchical/tensormap.h @@ -20,7 +20,7 @@ * * `TensorFootprint` is what tells them apart. Each key holds every producer whose written view is * still the last word on some byte of the backing, so `x[0]` and `x[1]` coexist under one key and - * neither resolves as the other's producer. This mirrors the L2 PTO2TensorMap, which hashes by + * neither resolves as the other's producer. This mirrors the L2 ChipTensorMap, which hashes by * base address only and walks the chain running the same overlap cascade — a bounding-range * reject, then a per-dimension hyper-rectangle test that separates two column blocks whose * bounding boxes interleave. @@ -29,7 +29,7 @@ * overlapping, and a producer a later write only partly covers stays live. So an extra edge is * possible where the truth is subtler, and a real dependency is never dropped. * - * Unlike the L2 PTO2TensorMap, this implementation: + * Unlike the L2 ChipTensorMap, this implementation: * - Uses std::unordered_map (no ring buffer entry pool) * - Cleans up a task's entries when it is CONSUMED, skipping the views a * newer same-run producer has taken over diff --git a/src/common/runtime_status/error_names.h b/src/common/runtime_status/error_names.h index d298ba1f15..5f542db01e 100644 --- a/src/common/runtime_status/error_names.h +++ b/src/common/runtime_status/error_names.h @@ -162,7 +162,7 @@ static inline const char *error_hint(int32_t code) { return "not expected in normal operation -- keep the device log and report it to the runtime " "maintainers; tuning the ring capacities will not help"; case SIMPLER_ERROR_TENSORMAP_OVERFLOW: - return "follow the preceding resource diagnostic: in HBG increase PTO2_TENSORMAP_POOL_SIZE " + return "follow the preceding resource diagnostic: in HBG increase CHIP_TENSORMAP_POOL_SIZE " "or shrink the graph; in a reclaiming runtime also inspect entry-reclaim progress"; case SIMPLER_ERROR_SCHEDULER_TIMEOUT: return "read the sub_class= line below first, then follow the S1-S5 table in the doc; raise " diff --git a/src/common/task_interface/buffer.h b/src/common/task_interface/buffer.h index 22975eed45..cec34933b8 100644 --- a/src/common/task_interface/buffer.h +++ b/src/common/task_interface/buffer.h @@ -490,7 +490,7 @@ inline TensorFootprint tensor_footprint(const Tensor &r) { /** * Do two views of the same backing touch a common byte, and does `probe` swallow `entry` whole? * - * Three stages, mirroring the L2 `PTO2TensorMap::check_overlap` this is the host-side counterpart + * Three stages, mirroring the L2 `ChipTensorMap::check_overlap` this is the host-side counterpart * of (`src/{arch}/runtime/tensormap_and_ringbuffer/runtime/tensormap.h`): * * 1. Bounding-range intersection. O(1), and the only stage a whole-backing footprint reaches. diff --git a/src/common/task_interface/tensor.h b/src/common/task_interface/tensor.h index 910aa07f5c..b33f9c0479 100644 --- a/src/common/task_interface/tensor.h +++ b/src/common/task_interface/tensor.h @@ -121,7 +121,7 @@ struct alignas(64) ChipTensor { // --- Copy / move / destroy --- // Kept trivially copyable (default copy = byte-for-byte) so other modules - // (PTO2TensorMapEntry::copy_from_tensor, TensorCreateInfo memcpy path) + // (ChipTensorMapEntry::copy_from_tensor, TensorCreateInfo memcpy path) // can rely on memcpy semantics. The contiguous fast-path optimization // lives in `init(const ChipTensor&)`; call sites that care should use // `result.init(*this)` instead of the default copy ctor. diff --git a/tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py b/tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py index 41f4d0e50a..75650636fd 100644 --- a/tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py +++ b/tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py @@ -287,7 +287,7 @@ def _post_validate(self, case, run_marker): _assert_annotations(deps) # A tensormap edge is the only one that carries the producer's slice, and - # it is read off the live PTO2TensorMapEntry before an INOUT+COVERED + # it is read off the live ChipTensorMapEntry before an INOUT+COVERED # lookup removes it — an empty producer block means that ordering broke. for e in deps["edges"]: if e["source"] != "tensormap": diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py index e8e39dc36f..4790c2325c 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py @@ -13,7 +13,7 @@ Verifies the end-to-end dep_gen pipeline on a2a3sim: ``/deps.json`` is produced by the host replay - (PTO2TensorMap replay → JSON edge list), and contains exactly the + (ChipTensorMap replay → JSON edge list), and contains exactly the 6 edges documented in example_orchestration.cpp. The capture path (host collector drains the device ring buffer into memory and feeds the replay directly — no submit_trace.bin on disk) is exercised diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py index cd4d38bdf0..4a25a5b369 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py @@ -13,7 +13,7 @@ Verifies the end-to-end dep_gen pipeline on a5sim: ``/deps.json`` is produced by the host replay - (PTO2TensorMap replay → JSON edge list), and contains exactly the + (ChipTensorMap replay → JSON edge list), and contains exactly the 6 edges documented in example_orchestration.cpp. The capture path (host collector drains the device ring buffer into memory and feeds the replay directly — no submit_trace.bin on disk) is exercised diff --git a/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py b/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py index 424383abc1..e0ccef3261 100644 --- a/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py +++ b/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py @@ -208,11 +208,11 @@ # reproducible scheduler-timeout e2e case (aicore_hang_orch.cpp). # # -- Reachable only by exhausting a fixed compile-time cap -- - # * TENSORMAP_OVERFLOW (11): the tensormap entry pool (PTO2_TENSORMAP_POOL_SIZE + # * TENSORMAP_OVERFLOW (11): the tensormap entry pool (CHIP_TENSORMAP_POOL_SIZE # == 65536) is compile-time and not shrinkable via runtime_env, so wedging it # needs a 65536-entry flood plus a stalled producer holding entries. Its latch # is also orchestrator-level (a reclaim deadlock), so even a device UT would be - # a runtime-integration test, not a PTO2TensorMap unit test. + # a runtime-integration test, not a ChipTensorMap unit test. # # TENSOR_WAIT_TIMEOUT (8) IS covered (onboard, both arches) — see the # tensor_wait_timeout case. diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 109139b4a7..57df448556 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -1020,7 +1020,7 @@ target_sources(test_a5_graph_activation PRIVATE ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/host_build_graph/runtime/shared/runtime.cpp ) -# PTO2TensorMap's out-of-line members (reserve_layout / init / valid_count) live +# ChipTensorMap's out-of-line members (reserve_layout / init / valid_count) live # in this .cpp; no other add_a2a3_hbg_runtime_test target needs them. target_sources(test_hbg_tensormap PRIVATE ${CMAKE_SOURCE_DIR}/../../../src/a2a3/runtime/host_build_graph/runtime/shared/tensormap.cpp diff --git a/tests/ut/cpp/a2a3/test_hbg_tensormap.cpp b/tests/ut/cpp/a2a3/test_hbg_tensormap.cpp index eeccba0cc2..1bfd4e1f7b 100644 --- a/tests/ut/cpp/a2a3/test_hbg_tensormap.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_tensormap.cpp @@ -9,9 +9,9 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Entry-lifetime tests for the host_build_graph copy of PTO2TensorMap. + * Entry-lifetime tests for the host_build_graph copy of ChipTensorMap. * - * This is a distinct type from the tensormap_and_ringbuffer PTO2TensorMap + * This is a distinct type from the tensormap_and_ringbuffer ChipTensorMap * covered by a2a3/test_tensormap.cpp: single-ring, no entry epochs, and — the * subject of this file — no completion-watermark retirement. A registered * output stays visible until dependency computation explicitly removes it as @@ -30,15 +30,15 @@ namespace { struct TestLookupResult { struct Entry { - PTO2TensorMapEntry *entry; + ChipTensorMapEntry *entry; OverlapStatus overlap_status; }; std::vector entries; int count = 0; }; -void run_lookup(PTO2TensorMap &tmap, const ChipTensor &tensor, TestLookupResult &out) { - tmap.lookup(tensor, [&](PTO2TensorMapEntry &e, OverlapStatus s) -> bool { +void run_lookup(ChipTensorMap &tmap, const ChipTensor &tensor, TestLookupResult &out) { + tmap.lookup(tensor, [&](ChipTensorMapEntry &e, OverlapStatus s) -> bool { out.entries.push_back({&e, s}); out.count++; return true; @@ -56,7 +56,7 @@ class HbgTensorMapTest : public ::testing::Test { static constexpr int32_t POOL_SIZE = 64; static constexpr int32_t WINDOW_SIZE = 32; - PTO2TensorMap tmap{}; + ChipTensorMap tmap{}; void SetUp() override { ASSERT_TRUE(tmap.init(NUM_BUCKETS, POOL_SIZE, WINDOW_SIZE)); } }; diff --git a/tests/ut/cpp/a2a3/test_tensormap.cpp b/tests/ut/cpp/a2a3/test_tensormap.cpp index a3626005bc..acc0cd746e 100644 --- a/tests/ut/cpp/a2a3/test_tensormap.cpp +++ b/tests/ut/cpp/a2a3/test_tensormap.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2TensorMap from tensormap.h / tensormap.cpp + * Unit tests for ChipTensorMap from tensormap.h / tensormap.cpp * * Tests hash-table-based producer lookup with overlap detection: * - Hash function distribution (golden-ratio multiplicative hash) @@ -37,19 +37,19 @@ // ============================================================================= // Test-local mirror of the old stack-buffered lookup result. PR #669 removed -// PTO2LookupResult in favor of a callback-based API; these tests collect +// LookupResult in favor of a callback-based API; these tests collect // matches into a vector-like struct so assertions remain readable. struct TestLookupResult { struct Entry { - PTO2TensorMapEntry *entry; + ChipTensorMapEntry *entry; OverlapStatus overlap_status; }; std::vector entries; int count = 0; }; -static void run_lookup(PTO2TensorMap &tmap, const ChipTensor &tensor, TestLookupResult &out) { - tmap.lookup(tensor, [&](PTO2TensorMapEntry &e, OverlapStatus s) -> bool { +static void run_lookup(ChipTensorMap &tmap, const ChipTensor &tensor, TestLookupResult &out) { + tmap.lookup(tensor, [&](ChipTensorMapEntry &e, OverlapStatus s) -> bool { out.entries.push_back({&e, s}); out.count++; return true; @@ -76,12 +76,12 @@ class TensorMapTest : public ::testing::Test { static constexpr int32_t POOL_SIZE = 64; static constexpr int32_t WINDOW_SIZE = 32; - PTO2TensorMap tmap{}; + ChipTensorMap tmap{}; DeviceArena arena; void SetUp() override { int32_t window_sizes[CHIP_MAX_RING_DEPTH] = {WINDOW_SIZE, WINDOW_SIZE, WINDOW_SIZE, WINDOW_SIZE}; - auto layout = PTO2TensorMap::reserve_layout(arena, NUM_BUCKETS, POOL_SIZE, window_sizes); + auto layout = ChipTensorMap::reserve_layout(arena, NUM_BUCKETS, POOL_SIZE, window_sizes); ASSERT_NE(arena.commit(), nullptr); ASSERT_TRUE(tmap.init_data_from_layout(layout, arena)); tmap.wire_arena_pointers(layout, arena); @@ -109,10 +109,10 @@ TEST_F(TensorMapTest, InitRequiresPowerOfTwoBuckets) { // Non-power-of-2 bucket counts trip an always_assert inside reserve_layout // (asserting EXPECT_DEATH is impossible in release builds where // always_assert may compile out). Smoke-test only the success path here. - PTO2TensorMap bad{}; + ChipTensorMap bad{}; DeviceArena bad_arena; int32_t ws[CHIP_MAX_RING_DEPTH] = {8, 8, 8, 8}; - auto layout = PTO2TensorMap::reserve_layout(bad_arena, 8, 64, ws); + auto layout = ChipTensorMap::reserve_layout(bad_arena, 8, 64, ws); ASSERT_NE(bad_arena.commit(), nullptr); EXPECT_TRUE(bad.init_data_from_layout(layout, bad_arena)); bad.wire_arena_pointers(layout, bad_arena); diff --git a/tests/ut/cpp/a5/test_tensormap.cpp b/tests/ut/cpp/a5/test_tensormap.cpp index 67f5e67268..7ae1de1691 100644 --- a/tests/ut/cpp/a5/test_tensormap.cpp +++ b/tests/ut/cpp/a5/test_tensormap.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2TensorMap from tensormap.h / tensormap.cpp + * Unit tests for ChipTensorMap from tensormap.h / tensormap.cpp * * Tests hash-table-based producer lookup with overlap detection: * - Hash function distribution (golden-ratio multiplicative hash) @@ -37,19 +37,19 @@ // ============================================================================= // Test-local mirror of the old stack-buffered lookup result. PR #669 removed -// PTO2LookupResult in favor of a callback-based API; these tests collect +// LookupResult in favor of a callback-based API; these tests collect // matches into a vector-like struct so assertions remain readable. struct TestLookupResult { struct Entry { - PTO2TensorMapEntry *entry; + ChipTensorMapEntry *entry; OverlapStatus overlap_status; }; std::vector entries; int count = 0; }; -static void run_lookup(PTO2TensorMap &tmap, const ChipTensor &tensor, TestLookupResult &out) { - tmap.lookup(tensor, [&](PTO2TensorMapEntry &e, OverlapStatus s) -> bool { +static void run_lookup(ChipTensorMap &tmap, const ChipTensor &tensor, TestLookupResult &out) { + tmap.lookup(tensor, [&](ChipTensorMapEntry &e, OverlapStatus s) -> bool { out.entries.push_back({&e, s}); out.count++; return true; @@ -76,12 +76,12 @@ class TensorMapTest : public ::testing::Test { static constexpr int32_t POOL_SIZE = 64; static constexpr int32_t WINDOW_SIZE = 32; - PTO2TensorMap tmap{}; + ChipTensorMap tmap{}; DeviceArena arena; void SetUp() override { int32_t window_sizes[CHIP_MAX_RING_DEPTH] = {WINDOW_SIZE, WINDOW_SIZE, WINDOW_SIZE, WINDOW_SIZE}; - auto layout = PTO2TensorMap::reserve_layout(arena, NUM_BUCKETS, POOL_SIZE, window_sizes); + auto layout = ChipTensorMap::reserve_layout(arena, NUM_BUCKETS, POOL_SIZE, window_sizes); ASSERT_NE(arena.commit(), nullptr); ASSERT_TRUE(tmap.init_data_from_layout(layout, arena)); tmap.wire_arena_pointers(layout, arena); @@ -110,10 +110,10 @@ TEST_F(TensorMapTest, InitWithPowerOfTwoBucketsSucceeds) { // always_assert inside reserve_layout. It is not asserted here because // EXPECT_DEATH cannot run reliably in release builds where always_assert // may compile out. Cover only the accepted (power-of-2) shape. - PTO2TensorMap ok{}; + ChipTensorMap ok{}; DeviceArena ok_arena; int32_t ws[CHIP_MAX_RING_DEPTH] = {8, 8, 8, 8}; - auto layout = PTO2TensorMap::reserve_layout(ok_arena, 8, 64, ws); + auto layout = ChipTensorMap::reserve_layout(ok_arena, 8, 64, ws); ASSERT_NE(ok_arena.commit(), nullptr); EXPECT_TRUE(ok.init_data_from_layout(layout, ok_arena)); ok.wire_arena_pointers(layout, ok_arena); From 016446e57d51a94a075602925e8a8b89fd6ca78c Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:20:50 -0700 Subject: [PATCH 07/22] Refactor: name the chip runtime's ready queue by its context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PTO2ReadyQueue` becomes `ChipReadyQueue` — not `ReadyQueue`, because `src/common/hierarchical/types.h:509` already declares a `class ReadyQueue` for the host orchestrator. This one is the device-side sharded queue the AICPU scheduler threads claim from. Rule 13 gives the bare name to the L3+ side. `ChipReadyQueueSlot` and the two capacity constants `CHIP_READY_QUEUE_SIZE` / `CHIP_PROF_READYQUEUE_SIZE` follow it, so the queue and its slot type read the same way. `PTO2LocalReadyBuffer` is a plain strip to `LocalReadyBuffer` — it is a per-thread staging buffer rather than a queue, and nothing else claims the name. 212 occurrences across 32 files. Co-Authored-By: Claude Opus 5 (1M context) --- ...al-buffer-removal-ep-combine-regression.md | 6 +- docs/investigations/README.md | 2 +- .../host_build_graph/docs/RUNTIME_LOGIC.md | 2 +- .../host_build_graph/runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler.h | 58 +++++++++---------- .../runtime/scheduler/scheduler_context.h | 6 +- .../runtime/scheduler/scheduler_dispatch.cpp | 8 +-- .../runtime/shared/runtime_init.cpp | 24 ++++---- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 2 +- .../runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler.h | 50 ++++++++-------- .../runtime/scheduler/scheduler_context.h | 6 +- .../runtime/scheduler/scheduler_dispatch.cpp | 8 +-- .../runtime/shared/runtime_init.cpp | 20 +++---- .../host_build_graph/docs/RUNTIME_LOGIC.md | 2 +- .../host_build_graph/runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler.h | 58 +++++++++---------- .../runtime/scheduler/scheduler_context.h | 6 +- .../runtime/scheduler/scheduler_dispatch.cpp | 8 +-- .../runtime/shared/runtime_init.cpp | 24 ++++---- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 2 +- .../runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler.h | 48 +++++++-------- .../runtime/scheduler/scheduler_context.h | 6 +- .../runtime/scheduler/scheduler_dispatch.cpp | 8 +-- .../runtime/shared/runtime_init.cpp | 20 +++---- tests/ut/cpp/a2a3/test_ready_queue.cpp | 8 +-- tests/ut/cpp/a5/test_ready_queue.cpp | 8 +-- tests/ut/cpp/common/test_hbg_graph_cache.cpp | 4 +- .../cpp/common/test_hbg_ready_queue_seed.cpp | 16 ++--- 32 files changed, 211 insertions(+), 211 deletions(-) diff --git a/docs/investigations/2026-07-local-buffer-removal-ep-combine-regression.md b/docs/investigations/2026-07-local-buffer-removal-ep-combine-regression.md index 8d7a371b5e..ea738dcc03 100644 --- a/docs/investigations/2026-07-local-buffer-removal-ep-combine-regression.md +++ b/docs/investigations/2026-07-local-buffer-removal-ep-combine-regression.md @@ -1,10 +1,10 @@ -# 2026-07 — Removing PTO2LocalReadyBuffer exposed a missing dcci in EP dispatch +# 2026-07 — Removing LocalReadyBuffer exposed a missing dcci in EP dispatch ## Status **RESOLVED** in PR #1245. Root cause was a latent kernel bug (dispatch never flushed `recv_count_out` to HBM), unmasked by the dispatch-timing change from -removing `PTO2LocalReadyBuffer`. Fixed by a one-line `dcci` in the example +removing `LocalReadyBuffer`. Fixed by a one-line `dcci` in the example kernel; the local-buffer removal itself is correct. Onboard a2a3 3/3 pass after the fix. See "Fix" below. @@ -110,7 +110,7 @@ because they go out through `TSTORE` (a real vector→GM write). `recv_count_out sits in cache; whether the downstream `local_expert` task's AICore sees it in HBM depended on incidental timing. -Removing `PTO2LocalReadyBuffer` changed AICPU dispatch timing enough that +Removing `LocalReadyBuffer` changed AICPU dispatch timing enough that local_expert now reads `recv_count_out` before dispatch's cached scalar store lands in HBM → every expert runs 0 rows → recv_y all-zero → combine faithfully pushes zeros → routed_y all-zero. Baseline timing happened to let the store diff --git a/docs/investigations/README.md b/docs/investigations/README.md index 8ab24e8f2d..6f301ff26e 100644 --- a/docs/investigations/README.md +++ b/docs/investigations/README.md @@ -93,7 +93,7 @@ Newest first. - [2026-07 — Containing A2/A3 SDMA stream teardown after an AICore fault](2026-07-a2a3-sdma-fault-teardown.md) — CANN exposes no remote-channel retirement fence: even device-confirmed stream frees followed by a soft-reset success that did not complete context teardown can add minutes to a later ordinary fault. Contained by making SDMA an explicit per-Worker opt-in (`enable_sdma`) so only opt-in Workers carry the slow teardown and ordinary Workers keep fast recovery; full closure needs CANN runtime + driver changes (#1425) - [2026-07 — AICore-side arg fill for ALL dispatches (not just the gated path)](2026-07-aicore-fills-all-args-ready-path.md) — measured & rejected for the ready path: making the AICore fill its own `args[]` for every task adds ~1.0 µs to each task's `receive→start` setup (paged_attention_unroll: 349 ns → 1356 ns), because a ready task has no idle doorbell gate to hide the fill. Offload is a win only on `not_ready` (early-dispatch), where the AICore already spins at the gate; shipped design keeps the AICPU filling ready tasks (#1328) - [2026-07 — chip swimlane AICore: switch-overhead source + FIN-early reorder & ACK-gate](2026-07-aicore-swimlane-switch-overhead-and-ack-gate.md) — measured: the ~0.8 µs inter-task switch is the record write-back `dcci(record,OUT)+dsb` (~0.5 µs) + payload setup (~0.28 µs), inherent and not reducible by moving FIN; the WAIT gap (p99 ~700 µs) dominates decode. Shipped: sample `end_time` after an early FIN, and an AICPU ACK-gate on buffer rotation (release the old buffer only when AICore ACKs the new buffer's first task) to close the FIN-before-record boundary race the reorder introduced -- [2026-07 — Removing PTO2LocalReadyBuffer exposed a missing dcci in EP dispatch](2026-07-local-buffer-removal-ep-combine-regression.md) — RESOLVED in #1245: local-buffer removal changed dispatch timing and unmasked a latent kernel bug (dispatch never dcci'd `recv_count_out` to HBM → local_expert read count=0 → all-zero output); fixed with a one-line dcci in the example kernel +- [2026-07 — Removing LocalReadyBuffer exposed a missing dcci in EP dispatch](2026-07-local-buffer-removal-ep-combine-regression.md) — RESOLVED in #1245: local-buffer removal changed dispatch timing and unmasked a latent kernel bug (dispatch never dcci'd `recv_count_out` to HBM → local_expert read count=0 → all-zero output); fixed with a one-line dcci in the example kernel - [2026-06 — Gating the two residual profiling enable() calls on the orch/scheduler hot path](2026-06-orch-profiling-enable-gates-hot-path.md) — gated under existing `SIMPLER_DFX`; magnitude unmeasured, no new macro - [2026-06 — Replacing COND with GM+dcci for AICore→AICPU notification](2026-06-cond-vs-gm-notification.md) - [2026-06 — Letting AICore directly read or write the SPR MMIO window](2026-06-aicore-mmio-to-spr.md) diff --git a/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md index 7e4f1ac7be..7ca208cabd 100644 --- a/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/host_build_graph/docs/RUNTIME_LOGIC.md @@ -150,7 +150,7 @@ uninitialized queue. slot's sequence tracks the position it serves, and `pop` releases a slot with exactly the value the next lap's `push` expects. A drained queue is therefore already an empty queue, which is why `tensormap_and_ringbuffer` can leave -`PTO2ReadyQueue::reset_for_reuse()` empty and never touch the positions. hbg +`ChipReadyQueue::reset_for_reuse()` empty and never touch the positions. hbg re-establishes both on every attach today because the queue *headers* are reset per bind; the combination to avoid is resetting the positions while leaving the sequences mid-lap, which makes `push` read a sequence above its position and spin diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index f429363d09..b6e7f21356 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -125,7 +125,7 @@ inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HE // simultaneously ready in any one queue. Overflow on the ready/sync/dummy queues // latches SIMPLER_ERROR_READY_QUEUE_OVERFLOW (safe-fail), so it must exceed the // worst-case ready burst with margin. -#define PTO2_READY_QUEUE_SIZE 8192 +#define CHIP_READY_QUEUE_SIZE 8192 // Cross-thread early-dispatch work queue (power of two) #define PTO2_EARLY_DISPATCH_QUEUE_SIZE 64 diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h index 6b5385bab5..e8c39a9fb2 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -61,7 +61,7 @@ /** * Per-slot entry: sequence counter for ABA safety + task payload */ -struct PTO2ReadyQueueSlot { +struct ChipReadyQueueSlot { std::atomic sequence; ChipTaskSlotState *slot_state; uint64_t task_id_snapshot; @@ -77,8 +77,8 @@ struct PTO2ReadyQueueSlot { * - CAS contention is split: producers only touch enqueue_pos, * consumers only touch dequeue_pos */ -struct alignas(64) PTO2ReadyQueue { - PTO2ReadyQueueSlot *slots; +struct alignas(64) ChipReadyQueue { + ChipReadyQueueSlot *slots; uint64_t capacity; uint64_t mask; // capacity - 1 char _pad0[64 - 24]; // Pad to own cache line @@ -125,7 +125,7 @@ struct alignas(64) PTO2ReadyQueue { bool push_tagged(ChipTaskSlotState *slot_state, uint64_t task_id_snapshot) { uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; while (true) { pos = enqueue_pos.load(std::memory_order_relaxed); slot = &slots[pos & mask]; @@ -166,7 +166,7 @@ struct alignas(64) PTO2ReadyQueue { pos = enqueue_pos.load(std::memory_order_relaxed); bool ready = true; for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + i); if (diff < 0) { @@ -188,7 +188,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; slot->slot_state = items[i]; slot->task_id_snapshot = task_id_snapshots == nullptr ? 0 : task_id_snapshots[i]; slot->sequence.store(static_cast(pos + i + 1), std::memory_order_release); @@ -200,7 +200,7 @@ struct alignas(64) PTO2ReadyQueue { #if SIMPLER_ORCH_PROFILING || SIMPLER_SCHED_PROFILING bool push(ChipTaskSlotState *slot_state, uint64_t &atomic_count, uint64_t &wait_cycle) { uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; uint64_t t0 = get_sys_cnt_aicpu(); bool contended = false; uint32_t atomic_ops = 0; @@ -248,7 +248,7 @@ struct alignas(64) PTO2ReadyQueue { } uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; while (true) { pos = dequeue_pos.load(std::memory_order_relaxed); slot = &slots[pos & mask]; @@ -281,7 +281,7 @@ struct alignas(64) PTO2ReadyQueue { } uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; uint64_t t0 = get_sys_cnt_aicpu(); bool contended = false; uint32_t atomic_ops = 0; @@ -330,7 +330,7 @@ struct alignas(64) PTO2ReadyQueue { pos = dequeue_pos.load(std::memory_order_relaxed); count = 0; while (count < max_count) { - PTO2ReadyQueueSlot *slot = &slots[(pos + count) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + count) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + count + 1); if (diff == 0) { @@ -353,7 +353,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; out[i] = slot->slot_state; if (task_id_snapshots != nullptr) task_id_snapshots[i] = slot->task_id_snapshot; slot->sequence.store(static_cast(pos + i + mask + 1), std::memory_order_release); @@ -373,7 +373,7 @@ struct alignas(64) PTO2ReadyQueue { atomic_ops++; // dequeue_pos.load count = 0; while (count < max_count) { - PTO2ReadyQueueSlot *slot = &slots[(pos + count) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + count) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + count + 1); atomic_ops++; // sequence.load @@ -406,7 +406,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; out[i] = slot->slot_state; slot->sequence.store(static_cast(pos + i + mask + 1), std::memory_order_release); atomic_ops++; // sequence.store @@ -421,7 +421,7 @@ struct alignas(64) PTO2ReadyQueue { }; // Cold-path ready queue operations (defined in scheduler.cpp). Declared -// as non-member so PTO2ReadyQueue stays a POD-like struct with cache-line +// as non-member so ChipReadyQueue stays a POD-like struct with cache-line // alignment. Storage is owned by the caller-supplied arena. // reserve_layout: declare the slots[] region on the arena (must precede commit) // init_from_layout: initialize the queue header (capacity, mask, positions) @@ -430,12 +430,12 @@ struct alignas(64) PTO2ReadyQueue { size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity); // Writes the header fields only: capacity, mask, positions, occupancy counter. // The slot array is neither addressed nor written — it lives past the uploaded -// range and PTO2ReadyQueue::seed_slots() fills it on the device. Call +// range and ChipReadyQueue::seed_slots() fills it on the device. Call // `ready_queue_wire_arena_pointers` to set the `slots` pointer itself. -void ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, uint64_t capacity); +void ready_queue_init_data_from_layout(ChipReadyQueue *queue, uint64_t capacity); // Stores queue->slots = arena.region_ptr(slots_off). Idempotent. -void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off); -void ready_queue_destroy(PTO2ReadyQueue *queue); +void ready_queue_wire_arena_pointers(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off); +void ready_queue_destroy(ChipReadyQueue *queue); /** * Statistics returned by mixed-task completion processing @@ -489,24 +489,24 @@ struct SchedulerState { } ring_sched_state; // Ready queues remain global (scheduling is ring-agnostic) - PTO2ReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; // Ready sync_start queues, one per shape. A ready sync_start cohort parks here // instead of ready_queues[] so the dispatch loop can drain it as a strict Tier-0 // (sync_start > MIX > C/V) before any regular ready task takes a core, while // reusing the same per-shape dispatch_shape machinery (fits-local inline vs // stop-the-world drain, per-core MIX placement, head-start spacing). - PTO2ReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; // Dependency-only tasks (active_mask is empty, shape == DUMMY). Drained by // the dispatch loop and completed inline -- never goes to AICore. - PTO2ReadyQueue dummy_ready_queue; + ChipReadyQueue dummy_ready_queue; // An outer Graph is control work, never an AICore task. External dependency // readiness and bounded materialization progress independently and meet at // the submission's single atomic activation gate. - PTO2ReadyQueue graph_ready_queue; - PTO2ReadyQueue graph_prepare_queue; + ChipReadyQueue graph_ready_queue; + ChipReadyQueue graph_prepare_queue; alignas(64) AsyncWaitList async_wait_list; @@ -560,7 +560,7 @@ struct SchedulerState { } // Every ready / sync / dummy / graph task routes to exactly one queue. A // false push means that queue's peak concurrent occupancy exceeded - // PTO2_READY_QUEUE_SIZE — a capacity mis-sizing, not a normal condition. + // CHIP_READY_QUEUE_SIZE — a capacity mis-sizing, not a normal condition. // Silently dropping the task would stall the run, so latch a named error // (surfaces as READY_QUEUE_OVERFLOW rather than an anonymous // forward-progress timeout). The graph_ready push is checked identically @@ -689,7 +689,7 @@ struct SchedulerState { }; EarlyDispatchDoorbell early_dispatch_doorbell_table[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; - // Cross-thread early-dispatch work queues, one PTO2ReadyQueue MPMC instance per + // Cross-thread early-dispatch work queues, one ChipReadyQueue MPMC instance per // resource shape (AIC/AIV/MIX) — arena-backed, reserved/wired in pto_runtime2_init // alongside the per-shape ready queues, and indexed the same way. A candidate is // pushed to the queue for its own shape (active_mask.to_shape()) so the drain can @@ -707,7 +707,7 @@ struct SchedulerState { // the ready queue (scheduler_dispatch: pop -> claim -> re-push). A stale/released // entry fails the STAGING check on pop and is dropped; a push that overflows is // logged and the consumer's blocks fall back to normal dispatch. - PTO2ReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; // sync_start early-dispatch candidates park here instead of early_dispatch_queues[]: // they need an atomic all-or-nothing stage, not early_dispatch_shape's @@ -720,7 +720,7 @@ struct SchedulerState { // per-shape dispatch_shape. An EARLY sync cohort carries a non-zero src_payload // gate; its owner stages locally when one tracker fits and uses the global drain // otherwise. HBG producer propagation currently leaves this queue dormant. - PTO2ReadyQueue early_sync_start_queue; + ChipReadyQueue early_sync_start_queue; static inline void ring_one_doorbell(uint64_t reg_addr, uint32_t token) { volatile uint64_t *dmb = reinterpret_cast(get_reg_ptr(reg_addr, RegId::DATA_MAIN_BASE)); @@ -879,13 +879,13 @@ struct SchedulerState { // ready_sync_queues (unaffected). void propagate_dispatch_fanin(ChipTaskSlotState & /*p*/) {} - int get_ready_tasks_batch(PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { + int get_ready_tasks_batch(ChipReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { return queues[static_cast(shape)].pop_batch(out, max_count); } #if SIMPLER_SCHED_PROFILING int get_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, + ChipReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, uint64_t &wait_cycle ) { return queues[static_cast(shape)].pop_batch(out, max_count, atomic_count, wait_cycle); diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index b80cfd6c8a..41019cf4cc 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -299,7 +299,7 @@ class SchedulerContext { } int pop_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + ChipReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ); void build_payload( @@ -385,7 +385,7 @@ class SchedulerContext { ); void dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, ChipReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ); @@ -451,7 +451,7 @@ class SchedulerContext { bool has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const; // True if mix tasks remain in the global MIX ready queue. Approximate — - // PTO2ReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue + // ChipReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 1713aae3c3..7d2b580660 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -92,7 +92,7 @@ bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, Resour } int SchedulerContext::pop_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + ChipReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ) { #if SIMPLER_DFX auto &chip_swimlane = sched_chip_swimlane_[thread_idx]; @@ -313,7 +313,7 @@ int SchedulerContext::prepare_block_for_dispatch( } void SchedulerContext::dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, ChipReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ) { #if SIMPLER_SCHED_PROFILING @@ -1109,7 +1109,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches NUM_RESOURCE_SHAPES: AIC/AIV/MIX. // - // **Hot-path cost discipline.** Shared depth (PTO2ReadyQueue::size) is two + // **Hot-path cost discipline.** Shared depth (ChipReadyQueue::size) is two // atomic relaxed loads against cache lines that all peer sched threads also // write to (enqueue_pos and dequeue_pos bounce on every push + every pop). // With both phases emitting per iter that's cross-core loads × thousands of @@ -1126,7 +1126,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ bool iter_shared_sampled = false; auto get_or_sample_shared = [&]() -> const int16_t * { if (!iter_shared_sampled) { - // Clamp to int16_t max before narrowing. PTO2_PROF_READYQUEUE_SIZE + // Clamp to int16_t max before narrowing. CHIP_PROF_READYQUEUE_SIZE // is in the low thousands today but could grow with platform // scaling — without clamp, sizes above 32767 wrap to negatives // and silently corrupt the snapshot. diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 5b87b39445..cc46987eb1 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -36,13 +36,13 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Align the slots[] base to a full cache line so MPMC CAS traffic on the // first slot cannot false-share with whatever region sits in front of us. - return arena.reserve(capacity * sizeof(PTO2ReadyQueueSlot), PTO2_ALIGN_SIZE); + return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), PTO2_ALIGN_SIZE); } // Initialize the queue header only. slots[] carries the sequence ramp push // compares against, but it lives past the uploaded range and is seeded on the // device by SchedulerState::seed_queue_slots(), so nothing writes it here. -void ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, uint64_t capacity) { +void ready_queue_init_data_from_layout(ChipReadyQueue *queue, uint64_t capacity) { queue->capacity = capacity; queue->mask = capacity - 1; queue->enqueue_pos.store(0, std::memory_order_relaxed); @@ -50,11 +50,11 @@ void ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, uint64_t capacity) queue->max_occupancy.store(0, std::memory_order_relaxed); } -void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off) { - queue->slots = static_cast(arena.region_ptr(slots_off)); +void ready_queue_wire_arena_pointers(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off) { + queue->slots = static_cast(arena.region_ptr(slots_off)); } -void ready_queue_destroy(PTO2ReadyQueue *queue) { +void ready_queue_destroy(ChipReadyQueue *queue) { // Arena owns the slots[] buffer; just forget the pointer. queue->slots = nullptr; } @@ -80,9 +80,9 @@ void SchedulerState::RingSchedState::destroy() { ring = nullptr; } SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena) { SchedulerLayout layout{}; - layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; + layout.ready_queue_capacity = CHIP_READY_QUEUE_SIZE; - // Fixed-capacity early-dispatch queues first, then the PTO2_READY_QUEUE_SIZE + // Fixed-capacity early-dispatch queues first, then the CHIP_READY_QUEUE_SIZE // ones. The big nine are the arena's last reservations so that the bytes bind // uploads stay one contiguous range no matter how much of them is in use. for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { @@ -90,14 +90,14 @@ SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena) { } layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } - layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); - layout.off_graph_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); - layout.off_graph_prepare_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); + layout.off_graph_ready_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); + layout.off_graph_prepare_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); // Polling: no dep_pool arena region — producer dependencies are inline ids on // the payload and readiness is via completion_flags. return layout; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index daa94eae31..fe6f3404fe 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -167,7 +167,7 @@ bool entry_valid(const ChipTensorMapEntry& e) { | --------- | ------ | | `PTO2DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | | `TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | -| `PTO2ReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | +| `ChipReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | | `PTO2DispatchPayload` | Built per-dispatch, no ring state needed | ## 5. Reclamation diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 7351701b4e..dde9c1e577 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -566,7 +566,7 @@ After these phases, the scheduler updates profiling headers and checks for termi Ready queues use a lock-free bounded MPMC (Vyukov) design: -- One `PTO2ReadyQueue` per resource shape — 3 shapes (`NUM_RESOURCE_SHAPES`): `MIX` +- One `ChipReadyQueue` per resource shape — 3 shapes (`NUM_RESOURCE_SHAPES`): `MIX` (AIC+AIV cluster), `AIC`, `AIV`. Alongside `ready_queues[]` there is a per-shape `ready_sync_queues[]` (sync_start Tier-0) and the speculative `early_dispatch_queues[]` / `early_sync_start_queue` — see §8.6 for the full source × tier model. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index a16995a418..18231d70ad 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -84,7 +84,7 @@ #define PTO2_SCOPE_TASKS_CAP (PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) // Ready queue -#define PTO2_READY_QUEUE_SIZE 65536 // Per-shape queue size +#define CHIP_READY_QUEUE_SIZE 65536 // Per-shape queue size // Cross-thread early-dispatch candidate queue (power of two). A single wide // producer can publish more candidates than there are physical cores, so the diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 0a150c8b67..03601f08a3 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -57,7 +57,7 @@ /** * Per-slot entry: sequence counter for ABA safety + task payload */ -struct PTO2ReadyQueueSlot { +struct ChipReadyQueueSlot { std::atomic sequence; ChipTaskSlotState *slot_state; uint64_t task_id_snapshot; @@ -73,8 +73,8 @@ struct PTO2ReadyQueueSlot { * - CAS contention is split: producers only touch enqueue_pos, * consumers only touch dequeue_pos */ -struct alignas(64) PTO2ReadyQueue { - PTO2ReadyQueueSlot *slots; +struct alignas(64) ChipReadyQueue { + ChipReadyQueueSlot *slots; uint64_t capacity; uint64_t mask; // capacity - 1 char _pad0[64 - 24]; // Pad to own cache line @@ -97,7 +97,7 @@ struct alignas(64) PTO2ReadyQueue { bool push_tagged(ChipTaskSlotState *slot_state, uint64_t task_id_snapshot) { uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; while (true) { pos = enqueue_pos.load(std::memory_order_relaxed); slot = &slots[pos & mask]; @@ -137,7 +137,7 @@ struct alignas(64) PTO2ReadyQueue { pos = enqueue_pos.load(std::memory_order_relaxed); bool ready = true; for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + i); if (diff < 0) { @@ -159,7 +159,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; slot->slot_state = items[i]; slot->task_id_snapshot = task_id_snapshots == nullptr ? 0 : task_id_snapshots[i]; slot->sequence.store(static_cast(pos + i + 1), std::memory_order_release); @@ -170,7 +170,7 @@ struct alignas(64) PTO2ReadyQueue { #if SIMPLER_ORCH_PROFILING || SIMPLER_SCHED_PROFILING bool push(ChipTaskSlotState *slot_state, uint64_t &atomic_count, uint64_t &wait_cycle) { uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; uint64_t t0 = get_sys_cnt_aicpu(); bool contended = false; uint32_t atomic_ops = 0; @@ -218,7 +218,7 @@ struct alignas(64) PTO2ReadyQueue { } uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; while (true) { pos = dequeue_pos.load(std::memory_order_relaxed); slot = &slots[pos & mask]; @@ -251,7 +251,7 @@ struct alignas(64) PTO2ReadyQueue { } uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; uint64_t t0 = get_sys_cnt_aicpu(); bool contended = false; uint32_t atomic_ops = 0; @@ -300,7 +300,7 @@ struct alignas(64) PTO2ReadyQueue { pos = dequeue_pos.load(std::memory_order_relaxed); count = 0; while (count < max_count) { - PTO2ReadyQueueSlot *slot = &slots[(pos + count) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + count) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + count + 1); if (diff == 0) { @@ -323,7 +323,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; out[i] = slot->slot_state; if (task_id_snapshots != nullptr) task_id_snapshots[i] = slot->task_id_snapshot; slot->sequence.store(static_cast(pos + i + mask + 1), std::memory_order_release); @@ -343,7 +343,7 @@ struct alignas(64) PTO2ReadyQueue { atomic_ops++; // dequeue_pos.load count = 0; while (count < max_count) { - PTO2ReadyQueueSlot *slot = &slots[(pos + count) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + count) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + count + 1); atomic_ops++; // sequence.load @@ -376,7 +376,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; out[i] = slot->slot_state; slot->sequence.store(static_cast(pos + i + mask + 1), std::memory_order_release); atomic_ops++; // sequence.store @@ -391,7 +391,7 @@ struct alignas(64) PTO2ReadyQueue { }; // Cold-path ready queue operations (defined in scheduler.cpp). Declared -// as non-member so PTO2ReadyQueue stays a POD-like struct with cache-line +// as non-member so ChipReadyQueue stays a POD-like struct with cache-line // alignment. Storage is owned by the caller-supplied arena. // reserve_layout: declare the slots[] region on the arena (must precede commit) // init_data_from_layout: initialize sequence counters and queue metadata @@ -403,10 +403,10 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity); // arena.region_ptr(slots_off) only to address the slot array for writes; // does NOT store the pointer in `queue->slots`. Call // `ready_queue_wire_arena_pointers` afterwards to set the field itself. -bool ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity); +bool ready_queue_init_data_from_layout(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity); // Stores queue->slots = arena.region_ptr(slots_off). Idempotent. -void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off); -void ready_queue_destroy(PTO2ReadyQueue *queue); +void ready_queue_wire_arena_pointers(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off); +void ready_queue_destroy(ChipReadyQueue *queue); /** * Statistics returned by mixed-task completion processing @@ -512,18 +512,18 @@ struct SchedulerState { alignas(64) std::atomic advance_pending_mask; // Ready queues remain global (scheduling is ring-agnostic) - PTO2ReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; // Ready sync_start queues, one per shape. A ready sync_start cohort parks here // instead of ready_queues[] so the dispatch loop can drain it as a strict Tier-0 // (sync_start > MIX > C/V) before any regular ready task takes a core, while // reusing the same per-shape dispatch_shape machinery (fits-local inline vs // stop-the-world drain, per-core MIX placement, head-start spacing). - PTO2ReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; // Dependency-only tasks (active_mask is empty, shape == DUMMY). Drained by // the dispatch loop and completed inline -- never goes to AICore. - PTO2ReadyQueue dummy_ready_queue; + ChipReadyQueue dummy_ready_queue; alignas(64) AsyncWaitList async_wait_list; @@ -734,7 +734,7 @@ struct SchedulerState { }; EarlyDispatchDoorbell early_dispatch_doorbell_table[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; - // Cross-thread early-dispatch work queues, one PTO2ReadyQueue MPMC instance per + // Cross-thread early-dispatch work queues, one ChipReadyQueue MPMC instance per // resource shape (AIC/AIV/MIX) — arena-backed, reserved/wired in pto_runtime2_init // alongside the per-shape ready queues, and indexed the same way. A candidate is // pushed to the queue for its own shape (active_mask.to_shape()) so the drain can @@ -756,7 +756,7 @@ struct SchedulerState { // staged and sends the rest to ready_queues. An initial push that overflows also // rolls STAGING back to NONE, since nothing is staged yet and a task with no // claim should not read as speculatively claimed. - PTO2ReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; // sync_start early-dispatch candidates park here instead of early_dispatch_queues[]: // they need an atomic all-or-nothing owner stage, not early_dispatch_shape's @@ -770,7 +770,7 @@ struct SchedulerState { // src_payload gate; it stages directly when one owner tracker can hold the whole // cohort and falls back to the global drain otherwise. The shape-agnostic // rendezvous makes one queue sufficient. - PTO2ReadyQueue early_sync_start_queue; + ChipReadyQueue early_sync_start_queue; static inline void ring_one_doorbell(uint64_t reg_addr, uint32_t token) { volatile uint64_t *dmb = reinterpret_cast(get_reg_ptr(reg_addr, RegId::DATA_MAIN_BASE)); @@ -1179,13 +1179,13 @@ struct SchedulerState { } #endif - int get_ready_tasks_batch(PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { + int get_ready_tasks_batch(ChipReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { return queues[static_cast(shape)].pop_batch(out, max_count); } #if SIMPLER_SCHED_PROFILING int get_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, + ChipReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, uint64_t &wait_cycle ) { return queues[static_cast(shape)].pop_batch(out, max_count, atomic_count, wait_cycle); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 197aa80d43..e30245dfb2 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -249,7 +249,7 @@ class SchedulerContext { } int pop_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + ChipReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ); void build_payload( @@ -335,7 +335,7 @@ class SchedulerContext { ); void dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, ChipReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ); @@ -401,7 +401,7 @@ class SchedulerContext { bool has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const; // True if mix tasks remain in the global MIX ready queue. Approximate — - // PTO2ReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue + // ChipReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index f18ca65512..7672f5cb0b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -94,7 +94,7 @@ bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, Resour } int SchedulerContext::pop_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + ChipReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ) { #if SIMPLER_DFX auto &chip_swimlane = sched_chip_swimlane_[thread_idx]; @@ -308,7 +308,7 @@ int SchedulerContext::prepare_block_for_dispatch( } void SchedulerContext::dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, ChipReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ) { #if SIMPLER_SCHED_PROFILING @@ -942,7 +942,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches NUM_RESOURCE_SHAPES: AIC/AIV/MIX. // - // **Hot-path cost discipline.** Shared depth (PTO2ReadyQueue::size) is two + // **Hot-path cost discipline.** Shared depth (ChipReadyQueue::size) is two // atomic relaxed loads against cache lines that all peer sched threads also // write to (enqueue_pos and dequeue_pos bounce on every push + every pop). // With both phases emitting per iter that's cross-core loads × thousands of @@ -959,7 +959,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ bool iter_shared_sampled = false; auto get_or_sample_shared = [&]() -> const int16_t * { if (!iter_shared_sampled) { - // Clamp to int16_t max before narrowing. PTO2_PROF_READYQUEUE_SIZE + // Clamp to int16_t max before narrowing. CHIP_PROF_READYQUEUE_SIZE // is in the low thousands today but could grow with platform // scaling — without clamp, sizes above 32767 wrap to negatives // and silently corrupt the snapshot. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 8a01aa0f03..c98a19a53b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -52,13 +52,13 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Align the slots[] base to a full cache line so MPMC CAS traffic on the // first slot cannot false-share with whatever region sits in front of us // (e.g. orchestrator tensormap heads written by the orch thread). - return arena.reserve(capacity * sizeof(PTO2ReadyQueueSlot), PTO2_ALIGN_SIZE); + return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), PTO2_ALIGN_SIZE); } -bool ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity) { +bool ready_queue_init_data_from_layout(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity) { // Address the slots region for data writes without storing the pointer in // queue->slots — that field is set by ready_queue_wire_arena_pointers. - auto *slots_arena = static_cast(arena.region_ptr(slots_off)); + auto *slots_arena = static_cast(arena.region_ptr(slots_off)); queue->capacity = capacity; queue->mask = capacity - 1; queue->enqueue_pos.store(0, std::memory_order_relaxed); @@ -72,11 +72,11 @@ bool ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, DeviceArena &arena return true; } -void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off) { - queue->slots = static_cast(arena.region_ptr(slots_off)); +void ready_queue_wire_arena_pointers(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off) { + queue->slots = static_cast(arena.region_ptr(slots_off)); } -void ready_queue_destroy(PTO2ReadyQueue *queue) { +void ready_queue_destroy(ChipReadyQueue *queue) { // Arena owns the slots[] buffer; just forget the pointer. queue->slots = nullptr; } @@ -130,18 +130,18 @@ SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena, int32_t dep_p SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]) { SchedulerLayout layout{}; - layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; + layout.ready_queue_capacity = CHIP_READY_QUEUE_SIZE; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { layout.dep_pool_capacities[r] = dep_pool_capacities[r]; } for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } - layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); } diff --git a/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md b/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md index 7e4f1ac7be..7ca208cabd 100644 --- a/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/host_build_graph/docs/RUNTIME_LOGIC.md @@ -150,7 +150,7 @@ uninitialized queue. slot's sequence tracks the position it serves, and `pop` releases a slot with exactly the value the next lap's `push` expects. A drained queue is therefore already an empty queue, which is why `tensormap_and_ringbuffer` can leave -`PTO2ReadyQueue::reset_for_reuse()` empty and never touch the positions. hbg +`ChipReadyQueue::reset_for_reuse()` empty and never touch the positions. hbg re-establishes both on every attach today because the queue *headers* are reset per bind; the combination to avoid is resetting the positions while leaving the sequences mid-lap, which makes `push` read a sequence above its position and spin diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 2c7306630c..5d6b3487fe 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -126,7 +126,7 @@ inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HE // simultaneously ready in any one queue. Overflow on the ready/sync/dummy queues // latches SIMPLER_ERROR_READY_QUEUE_OVERFLOW (safe-fail), so it must exceed the // worst-case ready burst with margin. -#define PTO2_READY_QUEUE_SIZE 8192 +#define CHIP_READY_QUEUE_SIZE 8192 // Cross-thread early-dispatch work queue (power of two) #define PTO2_EARLY_DISPATCH_QUEUE_SIZE 64 diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h index 6b5385bab5..e8c39a9fb2 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -61,7 +61,7 @@ /** * Per-slot entry: sequence counter for ABA safety + task payload */ -struct PTO2ReadyQueueSlot { +struct ChipReadyQueueSlot { std::atomic sequence; ChipTaskSlotState *slot_state; uint64_t task_id_snapshot; @@ -77,8 +77,8 @@ struct PTO2ReadyQueueSlot { * - CAS contention is split: producers only touch enqueue_pos, * consumers only touch dequeue_pos */ -struct alignas(64) PTO2ReadyQueue { - PTO2ReadyQueueSlot *slots; +struct alignas(64) ChipReadyQueue { + ChipReadyQueueSlot *slots; uint64_t capacity; uint64_t mask; // capacity - 1 char _pad0[64 - 24]; // Pad to own cache line @@ -125,7 +125,7 @@ struct alignas(64) PTO2ReadyQueue { bool push_tagged(ChipTaskSlotState *slot_state, uint64_t task_id_snapshot) { uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; while (true) { pos = enqueue_pos.load(std::memory_order_relaxed); slot = &slots[pos & mask]; @@ -166,7 +166,7 @@ struct alignas(64) PTO2ReadyQueue { pos = enqueue_pos.load(std::memory_order_relaxed); bool ready = true; for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + i); if (diff < 0) { @@ -188,7 +188,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; slot->slot_state = items[i]; slot->task_id_snapshot = task_id_snapshots == nullptr ? 0 : task_id_snapshots[i]; slot->sequence.store(static_cast(pos + i + 1), std::memory_order_release); @@ -200,7 +200,7 @@ struct alignas(64) PTO2ReadyQueue { #if SIMPLER_ORCH_PROFILING || SIMPLER_SCHED_PROFILING bool push(ChipTaskSlotState *slot_state, uint64_t &atomic_count, uint64_t &wait_cycle) { uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; uint64_t t0 = get_sys_cnt_aicpu(); bool contended = false; uint32_t atomic_ops = 0; @@ -248,7 +248,7 @@ struct alignas(64) PTO2ReadyQueue { } uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; while (true) { pos = dequeue_pos.load(std::memory_order_relaxed); slot = &slots[pos & mask]; @@ -281,7 +281,7 @@ struct alignas(64) PTO2ReadyQueue { } uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; uint64_t t0 = get_sys_cnt_aicpu(); bool contended = false; uint32_t atomic_ops = 0; @@ -330,7 +330,7 @@ struct alignas(64) PTO2ReadyQueue { pos = dequeue_pos.load(std::memory_order_relaxed); count = 0; while (count < max_count) { - PTO2ReadyQueueSlot *slot = &slots[(pos + count) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + count) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + count + 1); if (diff == 0) { @@ -353,7 +353,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; out[i] = slot->slot_state; if (task_id_snapshots != nullptr) task_id_snapshots[i] = slot->task_id_snapshot; slot->sequence.store(static_cast(pos + i + mask + 1), std::memory_order_release); @@ -373,7 +373,7 @@ struct alignas(64) PTO2ReadyQueue { atomic_ops++; // dequeue_pos.load count = 0; while (count < max_count) { - PTO2ReadyQueueSlot *slot = &slots[(pos + count) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + count) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + count + 1); atomic_ops++; // sequence.load @@ -406,7 +406,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; out[i] = slot->slot_state; slot->sequence.store(static_cast(pos + i + mask + 1), std::memory_order_release); atomic_ops++; // sequence.store @@ -421,7 +421,7 @@ struct alignas(64) PTO2ReadyQueue { }; // Cold-path ready queue operations (defined in scheduler.cpp). Declared -// as non-member so PTO2ReadyQueue stays a POD-like struct with cache-line +// as non-member so ChipReadyQueue stays a POD-like struct with cache-line // alignment. Storage is owned by the caller-supplied arena. // reserve_layout: declare the slots[] region on the arena (must precede commit) // init_from_layout: initialize the queue header (capacity, mask, positions) @@ -430,12 +430,12 @@ struct alignas(64) PTO2ReadyQueue { size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity); // Writes the header fields only: capacity, mask, positions, occupancy counter. // The slot array is neither addressed nor written — it lives past the uploaded -// range and PTO2ReadyQueue::seed_slots() fills it on the device. Call +// range and ChipReadyQueue::seed_slots() fills it on the device. Call // `ready_queue_wire_arena_pointers` to set the `slots` pointer itself. -void ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, uint64_t capacity); +void ready_queue_init_data_from_layout(ChipReadyQueue *queue, uint64_t capacity); // Stores queue->slots = arena.region_ptr(slots_off). Idempotent. -void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off); -void ready_queue_destroy(PTO2ReadyQueue *queue); +void ready_queue_wire_arena_pointers(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off); +void ready_queue_destroy(ChipReadyQueue *queue); /** * Statistics returned by mixed-task completion processing @@ -489,24 +489,24 @@ struct SchedulerState { } ring_sched_state; // Ready queues remain global (scheduling is ring-agnostic) - PTO2ReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; // Ready sync_start queues, one per shape. A ready sync_start cohort parks here // instead of ready_queues[] so the dispatch loop can drain it as a strict Tier-0 // (sync_start > MIX > C/V) before any regular ready task takes a core, while // reusing the same per-shape dispatch_shape machinery (fits-local inline vs // stop-the-world drain, per-core MIX placement, head-start spacing). - PTO2ReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; // Dependency-only tasks (active_mask is empty, shape == DUMMY). Drained by // the dispatch loop and completed inline -- never goes to AICore. - PTO2ReadyQueue dummy_ready_queue; + ChipReadyQueue dummy_ready_queue; // An outer Graph is control work, never an AICore task. External dependency // readiness and bounded materialization progress independently and meet at // the submission's single atomic activation gate. - PTO2ReadyQueue graph_ready_queue; - PTO2ReadyQueue graph_prepare_queue; + ChipReadyQueue graph_ready_queue; + ChipReadyQueue graph_prepare_queue; alignas(64) AsyncWaitList async_wait_list; @@ -560,7 +560,7 @@ struct SchedulerState { } // Every ready / sync / dummy / graph task routes to exactly one queue. A // false push means that queue's peak concurrent occupancy exceeded - // PTO2_READY_QUEUE_SIZE — a capacity mis-sizing, not a normal condition. + // CHIP_READY_QUEUE_SIZE — a capacity mis-sizing, not a normal condition. // Silently dropping the task would stall the run, so latch a named error // (surfaces as READY_QUEUE_OVERFLOW rather than an anonymous // forward-progress timeout). The graph_ready push is checked identically @@ -689,7 +689,7 @@ struct SchedulerState { }; EarlyDispatchDoorbell early_dispatch_doorbell_table[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; - // Cross-thread early-dispatch work queues, one PTO2ReadyQueue MPMC instance per + // Cross-thread early-dispatch work queues, one ChipReadyQueue MPMC instance per // resource shape (AIC/AIV/MIX) — arena-backed, reserved/wired in pto_runtime2_init // alongside the per-shape ready queues, and indexed the same way. A candidate is // pushed to the queue for its own shape (active_mask.to_shape()) so the drain can @@ -707,7 +707,7 @@ struct SchedulerState { // the ready queue (scheduler_dispatch: pop -> claim -> re-push). A stale/released // entry fails the STAGING check on pop and is dropped; a push that overflows is // logged and the consumer's blocks fall back to normal dispatch. - PTO2ReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; // sync_start early-dispatch candidates park here instead of early_dispatch_queues[]: // they need an atomic all-or-nothing stage, not early_dispatch_shape's @@ -720,7 +720,7 @@ struct SchedulerState { // per-shape dispatch_shape. An EARLY sync cohort carries a non-zero src_payload // gate; its owner stages locally when one tracker fits and uses the global drain // otherwise. HBG producer propagation currently leaves this queue dormant. - PTO2ReadyQueue early_sync_start_queue; + ChipReadyQueue early_sync_start_queue; static inline void ring_one_doorbell(uint64_t reg_addr, uint32_t token) { volatile uint64_t *dmb = reinterpret_cast(get_reg_ptr(reg_addr, RegId::DATA_MAIN_BASE)); @@ -879,13 +879,13 @@ struct SchedulerState { // ready_sync_queues (unaffected). void propagate_dispatch_fanin(ChipTaskSlotState & /*p*/) {} - int get_ready_tasks_batch(PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { + int get_ready_tasks_batch(ChipReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { return queues[static_cast(shape)].pop_batch(out, max_count); } #if SIMPLER_SCHED_PROFILING int get_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, + ChipReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, uint64_t &wait_cycle ) { return queues[static_cast(shape)].pop_batch(out, max_count, atomic_count, wait_cycle); diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index f2fdba1daa..19d2265309 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -299,7 +299,7 @@ class SchedulerContext { } int pop_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + ChipReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ); void build_payload( @@ -385,7 +385,7 @@ class SchedulerContext { ); void dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, ChipReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ); @@ -451,7 +451,7 @@ class SchedulerContext { bool has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const; // True if mix tasks remain in the global MIX ready queue. Approximate — - // PTO2ReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue + // ChipReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 1713aae3c3..7d2b580660 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -92,7 +92,7 @@ bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, Resour } int SchedulerContext::pop_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + ChipReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ) { #if SIMPLER_DFX auto &chip_swimlane = sched_chip_swimlane_[thread_idx]; @@ -313,7 +313,7 @@ int SchedulerContext::prepare_block_for_dispatch( } void SchedulerContext::dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, ChipReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ) { #if SIMPLER_SCHED_PROFILING @@ -1109,7 +1109,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches NUM_RESOURCE_SHAPES: AIC/AIV/MIX. // - // **Hot-path cost discipline.** Shared depth (PTO2ReadyQueue::size) is two + // **Hot-path cost discipline.** Shared depth (ChipReadyQueue::size) is two // atomic relaxed loads against cache lines that all peer sched threads also // write to (enqueue_pos and dequeue_pos bounce on every push + every pop). // With both phases emitting per iter that's cross-core loads × thousands of @@ -1126,7 +1126,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ bool iter_shared_sampled = false; auto get_or_sample_shared = [&]() -> const int16_t * { if (!iter_shared_sampled) { - // Clamp to int16_t max before narrowing. PTO2_PROF_READYQUEUE_SIZE + // Clamp to int16_t max before narrowing. CHIP_PROF_READYQUEUE_SIZE // is in the low thousands today but could grow with platform // scaling — without clamp, sizes above 32767 wrap to negatives // and silently corrupt the snapshot. diff --git a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 5b87b39445..cc46987eb1 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -36,13 +36,13 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Align the slots[] base to a full cache line so MPMC CAS traffic on the // first slot cannot false-share with whatever region sits in front of us. - return arena.reserve(capacity * sizeof(PTO2ReadyQueueSlot), PTO2_ALIGN_SIZE); + return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), PTO2_ALIGN_SIZE); } // Initialize the queue header only. slots[] carries the sequence ramp push // compares against, but it lives past the uploaded range and is seeded on the // device by SchedulerState::seed_queue_slots(), so nothing writes it here. -void ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, uint64_t capacity) { +void ready_queue_init_data_from_layout(ChipReadyQueue *queue, uint64_t capacity) { queue->capacity = capacity; queue->mask = capacity - 1; queue->enqueue_pos.store(0, std::memory_order_relaxed); @@ -50,11 +50,11 @@ void ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, uint64_t capacity) queue->max_occupancy.store(0, std::memory_order_relaxed); } -void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off) { - queue->slots = static_cast(arena.region_ptr(slots_off)); +void ready_queue_wire_arena_pointers(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off) { + queue->slots = static_cast(arena.region_ptr(slots_off)); } -void ready_queue_destroy(PTO2ReadyQueue *queue) { +void ready_queue_destroy(ChipReadyQueue *queue) { // Arena owns the slots[] buffer; just forget the pointer. queue->slots = nullptr; } @@ -80,9 +80,9 @@ void SchedulerState::RingSchedState::destroy() { ring = nullptr; } SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena) { SchedulerLayout layout{}; - layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; + layout.ready_queue_capacity = CHIP_READY_QUEUE_SIZE; - // Fixed-capacity early-dispatch queues first, then the PTO2_READY_QUEUE_SIZE + // Fixed-capacity early-dispatch queues first, then the CHIP_READY_QUEUE_SIZE // ones. The big nine are the arena's last reservations so that the bytes bind // uploads stay one contiguous range no matter how much of them is in use. for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { @@ -90,14 +90,14 @@ SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena) { } layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } - layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); - layout.off_graph_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); - layout.off_graph_prepare_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); + layout.off_graph_ready_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); + layout.off_graph_prepare_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); // Polling: no dep_pool arena region — producer dependencies are inline ids on // the payload and readiness is via completion_flags. return layout; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 2e0471fd0c..d92cf4f5ff 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -167,7 +167,7 @@ bool entry_valid(const ChipTensorMapEntry& e) { | --------- | ------ | | `PTO2DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | | `TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | -| `PTO2ReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | +| `ChipReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | | `PTO2DispatchPayload` | Built per-dispatch, no ring state needed | ## 5. Reclamation diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index db35f3ff70..0afc3ee6fe 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -581,7 +581,7 @@ After these phases, the scheduler updates profiling headers and checks for termi Ready queues use a lock-free bounded MPMC (Vyukov) design: -- One `PTO2ReadyQueue` per resource shape — 3 shapes (`NUM_RESOURCE_SHAPES`): `MIX` +- One `ChipReadyQueue` per resource shape — 3 shapes (`NUM_RESOURCE_SHAPES`): `MIX` (AIC+AIV cluster), `AIC`, `AIV`. Alongside `ready_queues[]` there is a per-shape `ready_sync_queues[]` (sync_start Tier-0) and the speculative `early_dispatch_queues[]` / `early_sync_start_queue` — see §8.6 for the full source × tier model. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 8f3246bfa1..6c6204c203 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -84,7 +84,7 @@ #define PTO2_SCOPE_TASKS_CAP (PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) // Ready queue -#define PTO2_READY_QUEUE_SIZE 65536 // Per-shape queue size +#define CHIP_READY_QUEUE_SIZE 65536 // Per-shape queue size // Cross-thread early-dispatch candidate queue (power of two). A single wide // producer can publish more candidates than there are physical cores, so the diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 027a8aed6c..f91d6daed9 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -57,7 +57,7 @@ /** * Per-slot entry: sequence counter for ABA safety + task payload */ -struct PTO2ReadyQueueSlot { +struct ChipReadyQueueSlot { std::atomic sequence; ChipTaskSlotState *slot_state; uint64_t task_id_snapshot; // generation tag for early-dispatch queue entries @@ -73,8 +73,8 @@ struct PTO2ReadyQueueSlot { * - CAS contention is split: producers only touch enqueue_pos, * consumers only touch dequeue_pos */ -struct alignas(64) PTO2ReadyQueue { - PTO2ReadyQueueSlot *slots; +struct alignas(64) ChipReadyQueue { + ChipReadyQueueSlot *slots; uint64_t capacity; uint64_t mask; // capacity - 1 char _pad0[64 - 24]; // Pad to own cache line @@ -97,7 +97,7 @@ struct alignas(64) PTO2ReadyQueue { bool push_tagged(ChipTaskSlotState *slot_state, uint64_t task_id_snapshot) { uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; while (true) { pos = enqueue_pos.load(std::memory_order_relaxed); slot = &slots[pos & mask]; @@ -137,7 +137,7 @@ struct alignas(64) PTO2ReadyQueue { pos = enqueue_pos.load(std::memory_order_relaxed); bool ready = true; for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + i); if (diff < 0) { @@ -159,7 +159,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; slot->slot_state = items[i]; slot->task_id_snapshot = task_id_snapshots == nullptr ? 0 : task_id_snapshots[i]; slot->sequence.store(static_cast(pos + i + 1), std::memory_order_release); @@ -170,7 +170,7 @@ struct alignas(64) PTO2ReadyQueue { #if SIMPLER_ORCH_PROFILING || SIMPLER_SCHED_PROFILING bool push(ChipTaskSlotState *slot_state, uint64_t &atomic_count, uint64_t &wait_cycle) { uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; uint64_t t0 = get_sys_cnt_aicpu(); bool contended = false; uint32_t atomic_ops = 0; @@ -219,7 +219,7 @@ struct alignas(64) PTO2ReadyQueue { } uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; while (true) { pos = dequeue_pos.load(std::memory_order_relaxed); slot = &slots[pos & mask]; @@ -252,7 +252,7 @@ struct alignas(64) PTO2ReadyQueue { } uint64_t pos; - PTO2ReadyQueueSlot *slot; + ChipReadyQueueSlot *slot; uint64_t t0 = get_sys_cnt_aicpu(); bool contended = false; uint32_t atomic_ops = 0; @@ -301,7 +301,7 @@ struct alignas(64) PTO2ReadyQueue { pos = dequeue_pos.load(std::memory_order_relaxed); count = 0; while (count < max_count) { - PTO2ReadyQueueSlot *slot = &slots[(pos + count) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + count) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + count + 1); if (diff == 0) { @@ -324,7 +324,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; out[i] = slot->slot_state; if (task_id_snapshots != nullptr) task_id_snapshots[i] = slot->task_id_snapshot; slot->sequence.store(static_cast(pos + i + mask + 1), std::memory_order_release); @@ -344,7 +344,7 @@ struct alignas(64) PTO2ReadyQueue { atomic_ops++; // dequeue_pos.load count = 0; while (count < max_count) { - PTO2ReadyQueueSlot *slot = &slots[(pos + count) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + count) & mask]; int64_t seq = slot->sequence.load(std::memory_order_acquire); int64_t diff = seq - static_cast(pos + count + 1); atomic_ops++; // sequence.load @@ -377,7 +377,7 @@ struct alignas(64) PTO2ReadyQueue { } for (int i = 0; i < count; i++) { - PTO2ReadyQueueSlot *slot = &slots[(pos + i) & mask]; + ChipReadyQueueSlot *slot = &slots[(pos + i) & mask]; out[i] = slot->slot_state; slot->sequence.store(static_cast(pos + i + mask + 1), std::memory_order_release); atomic_ops++; // sequence.store @@ -392,7 +392,7 @@ struct alignas(64) PTO2ReadyQueue { }; // Cold-path ready queue operations (defined in scheduler.cpp). Declared -// as non-member so PTO2ReadyQueue stays a POD-like struct with cache-line +// as non-member so ChipReadyQueue stays a POD-like struct with cache-line // alignment. Storage is owned by the caller-supplied arena. // reserve_layout: declare the slots[] region on the arena (must precede commit) // init_data_from_layout: initialize sequence counters and queue metadata @@ -404,10 +404,10 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity); // arena.region_ptr(slots_off) only to address the slot array for writes; // does NOT store the pointer in `queue->slots`. Call // `ready_queue_wire_arena_pointers` afterwards to set the field itself. -bool ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity); +bool ready_queue_init_data_from_layout(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity); // Stores queue->slots = arena.region_ptr(slots_off). Idempotent. -void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off); -void ready_queue_destroy(PTO2ReadyQueue *queue); +void ready_queue_wire_arena_pointers(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off); +void ready_queue_destroy(ChipReadyQueue *queue); /** * Statistics returned by mixed-task completion processing @@ -528,18 +528,18 @@ struct SchedulerState { std::atomic publication_ack_mask; // Ready queues remain global (scheduling is ring-agnostic) - PTO2ReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue ready_queues[NUM_RESOURCE_SHAPES]; // Ready sync_start queues, one per shape. A ready sync_start cohort parks here // instead of ready_queues[] so the dispatch loop can drain it as a strict Tier-0 // (sync_start > MIX > C/V) before any regular ready task takes a core, while // reusing the same per-shape dispatch_shape machinery (fits-local inline vs // stop-the-world drain, per-core MIX placement, head-start spacing). - PTO2ReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue ready_sync_queues[NUM_RESOURCE_SHAPES]; // Dependency-only tasks (active_mask is empty, shape == DUMMY). Drained by // the dispatch loop and completed inline -- never goes to AICore. - PTO2ReadyQueue dummy_ready_queue; + ChipReadyQueue dummy_ready_queue; alignas(64) AsyncWaitList async_wait_list; @@ -766,11 +766,11 @@ struct SchedulerState { uint32_t token{0}; }; EarlyDispatchDoorbell early_dispatch_doorbell_table[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; - PTO2ReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; + ChipReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; // Shape-agnostic sync_start candidates require an all-or-nothing owner // decision. The owner stages locally when one tracker can hold the complete // cohort and falls back to the generation-tagged global drain otherwise. - PTO2ReadyQueue early_sync_start_queue; + ChipReadyQueue early_sync_start_queue; static inline void ring_one_doorbell(uint64_t reg_addr, uint32_t token) { volatile uint64_t *dmb = reinterpret_cast(get_reg_ptr(reg_addr, RegId::DATA_MAIN_BASE)); @@ -1107,13 +1107,13 @@ struct SchedulerState { } #endif - int get_ready_tasks_batch(PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { + int get_ready_tasks_batch(ChipReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count) { return queues[static_cast(shape)].pop_batch(out, max_count); } #if SIMPLER_SCHED_PROFILING int get_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, + ChipReadyQueue *queues, ResourceShape shape, ChipTaskSlotState **out, int max_count, uint64_t &atomic_count, uint64_t &wait_cycle ) { return queues[static_cast(shape)].pop_batch(out, max_count, atomic_count, wait_cycle); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 496a11c920..7be805decb 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -247,7 +247,7 @@ class SchedulerContext { } int pop_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + ChipReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ); void build_payload( @@ -300,7 +300,7 @@ class SchedulerContext { ); void dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, ChipReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ); @@ -347,7 +347,7 @@ class SchedulerContext { bool has_idle_in_other_threads(int32_t self_thread_idx, ResourceShape shape) const; // True if mix tasks remain in the global MIX ready queue. Approximate — - // PTO2ReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue + // ChipReadyQueue::size() (see scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index a475c6b99e..77863b4cd0 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -83,7 +83,7 @@ bool SchedulerContext::has_idle_in_other_threads(int32_t self_thread_idx, Resour } int SchedulerContext::pop_ready_tasks_batch( - PTO2ReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count + ChipReadyQueue *queues, ResourceShape shape, int32_t thread_idx, ChipTaskSlotState **out, int max_count ) { #if SIMPLER_DFX auto &chip_swimlane = sched_chip_swimlane_[thread_idx]; @@ -275,7 +275,7 @@ int SchedulerContext::prepare_block_for_dispatch( } void SchedulerContext::dispatch_shape( - int32_t thread_idx, PTO2ReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, + int32_t thread_idx, ChipReadyQueue *disp_queues, ResourceShape shape, CoreTracker::DispatchPhase phase, CoreTracker &tracker, bool &entered_drain, bool &made_progress, bool &try_pushed ) { #if SIMPLER_SCHED_PROFILING @@ -893,7 +893,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // // CHIP_SWIMLANE_NUM_QUEUE_SHAPES (3) matches NUM_RESOURCE_SHAPES: AIC/AIV/MIX. // - // **Hot-path cost discipline.** Shared depth (PTO2ReadyQueue::size) is two + // **Hot-path cost discipline.** Shared depth (ChipReadyQueue::size) is two // atomic relaxed loads against cache lines that all peer sched threads also // write to (enqueue_pos and dequeue_pos bounce on every push + every pop). // With both phases emitting per iter that's cross-core loads × thousands of @@ -910,7 +910,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ bool iter_shared_sampled = false; auto get_or_sample_shared = [&]() -> const int16_t * { if (!iter_shared_sampled) { - // Clamp to int16_t max before narrowing. PTO2_PROF_READYQUEUE_SIZE + // Clamp to int16_t max before narrowing. CHIP_PROF_READYQUEUE_SIZE // is in the low thousands today but could grow with platform // scaling — without clamp, sizes above 32767 wrap to negatives // and silently corrupt the snapshot. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 9738a6dea0..5c3687d237 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -52,13 +52,13 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Align the slots[] base to a full cache line so MPMC CAS traffic on the // first slot cannot false-share with whatever region sits in front of us // (e.g. orchestrator tensormap heads written by the orch thread). - return arena.reserve(capacity * sizeof(PTO2ReadyQueueSlot), PTO2_ALIGN_SIZE); + return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), PTO2_ALIGN_SIZE); } -bool ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity) { +bool ready_queue_init_data_from_layout(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity) { // Address the slots region for data writes without storing the pointer in // queue->slots — that field is set by ready_queue_wire_arena_pointers. - auto *slots_arena = static_cast(arena.region_ptr(slots_off)); + auto *slots_arena = static_cast(arena.region_ptr(slots_off)); queue->capacity = capacity; queue->mask = capacity - 1; queue->enqueue_pos.store(0, std::memory_order_relaxed); @@ -72,11 +72,11 @@ bool ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, DeviceArena &arena return true; } -void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off) { - queue->slots = static_cast(arena.region_ptr(slots_off)); +void ready_queue_wire_arena_pointers(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off) { + queue->slots = static_cast(arena.region_ptr(slots_off)); } -void ready_queue_destroy(PTO2ReadyQueue *queue) { +void ready_queue_destroy(ChipReadyQueue *queue) { // Arena owns the slots[] buffer; just forget the pointer. queue->slots = nullptr; } @@ -137,18 +137,18 @@ SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena, int32_t dep_p SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]) { SchedulerLayout layout{}; - layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; + layout.ready_queue_capacity = CHIP_READY_QUEUE_SIZE; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { layout.dep_pool_capacities[r] = dep_pool_capacities[r]; } for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_ready_sync_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } - layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); + layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); } diff --git a/tests/ut/cpp/a2a3/test_ready_queue.cpp b/tests/ut/cpp/a2a3/test_ready_queue.cpp index 159c075906..50663e7afe 100644 --- a/tests/ut/cpp/a2a3/test_ready_queue.cpp +++ b/tests/ut/cpp/a2a3/test_ready_queue.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2ReadyQueue from scheduler.h + * Unit tests for ChipReadyQueue from scheduler.h * * Tests the lock-free bounded MPMC queue (Vyukov design). * @@ -50,7 +50,7 @@ class ReadyQueueTest : public ::testing::Test { protected: static constexpr uint64_t CAPACITY = 16; // Power of 2 - PTO2ReadyQueue queue; + ChipReadyQueue queue; DeviceArena arena; void SetUp() override { @@ -244,7 +244,7 @@ TEST_F(ReadyQueueTest, SizeAccuracy) { class ReadyQueueBoundaryTest : public ::testing::Test { protected: static constexpr uint64_t QUEUE_CAP = 8; // Small for boundary testing - PTO2ReadyQueue queue{}; + ChipReadyQueue queue{}; ChipTaskSlotState dummy[8]{}; DeviceArena arena; @@ -345,7 +345,7 @@ struct MPMCConfig { class ReadyQueueMPMCTest : public ::testing::TestWithParam { protected: static constexpr uint64_t CAPACITY = 1024; - PTO2ReadyQueue queue; + ChipReadyQueue queue; DeviceArena arena; diff --git a/tests/ut/cpp/a5/test_ready_queue.cpp b/tests/ut/cpp/a5/test_ready_queue.cpp index 0860d1a853..5b470f5170 100644 --- a/tests/ut/cpp/a5/test_ready_queue.cpp +++ b/tests/ut/cpp/a5/test_ready_queue.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2ReadyQueue from scheduler.h + * Unit tests for ChipReadyQueue from scheduler.h * * Tests the lock-free bounded MPMC queue (Vyukov design). * @@ -50,7 +50,7 @@ class ReadyQueueTest : public ::testing::Test { protected: static constexpr uint64_t CAPACITY = 16; // Power of 2 - PTO2ReadyQueue queue; + ChipReadyQueue queue; DeviceArena arena; void SetUp() override { @@ -219,7 +219,7 @@ TEST_F(ReadyQueueTest, SizeAccuracy) { class ReadyQueueBoundaryTest : public ::testing::Test { protected: static constexpr uint64_t QUEUE_CAP = 8; // Small for boundary testing - PTO2ReadyQueue queue{}; + ChipReadyQueue queue{}; ChipTaskSlotState dummy[8]{}; DeviceArena arena; @@ -320,7 +320,7 @@ struct MPMCConfig { class ReadyQueueMPMCTest : public ::testing::TestWithParam { protected: static constexpr uint64_t CAPACITY = 1024; - PTO2ReadyQueue queue; + ChipReadyQueue queue; DeviceArena arena; diff --git a/tests/ut/cpp/common/test_hbg_graph_cache.cpp b/tests/ut/cpp/common/test_hbg_graph_cache.cpp index dc81ed9be9..63c2ed7b7f 100644 --- a/tests/ut/cpp/common/test_hbg_graph_cache.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_cache.cpp @@ -638,7 +638,7 @@ TEST(GraphExecutionErrors, GraphReadyQueueOverflowIsReported) { SharedMemoryHeader header{}; SchedulerState scheduler{}; scheduler.sm_header = &header; - PTO2ReadyQueueSlot queue_slots[2]{}; + ChipReadyQueueSlot queue_slots[2]{}; queue_slots[0].sequence.store(0, std::memory_order_relaxed); queue_slots[1].sequence.store(1, std::memory_order_relaxed); scheduler.graph_ready_queue.slots = queue_slots; @@ -662,7 +662,7 @@ TEST(GraphExecutionErrors, GraphPrepareQueueOverflowIsReported) { SharedMemoryHeader header{}; SchedulerState scheduler{}; scheduler.sm_header = &header; - PTO2ReadyQueueSlot queue_slots[2]{}; + ChipReadyQueueSlot queue_slots[2]{}; queue_slots[0].sequence.store(0, std::memory_order_relaxed); queue_slots[1].sequence.store(1, std::memory_order_relaxed); scheduler.graph_prepare_queue.slots = queue_slots; diff --git a/tests/ut/cpp/common/test_hbg_ready_queue_seed.cpp b/tests/ut/cpp/common/test_hbg_ready_queue_seed.cpp index 1ed37d3302..8eccf7bab2 100644 --- a/tests/ut/cpp/common/test_hbg_ready_queue_seed.cpp +++ b/tests/ut/cpp/common/test_hbg_ready_queue_seed.cpp @@ -33,19 +33,19 @@ constexpr uint64_t CAPACITY = 8; class SlotsRegion { public: explicit SlotsRegion(uint8_t fill) : - storage_(CAPACITY * sizeof(PTO2ReadyQueueSlot)) { + storage_(CAPACITY * sizeof(ChipReadyQueueSlot)) { std::memset(storage_.data(), fill, storage_.size()); } - PTO2ReadyQueueSlot *slots() { return reinterpret_cast(storage_.data()); } + ChipReadyQueueSlot *slots() { return reinterpret_cast(storage_.data()); } private: std::vector storage_; }; -// PTO2ReadyQueue holds atomics and so is neither copyable nor movable: the caller +// ChipReadyQueue holds atomics and so is neither copyable nor movable: the caller // owns the storage and this only fills it, exactly as the arena path does. -void make_queue(PTO2ReadyQueue *queue, SlotsRegion ®ion) { +void make_queue(ChipReadyQueue *queue, SlotsRegion ®ion) { ready_queue_init_data_from_layout(queue, CAPACITY); queue->slots = region.slots(); } @@ -64,7 +64,7 @@ ChipTaskSlotState *fake_slot_state(size_t i) { // signal. One push then succeeds and the rest report full. TEST(HbgReadyQueueSeed, ZeroedSlotsReportFullAfterOnePush) { SlotsRegion region(0x00); - PTO2ReadyQueue queue{}; + ChipReadyQueue queue{}; make_queue(&queue, region); EXPECT_TRUE(queue.push(fake_slot_state(0))); @@ -74,7 +74,7 @@ TEST(HbgReadyQueueSeed, ZeroedSlotsReportFullAfterOnePush) { TEST(HbgReadyQueueSeed, SeededSlotsAcceptCapacityPushes) { SlotsRegion region(0xAA); - PTO2ReadyQueue queue{}; + ChipReadyQueue queue{}; make_queue(&queue, region); queue.seed_slots(); @@ -96,7 +96,7 @@ TEST(HbgReadyQueueSeed, SeededSlotsAcceptCapacityPushes) { // work across laps with no re-seed. TEST(HbgReadyQueueSeed, DrainedQueueWrapsWithoutReseeding) { SlotsRegion region(0xAA); - PTO2ReadyQueue queue{}; + ChipReadyQueue queue{}; make_queue(&queue, region); queue.seed_slots(); @@ -116,7 +116,7 @@ TEST(HbgReadyQueueSeed, DrainedQueueWrapsWithoutReseeding) { // mid-lap by an earlier run rather than assuming a drained predecessor. TEST(HbgReadyQueueSeed, ReseedRecoversAPartiallyUsedRegion) { SlotsRegion region(0xAA); - PTO2ReadyQueue queue{}; + ChipReadyQueue queue{}; make_queue(&queue, region); queue.seed_slots(); ASSERT_TRUE(queue.push(fake_slot_state(0))); From 1dde6df6c7933842403b9e8e5b694b3efe72e0c1 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:21:48 -0700 Subject: [PATCH 08/22] Refactor: name the chip runtime's task state by its context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PTO2TaskState` becomes `ChipTaskState`, with its three enumerators `CHIP_TASK_PENDING` / `CHIP_TASK_COMPLETED` / `CHIP_TASK_CONSUMED`. `TaskState` is not available: `src/common/hierarchical/types.h:203` declares an `enum class TaskState` for the host orchestrator, and that one is bound into Python — both `python/simpler/task_interface.py` and `tests/ut/py/test_task_interface.py` import the name. Rule 13 gives the bare name to the L3+ side; the enumerators keep the type's prefix so a reader cannot mistake a chip-side PENDING for the host's. The lifecycle flags in the same header are plain strips — `TaskLifecycleFlag`, `LIFECYCLE_FLAGS_NONE`, `READY_CLAIMED`, `COMPLETION_DONE`, `SUBTASK_DEFERRED`, `DISPATCH_PROPAGATED` — because nothing else in the tree, in the CANN headers or in pto-isa claims any of them. They move here rather than in a commit of their own: they are the other half of the same 64-byte slot's state word. 362 occurrences across 40 files. Co-Authored-By: Claude Opus 5 (1M context) --- docs/dfx/dep-gen.md | 2 +- .../orchestrator_core/orchestrator.cpp | 4 +- .../orchestrator_core/runtime_core.cpp | 2 +- .../host_build_graph/runtime/runtime_types.h | 12 +- .../runtime/scheduler/scheduler.h | 2 +- .../runtime/scheduler/scheduler_cold_path.cpp | 4 +- .../docs/RUNTIME_LOGIC.md | 2 +- .../docs/SCALAR_DATA_ACCESS.md | 2 +- .../runtime/orchestrator.cpp | 10 +- .../runtime/runtime_core.cpp | 2 +- .../runtime/runtime_types.h | 43 +++-- .../runtime/scheduler/scheduler.h | 18 +- .../runtime/scheduler/scheduler_cold_path.cpp | 8 +- .../orchestrator_core/orchestrator.cpp | 4 +- .../orchestrator_core/runtime_core.cpp | 2 +- .../host_build_graph/runtime/runtime_types.h | 12 +- .../runtime/scheduler/scheduler.h | 2 +- .../runtime/scheduler/scheduler_cold_path.cpp | 4 +- .../docs/RUNTIME_LOGIC.md | 4 +- .../docs/SCALAR_DATA_ACCESS.md | 2 +- .../runtime/orchestrator.cpp | 8 +- .../runtime/runtime_core.cpp | 2 +- .../runtime/runtime_types.h | 43 +++-- .../runtime/scheduler/scheduler.h | 18 +- .../runtime/scheduler/scheduler_cold_path.cpp | 8 +- .../host_build_graph/graph_execution.cpp | 2 +- tests/ut/cpp/a2a3/test_graph_activation.cpp | 16 +- tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp | 6 +- tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp | 4 +- tests/ut/cpp/a2a3/test_scheduler_state.cpp | 48 +++--- tests/ut/cpp/a2a3/test_task_state.cpp | 20 +-- tests/ut/cpp/a2a3/test_wiring.cpp | 156 +++++++++--------- tests/ut/cpp/a5/test_graph_activation.cpp | 16 +- tests/ut/cpp/a5/test_hbg_submit_poison.cpp | 6 +- tests/ut/cpp/a5/test_orchestrator_fanin.cpp | 4 +- tests/ut/cpp/a5/test_scheduler_state.cpp | 54 +++--- tests/ut/cpp/a5/test_task_state.cpp | 20 +-- tests/ut/cpp/a5/test_wiring.cpp | 108 ++++++------ tests/ut/cpp/common/test_hbg_graph_cache.cpp | 2 +- .../common/test_scope_deadlock_detection.cpp | 2 +- 40 files changed, 341 insertions(+), 343 deletions(-) diff --git a/docs/dfx/dep-gen.md b/docs/dfx/dep-gen.md index 76a5a8e990..c3c095b278 100644 --- a/docs/dfx/dep-gen.md +++ b/docs/dfx/dep-gen.md @@ -11,7 +11,7 @@ the record entirely. The device hot path no longer carries fanout; When it existed, each producer task carried its own `ChipSwimlaneAicpuTaskRecord.fanout[]`, populated by the AICPU scheduler at the moment it wired a downstream consumer. If a producer had already finished and -transitioned to `PTO2_TASK_COMPLETED` by the time a later submit wanted to +transitioned to `CHIP_TASK_COMPLETED` by the time a later submit wanted to register a dependency on it, the consumer's edge had nowhere to go — the record was sealed, the slot was closed, and the edge was silently dropped. This was not a bug in fanout itself; fanout is "successors known at runtime", diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index dbb7a56567..e5609a1825 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1319,7 +1319,7 @@ static bool prepare_task( // task_state is set to PENDING here as the orchestrator populates the slot // (host_build_graph does not recycle slots at runtime, so there is no // post-CONSUMED reset path). - out->slot_state->task_state.store(PTO2_TASK_PENDING, std::memory_order_relaxed); + out->slot_state->task_state.store(CHIP_TASK_PENDING, std::memory_order_relaxed); out->slot_state->total_required_subtasks = static_cast(total_required_subtasks); out->slot_state->logical_block_num = block_num; out->slot_state->active_mask = active_mask; @@ -1820,7 +1820,7 @@ bool graph_submit_outer( ); orch->tensor_pool_cursor += tensor_slots; orch->scalar_pool_cursor += scalar_span; - slot.task_state.store(PTO2_TASK_PENDING, std::memory_order_relaxed); + slot.task_state.store(CHIP_TASK_PENDING, std::memory_order_relaxed); slot.last_consumer_local_id = static_cast(task_id.local()); slot.active_mask = ActiveMask{}; slot.task_attrs = TaskAttrs{}; diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 0c934b1325..8653e6e327 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -216,7 +216,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo int32_t local_id = static_cast(slot.task->task_id.local()); uint64_t t0 = get_sys_cnt_aicpu(); int32_t spin_count = 0; - while (slot.task_state.load(std::memory_order_acquire) < PTO2_TASK_COMPLETED) { + while (slot.task_state.load(std::memory_order_acquire) < CHIP_TASK_COMPLETED) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { // A fatal latched elsewhere (e.g. the scheduler-side wiring diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index b6e7f21356..5d74582380 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -183,10 +183,10 @@ constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s * completed_watermark instead. */ typedef enum { - PTO2_TASK_PENDING = 0, // Submitted; awaiting fanin, queued, or dispatched - PTO2_TASK_COMPLETED = 1, // Execution finished, output may still be in use - PTO2_TASK_CONSUMED = 2 // Unused: host_build_graph never advances past COMPLETED -} PTO2TaskState; + CHIP_TASK_PENDING = 0, // Submitted; awaiting fanin, queued, or dispatched + CHIP_TASK_COMPLETED = 1, // Execution finished, output may still be in use + CHIP_TASK_CONSUMED = 2 // Unused: host_build_graph never advances past COMPLETED +} ChipTaskState; /** * Result of a unified task allocation. @@ -553,7 +553,7 @@ struct alignas(64) ChipTaskSlotState { // Host-visible completion mirror. PENDING at submit; COMPLETED at // on_mixed_task_complete. Read by the host completion-wait / deadlock // detector / cold-path dump; the device readiness path uses completion_flags. - std::atomic task_state; + std::atomic task_state; // --- Per-slot constant, re-bound by orch::prepare_task each submit --- // Self-relative, so the SM image needs no pointer fix-up on its way to the @@ -626,7 +626,7 @@ struct alignas(64) ChipTaskSlotState { // (completion_flags[slot]) is published by the scheduler's // on_mixed_task_complete; this store makes the same fact visible to the // host completion-wait / deadlock detector. - void mark_completed() { task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); } + void mark_completed() { task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); } void mark_any_subtask_deferred() { any_subtask_deferred.store(true, std::memory_order_release); } diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h index e8c39a9fb2..26569d83a7 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -902,7 +902,7 @@ struct SchedulerState { for (uint32_t edge = begin; edge < end; ++edge) { const uint16_t producer_index = execution.fanin_indices[edge]; const ChipTaskSlotState &producer = execution.node_at(producer_index).slot; - if (producer.task_state.load(std::memory_order_acquire) != PTO2_TASK_COMPLETED) { + if (producer.task_state.load(std::memory_order_acquire) != CHIP_TASK_COMPLETED) { return static_cast(producer_index); } } diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index 5e97bf3092..c08161b4dc 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -236,7 +236,7 @@ void SchedulerContext::log_stall_diagnostics( submitted_in_ring += ring_task_count; for (int32_t si = 0; si < ring_task_count; si++) { ChipTaskSlotState &slot_state = ring.get_slot_state_by_task_id(si); - PTO2TaskState st = slot_state.task_state.load(std::memory_order_relaxed); + ChipTaskState st = slot_state.task_state.load(std::memory_order_relaxed); // Polling: no fanin_refcount. Recompute met/total from the inline // fanin ids vs the ring completion_flags (rc = satisfied producers, // fi = raw producer count) so the stall dump still shows readiness. @@ -252,7 +252,7 @@ void SchedulerContext::log_stall_diagnostics( int32_t kid_aiv0 = slot_state.task->kernel_id[1]; int32_t kid_aiv1 = slot_state.task->kernel_id[2]; int64_t task_id = static_cast(slot_state.task->task_id.raw); - if (st >= PTO2_TASK_COMPLETED) continue; + if (st >= CHIP_TASK_COMPLETED) continue; // task_state has no intermediate ready/running value — it // stays PENDING until the worker stores COMPLETED. Classify // by the ground truth instead: a slot is RUNNING iff some diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index dde9c1e577..4176c4ec9e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -403,7 +403,7 @@ When `OrchestratorState::submit_task` processes parameters: └──────────────────── slot recycled ◄───────────────────────────┘ ``` -In the scheduler's `task_state[]` array (`std::atomic`): +In the scheduler's `task_state[]` array (`std::atomic`): - **0 (PENDING)**: slot is allocated and remains PENDING through "waiting on producers", "queued in ready queue", and "dispatched to a worker"; ready vs diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md index 44b398047f..046dd13447 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md @@ -33,7 +33,7 @@ addr null-check → TensorMap lookup → spin-wait producer COMPLETED → comput - **addr null-check**: `buffer.addr == 0` means unallocated — log error, return 0 - **TensorMap lookup**: find producer task by `buffer.addr` -- **spin-wait**: wait until producer `task_state >= PTO2_TASK_COMPLETED` +- **spin-wait**: wait until producer `task_state >= CHIP_TASK_COMPLETED` - **No producer** (lookup callback never fires): skip waiting, read immediately ### 3.2 set_tensor_data Flow diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 89eab4b76f..f9cf95a24b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -349,9 +349,9 @@ static bool append_fanin_or_fail( // edge so the stronger of several discovery reasons (creator vs modifier vs // explicit) always wins, independent of the order they are reached in. prod_state->lock_fanout(); - PTO2TaskState pstate = prod_state->task_state.load(std::memory_order_acquire); + ChipTaskState pstate = prod_state->task_state.load(std::memory_order_acquire); bool gone = prod_state->task == nullptr || prod_state->task->task_id.local() != producer_task_id.local() || - pstate == PTO2_TASK_CONSUMED; + pstate == CHIP_TASK_CONSUMED; bool already_seen = !gone && fanin_builder->mark_seen(prod_ring, prod_slot); bool claim = !gone && !already_seen; int32_t fanout_now = -1; @@ -419,7 +419,7 @@ static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) { // dispatch, so it is treated as satisfied here. return fanin_builder.for_each([](ChipTaskSlotState *producer, DepFlags flags) -> bool { if (!dep_has_wait(flags)) return true; - return producer != nullptr && producer->task_state.load(std::memory_order_acquire) >= PTO2_TASK_COMPLETED; + return producer != nullptr && producer->task_state.load(std::memory_order_acquire) >= CHIP_TASK_COMPLETED; }); } @@ -461,7 +461,7 @@ void OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t w if (!early_disqualified && !producer->task_attrs.allow_early_resolve()) { early_disqualified = true; } - if (pstate >= PTO2_TASK_COMPLETED) { + if (pstate >= CHIP_TASK_COMPLETED) { completed_fanin++; } else { producer->fanout_head = rss.dep_pool.prepend(producer->fanout_head, &slot_state); @@ -651,7 +651,7 @@ static bool prepare_task( // ring_id // task_state left as CONSUMED by eager reset (safe for stale wait_for_tensor // observers); set to PENDING here when orchestrator actually reuses the slot. - out->slot_state->task_state.store(PTO2_TASK_PENDING, std::memory_order_relaxed); + out->slot_state->task_state.store(CHIP_TASK_PENDING, std::memory_order_relaxed); int16_t block_num = args.launch_spec.block_num(); out->slot_state->total_required_subtasks = static_cast(block_num * __builtin_popcount(active_mask.core_mask())); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 83f3671398..4ad31ea0fe 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -117,7 +117,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo int32_t local_id = static_cast(slot.task->task_id.local()); uint64_t t0 = get_sys_cnt_aicpu(); int32_t spin_count = 0; - while (slot.task_state.load(std::memory_order_acquire) < PTO2_TASK_COMPLETED) { + while (slot.task_state.load(std::memory_order_acquire) < CHIP_TASK_COMPLETED) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { // A fatal latched elsewhere breaks this wait; cold path only. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 18231d70ad..7fb88fde8d 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -135,10 +135,10 @@ constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s * COMPLETED->CONSUMED: fanout_refcount == fanout_count && state == COMPLETED */ typedef enum { - PTO2_TASK_PENDING = 0, // Submitted; awaiting fanin, queued, or dispatched - PTO2_TASK_COMPLETED = 1, // Execution finished, output may still be in use - PTO2_TASK_CONSUMED = 2 // Output fully consumed, buffers can be released -} PTO2TaskState; + CHIP_TASK_PENDING = 0, // Submitted; awaiting fanin, queued, or dispatched + CHIP_TASK_COMPLETED = 1, // Execution finished, output may still be in use + CHIP_TASK_CONSUMED = 2 // Output fully consumed, buffers can be released +} ChipTaskState; /** * Result of a unified task allocation. @@ -458,18 +458,18 @@ static_assert( // task pinned by any open scope on that ring. static constexpr uint32_t PTO2_FANOUT_SCOPE_BIT = 0x80000000u; -enum PTO2TaskLifecycleFlag : uint8_t { - PTO2_LIFECYCLE_FLAGS_NONE = 0, - PTO2_READY_CLAIMED = 1U << 0, - PTO2_COMPLETION_DONE = 1U << 1, +enum TaskLifecycleFlag : uint8_t { + LIFECYCLE_FLAGS_NONE = 0, + READY_CLAIMED = 1U << 0, + COMPLETION_DONE = 1U << 1, // Deferred-completion discriminator packed into lifecycle_flags so the // slot remains 64B. Call sites use mark_any_subtask_deferred() and // has_any_subtask_deferred(). - PTO2_SUBTASK_DEFERRED = 1U << 2, - PTO2_DISPATCH_PROPAGATED = 1U << 3, + SUBTASK_DEFERRED = 1U << 2, + DISPATCH_PROPAGATED = 1U << 3, }; -static_assert((PTO2_DISPATCH_PROPAGATED & (PTO2_READY_CLAIMED | PTO2_COMPLETION_DONE | PTO2_SUBTASK_DEFERRED)) == 0); +static_assert((DISPATCH_PROPAGATED & (READY_CLAIMED | COMPLETION_DONE | SUBTASK_DEFERRED)) == 0); struct alignas(64) ChipTaskSlotState { // Fanout lock + list (accessed together under lock in on_task_complete) @@ -479,7 +479,7 @@ struct alignas(64) ChipTaskSlotState { PTO2DepListEntry *fanout_head; // Pointer to first fanout entry (nullptr = empty) // Task state (completion, consumed check, ready check) - std::atomic task_state; // PENDING/COMPLETED/CONSUMED + std::atomic task_state; // PENDING/COMPLETED/CONSUMED // Fanin (accessed together in release_fanin_and_check_ready) std::atomic fanin_refcount; // Dynamic: counts completed producers @@ -509,7 +509,7 @@ struct alignas(64) ChipTaskSlotState { TaskAttrs task_attrs{}; // Concurrent lifecycle updates preserve unrelated bits. Slot reuse clears // the byte only after the previous task lifetime is quiescent. - std::atomic lifecycle_flags{PTO2_LIFECYCLE_FLAGS_NONE}; + std::atomic lifecycle_flags{LIFECYCLE_FLAGS_NONE}; int32_t dep_pool_mark{0}; // Dep pool top after Orch-side wiring std::atomic completed_subtasks{0}; // Each core completion increments by 1 @@ -558,23 +558,22 @@ struct alignas(64) ChipTaskSlotState { // Lock-free callers use this as a fast-path hint. A false result is // rechecked by try_mark_dispatch_propagated() while holding fanout_lock. bool has_dispatch_propagated() const { - return (lifecycle_flags.load(std::memory_order_acquire) & PTO2_DISPATCH_PROPAGATED) != 0; + return (lifecycle_flags.load(std::memory_order_acquire) & DISPATCH_PROPAGATED) != 0; } // The propagation owner holds fanout_lock through its fanout snapshot, so // wiring can classify late edges exactly once. bool try_mark_dispatch_propagated() { - return (lifecycle_flags.fetch_or(PTO2_DISPATCH_PROPAGATED, std::memory_order_acq_rel) & - PTO2_DISPATCH_PROPAGATED) == 0; + return (lifecycle_flags.fetch_or(DISPATCH_PROPAGATED, std::memory_order_acq_rel) & DISPATCH_PROPAGATED) == 0; } void mark_completed() { - task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); - lifecycle_flags.fetch_or(PTO2_COMPLETION_DONE, std::memory_order_release); + task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); + lifecycle_flags.fetch_or(COMPLETION_DONE, std::memory_order_release); } bool is_completion_flag_set() const { - return (lifecycle_flags.load(std::memory_order_acquire) & PTO2_COMPLETION_DONE) != 0; + return (lifecycle_flags.load(std::memory_order_acquire) & COMPLETION_DONE) != 0; } // Set by any subtask FIN that pushed deferred-completion CONDITIONs to the @@ -582,10 +581,10 @@ struct alignas(64) ChipTaskSlotState { // needs MPSC-deferred completion or can complete inline on this thread. The // release write is sequenced before on_subtask_complete's acq_rel fetch_add; // the acquire read by the last subtask observes all earlier writes. - void mark_any_subtask_deferred() { lifecycle_flags.fetch_or(PTO2_SUBTASK_DEFERRED, std::memory_order_release); } + void mark_any_subtask_deferred() { lifecycle_flags.fetch_or(SUBTASK_DEFERRED, std::memory_order_release); } bool has_any_subtask_deferred() const { - return (lifecycle_flags.load(std::memory_order_acquire) & PTO2_SUBTASK_DEFERRED) != 0; + return (lifecycle_flags.load(std::memory_order_acquire) & SUBTASK_DEFERRED) != 0; } /** @@ -607,7 +606,7 @@ struct alignas(64) ChipTaskSlotState { fanout_refcount.store(0, std::memory_order_relaxed); completed_subtasks.store(0, std::memory_order_relaxed); next_block_idx.store(0, std::memory_order_relaxed); - lifecycle_flags.store(PTO2_LIFECYCLE_FLAGS_NONE, std::memory_order_relaxed); + lifecycle_flags.store(LIFECYCLE_FLAGS_NONE, std::memory_order_relaxed); // Note: active_mask and task_attrs are per-submit-constant fields // rewritten in prepare_task on every reuse, so they are not reset here. // Payload early-dispatch fields are initialized by TaskPayload::init diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 03601f08a3..051ee9f793 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -490,7 +490,7 @@ struct SchedulerState { while (last_task_alive < current_task_index) { ChipTaskSlotState &slot_state = ring->get_slot_state_by_task_id(last_task_alive); - if (slot_state.task_state.load(std::memory_order_acquire) != PTO2_TASK_CONSUMED) { + if (slot_state.task_state.load(std::memory_order_acquire) != CHIP_TASK_CONSUMED) { break; } last_task_alive++; @@ -595,8 +595,8 @@ struct SchedulerState { bool try_claim_ready_once(ChipTaskSlotState &slot_state) { uint8_t flags = slot_state.lifecycle_flags.load(std::memory_order_acquire); for (;;) { - if ((flags & PTO2_READY_CLAIMED) != 0) return false; - uint8_t desired_flags = flags | PTO2_READY_CLAIMED; + if ((flags & READY_CLAIMED) != 0) return false; + uint8_t desired_flags = flags | READY_CLAIMED; if (slot_state.lifecycle_flags.compare_exchange_weak( flags, desired_flags, std::memory_order_acq_rel, std::memory_order_acquire )) { @@ -617,9 +617,9 @@ struct SchedulerState { bool became_consumed = false; slot_state.lock_fanout(); if (slot_state.fanout_refcount.load(std::memory_order_acquire) == slot_state.fanout_count) { - PTO2TaskState expected = PTO2_TASK_COMPLETED; + ChipTaskState expected = CHIP_TASK_COMPLETED; became_consumed = slot_state.task_state.compare_exchange_strong( - expected, PTO2_TASK_CONSUMED, std::memory_order_acq_rel, std::memory_order_acquire + expected, CHIP_TASK_CONSUMED, std::memory_order_acq_rel, std::memory_order_acquire ); } slot_state.unlock_fanout(); @@ -653,9 +653,9 @@ struct SchedulerState { uint32_t rc = slot_state.fanout_refcount.load(std::memory_order_acquire); atomic_count += 1; // fanout_refcount.load (fanout_count is a plain read under lock) if (rc == fc) { - PTO2TaskState expected = PTO2_TASK_COMPLETED; + ChipTaskState expected = CHIP_TASK_COMPLETED; became_consumed = slot_state.task_state.compare_exchange_strong( - expected, PTO2_TASK_CONSUMED, std::memory_order_acq_rel, std::memory_order_acquire + expected, CHIP_TASK_CONSUMED, std::memory_order_acq_rel, std::memory_order_acquire ); atomic_count += 1; // CAS } @@ -1118,8 +1118,8 @@ struct SchedulerState { uint8_t flags = slot_state.lifecycle_flags.load(std::memory_order_acquire); atomic_count += 1; // lifecycle_flags load for (;;) { - if ((flags & PTO2_READY_CLAIMED) != 0) return false; - uint8_t desired_flags = flags | PTO2_READY_CLAIMED; + if ((flags & READY_CLAIMED) != 0) return false; + uint8_t desired_flags = flags | READY_CLAIMED; if (slot_state.lifecycle_flags.compare_exchange_weak( flags, desired_flags, std::memory_order_acq_rel, std::memory_order_acquire )) { diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 8a58633127..697fc0580c 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -238,14 +238,14 @@ void SchedulerContext::log_stall_diagnostics( int32_t ring_task_start = ring.fc.last_task_alive.load(std::memory_order_relaxed); for (int32_t si = ring_task_start; si < ring_task_count; si++) { ChipTaskSlotState &slot_state = ring.get_slot_state_by_task_id(si); - PTO2TaskState st = slot_state.task_state.load(std::memory_order_relaxed); + ChipTaskState st = slot_state.task_state.load(std::memory_order_relaxed); int32_t rc = slot_state.fanin_refcount.load(std::memory_order_relaxed); int32_t fi = slot_state.fanin_count; int32_t kid_aic = slot_state.task->kernel_id[0]; int32_t kid_aiv0 = slot_state.task->kernel_id[1]; int32_t kid_aiv1 = slot_state.task->kernel_id[2]; int64_t task_id = static_cast(slot_state.task->task_id.raw); - if (st >= PTO2_TASK_COMPLETED) continue; + if (st >= CHIP_TASK_COMPLETED) continue; // task_state has no intermediate ready/running value — it // stays PENDING until the worker stores COMPLETED. Classify // by the ground truth instead: a slot is RUNNING iff some @@ -374,8 +374,8 @@ SchedulerContext::StallClassification SchedulerContext::classify_stall_reason() int32_t ring_task_start = ring.fc.last_task_alive.load(std::memory_order_relaxed); for (int32_t si = ring_task_start; si < ring_task_count; si++) { ChipTaskSlotState &slot_state = ring.get_slot_state_by_task_id(si); - PTO2TaskState st = slot_state.task_state.load(std::memory_order_relaxed); - if (st >= PTO2_TASK_COMPLETED) continue; + ChipTaskState st = slot_state.task_state.load(std::memory_order_relaxed); + if (st >= CHIP_TASK_COMPLETED) continue; // Same ground truth as log_stall_diagnostics: task_state stays PENDING // until COMPLETED, so RUNNING is read from core ownership, not the slot. int32_t run_core = -1; diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index dbb7a56567..e5609a1825 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1319,7 +1319,7 @@ static bool prepare_task( // task_state is set to PENDING here as the orchestrator populates the slot // (host_build_graph does not recycle slots at runtime, so there is no // post-CONSUMED reset path). - out->slot_state->task_state.store(PTO2_TASK_PENDING, std::memory_order_relaxed); + out->slot_state->task_state.store(CHIP_TASK_PENDING, std::memory_order_relaxed); out->slot_state->total_required_subtasks = static_cast(total_required_subtasks); out->slot_state->logical_block_num = block_num; out->slot_state->active_mask = active_mask; @@ -1820,7 +1820,7 @@ bool graph_submit_outer( ); orch->tensor_pool_cursor += tensor_slots; orch->scalar_pool_cursor += scalar_span; - slot.task_state.store(PTO2_TASK_PENDING, std::memory_order_relaxed); + slot.task_state.store(CHIP_TASK_PENDING, std::memory_order_relaxed); slot.last_consumer_local_id = static_cast(task_id.local()); slot.active_mask = ActiveMask{}; slot.task_attrs = TaskAttrs{}; diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 0c934b1325..8653e6e327 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -216,7 +216,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo int32_t local_id = static_cast(slot.task->task_id.local()); uint64_t t0 = get_sys_cnt_aicpu(); int32_t spin_count = 0; - while (slot.task_state.load(std::memory_order_acquire) < PTO2_TASK_COMPLETED) { + while (slot.task_state.load(std::memory_order_acquire) < CHIP_TASK_COMPLETED) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { // A fatal latched elsewhere (e.g. the scheduler-side wiring diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 5d6b3487fe..cffb8ea6c0 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -184,10 +184,10 @@ constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s * completed_watermark instead. */ typedef enum { - PTO2_TASK_PENDING = 0, // Submitted; awaiting fanin, queued, or dispatched - PTO2_TASK_COMPLETED = 1, // Execution finished, output may still be in use - PTO2_TASK_CONSUMED = 2 // Unused: host_build_graph never advances past COMPLETED -} PTO2TaskState; + CHIP_TASK_PENDING = 0, // Submitted; awaiting fanin, queued, or dispatched + CHIP_TASK_COMPLETED = 1, // Execution finished, output may still be in use + CHIP_TASK_CONSUMED = 2 // Unused: host_build_graph never advances past COMPLETED +} ChipTaskState; /** * Result of a unified task allocation. @@ -554,7 +554,7 @@ struct alignas(64) ChipTaskSlotState { // Host-visible completion mirror. PENDING at submit; COMPLETED at // on_mixed_task_complete. Read by the host completion-wait / deadlock // detector / cold-path dump; the device readiness path uses completion_flags. - std::atomic task_state; + std::atomic task_state; // --- Per-slot constant, re-bound by orch::prepare_task each submit --- // Self-relative, so the SM image needs no pointer fix-up on its way to the @@ -624,7 +624,7 @@ struct alignas(64) ChipTaskSlotState { // (completion_flags[slot]) is published by the scheduler's // on_mixed_task_complete; this store makes the same fact visible to the // host completion-wait / deadlock detector. - void mark_completed() { task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); } + void mark_completed() { task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); } void mark_any_subtask_deferred() { any_subtask_deferred.store(true, std::memory_order_release); } diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h index e8c39a9fb2..26569d83a7 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -902,7 +902,7 @@ struct SchedulerState { for (uint32_t edge = begin; edge < end; ++edge) { const uint16_t producer_index = execution.fanin_indices[edge]; const ChipTaskSlotState &producer = execution.node_at(producer_index).slot; - if (producer.task_state.load(std::memory_order_acquire) != PTO2_TASK_COMPLETED) { + if (producer.task_state.load(std::memory_order_acquire) != CHIP_TASK_COMPLETED) { return static_cast(producer_index); } } diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index 5e97bf3092..c08161b4dc 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -236,7 +236,7 @@ void SchedulerContext::log_stall_diagnostics( submitted_in_ring += ring_task_count; for (int32_t si = 0; si < ring_task_count; si++) { ChipTaskSlotState &slot_state = ring.get_slot_state_by_task_id(si); - PTO2TaskState st = slot_state.task_state.load(std::memory_order_relaxed); + ChipTaskState st = slot_state.task_state.load(std::memory_order_relaxed); // Polling: no fanin_refcount. Recompute met/total from the inline // fanin ids vs the ring completion_flags (rc = satisfied producers, // fi = raw producer count) so the stall dump still shows readiness. @@ -252,7 +252,7 @@ void SchedulerContext::log_stall_diagnostics( int32_t kid_aiv0 = slot_state.task->kernel_id[1]; int32_t kid_aiv1 = slot_state.task->kernel_id[2]; int64_t task_id = static_cast(slot_state.task->task_id.raw); - if (st >= PTO2_TASK_COMPLETED) continue; + if (st >= CHIP_TASK_COMPLETED) continue; // task_state has no intermediate ready/running value — it // stays PENDING until the worker stores COMPLETED. Classify // by the ground truth instead: a slot is RUNNING iff some diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 0afc3ee6fe..f4f3eb4a7a 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -413,7 +413,7 @@ When `OrchestratorState::submit_task` processes parameters: └──────────────────── slot recycled ◄───────────────────────────┘ ``` -In the scheduler's `task_state[]` array (`std::atomic`): +In the scheduler's `task_state[]` array (`std::atomic`): - **0 (PENDING)**: slot is allocated and remains PENDING through "waiting on producers", "queued in ready queue", and "dispatched to a worker"; ready vs @@ -715,7 +715,7 @@ Producer-side `propagate_dispatch_fanin` no-ops until the producer is **fully pu prepare/publish makes payload + low-32 dispatch tokens visible. Early queue entries carry a `task_id_snapshot` generation tag (#1336) so recycled slots cannot revive stale candidates. -Wiring and propagation serialize under `fanout_lock` with `PTO2_DISPATCH_PROPAGATED` so +Wiring and propagation serialize under `fanout_lock` with `DISPATCH_PROPAGATED` so late-wired consumers still receive exactly one early-candidate contribution per eligible edge (#1405). diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md index 44b398047f..046dd13447 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md @@ -33,7 +33,7 @@ addr null-check → TensorMap lookup → spin-wait producer COMPLETED → comput - **addr null-check**: `buffer.addr == 0` means unallocated — log error, return 0 - **TensorMap lookup**: find producer task by `buffer.addr` -- **spin-wait**: wait until producer `task_state >= PTO2_TASK_COMPLETED` +- **spin-wait**: wait until producer `task_state >= CHIP_TASK_COMPLETED` - **No producer** (lookup callback never fires): skip waiting, read immediately ### 3.2 set_tensor_data Flow diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 4f1359ad28..c6be03176c 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -344,7 +344,7 @@ static bool append_fanin_or_fail( // explicit) always wins, independent of the order they are reached in. prod_state->lock_fanout(); bool gone = prod_state->task == nullptr || prod_state->task->task_id.local() != producer_task_id.local() || - prod_state->task_state.load(std::memory_order_acquire) == PTO2_TASK_CONSUMED; + prod_state->task_state.load(std::memory_order_acquire) == CHIP_TASK_CONSUMED; bool already_seen = !gone && fanin_builder->mark_seen(prod_ring, prod_slot); bool claim = !gone && !already_seen; int32_t fanout_now = -1; @@ -412,7 +412,7 @@ static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) { // dispatch, so it is treated as satisfied here. return fanin_builder.for_each([](ChipTaskSlotState *producer, DepFlags flags) -> bool { if (!dep_has_wait(flags)) return true; - return producer != nullptr && producer->task_state.load(std::memory_order_acquire) >= PTO2_TASK_COMPLETED; + return producer != nullptr && producer->task_state.load(std::memory_order_acquire) >= CHIP_TASK_COMPLETED; }); } @@ -455,7 +455,7 @@ void OrchestratorState::wire_fanin_task(ChipTaskSlotState &slot_state, int32_t w if (!early_disqualified && !producer->task_attrs.allow_early_resolve()) { early_disqualified = true; } - if (pstate >= PTO2_TASK_COMPLETED) { + if (pstate >= CHIP_TASK_COMPLETED) { completed_fanin++; } else { producer->fanout_head = rss.dep_pool.prepend(producer->fanout_head, &slot_state); @@ -639,7 +639,7 @@ static bool prepare_task( // ring_id // task_state left as CONSUMED by eager reset (safe for stale wait_for_tensor // observers); set to PENDING here when orchestrator actually reuses the slot. - out->slot_state->task_state.store(PTO2_TASK_PENDING, std::memory_order_relaxed); + out->slot_state->task_state.store(CHIP_TASK_PENDING, std::memory_order_relaxed); int16_t block_num = args.launch_spec.core_num(); out->slot_state->total_required_subtasks = static_cast(block_num * __builtin_popcount(active_mask.core_mask())); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 83f3671398..4ad31ea0fe 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -117,7 +117,7 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo int32_t local_id = static_cast(slot.task->task_id.local()); uint64_t t0 = get_sys_cnt_aicpu(); int32_t spin_count = 0; - while (slot.task_state.load(std::memory_order_acquire) < PTO2_TASK_COMPLETED) { + while (slot.task_state.load(std::memory_order_acquire) < CHIP_TASK_COMPLETED) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { // A fatal latched elsewhere breaks this wait; cold path only. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 6c6204c203..d07f6273c8 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -135,10 +135,10 @@ constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s * COMPLETED->CONSUMED: fanout_refcount == fanout_count && state == COMPLETED */ typedef enum { - PTO2_TASK_PENDING = 0, // Submitted; awaiting fanin, queued, or dispatched - PTO2_TASK_COMPLETED = 1, // Execution finished, output may still be in use - PTO2_TASK_CONSUMED = 2 // Output fully consumed, buffers can be released -} PTO2TaskState; + CHIP_TASK_PENDING = 0, // Submitted; awaiting fanin, queued, or dispatched + CHIP_TASK_COMPLETED = 1, // Execution finished, output may still be in use + CHIP_TASK_CONSUMED = 2 // Output fully consumed, buffers can be released +} ChipTaskState; /** * Result of a unified task allocation. @@ -458,18 +458,18 @@ static_assert( // task pinned by any open scope on that ring. static constexpr uint32_t PTO2_FANOUT_SCOPE_BIT = 0x80000000u; -enum PTO2TaskLifecycleFlag : uint8_t { - PTO2_LIFECYCLE_FLAGS_NONE = 0, - PTO2_READY_CLAIMED = 1U << 0, - PTO2_COMPLETION_DONE = 1U << 1, +enum TaskLifecycleFlag : uint8_t { + LIFECYCLE_FLAGS_NONE = 0, + READY_CLAIMED = 1U << 0, + COMPLETION_DONE = 1U << 1, // Deferred-completion discriminator packed into lifecycle_flags so the // slot remains 64B. Call sites use mark_any_subtask_deferred() and // has_any_subtask_deferred(). - PTO2_SUBTASK_DEFERRED = 1U << 2, - PTO2_DISPATCH_PROPAGATED = 1U << 3, + SUBTASK_DEFERRED = 1U << 2, + DISPATCH_PROPAGATED = 1U << 3, }; -static_assert((PTO2_DISPATCH_PROPAGATED & (PTO2_READY_CLAIMED | PTO2_COMPLETION_DONE | PTO2_SUBTASK_DEFERRED)) == 0); +static_assert((DISPATCH_PROPAGATED & (READY_CLAIMED | COMPLETION_DONE | SUBTASK_DEFERRED)) == 0); struct alignas(64) ChipTaskSlotState { // Fanout lock + list (accessed together under lock in on_task_complete) @@ -479,7 +479,7 @@ struct alignas(64) ChipTaskSlotState { PTO2DepListEntry *fanout_head; // Pointer to first fanout entry (nullptr = empty) // Task state (completion, consumed check, ready check) - std::atomic task_state; // PENDING/COMPLETED/CONSUMED + std::atomic task_state; // PENDING/COMPLETED/CONSUMED // Fanin (accessed together in release_fanin_and_check_ready) std::atomic fanin_refcount; // Dynamic: counts completed producers @@ -509,7 +509,7 @@ struct alignas(64) ChipTaskSlotState { TaskAttrs task_attrs{}; // Concurrent lifecycle updates preserve unrelated bits. Slot reuse clears // the byte only after the previous task lifetime is quiescent. - std::atomic lifecycle_flags{PTO2_LIFECYCLE_FLAGS_NONE}; + std::atomic lifecycle_flags{LIFECYCLE_FLAGS_NONE}; int32_t dep_pool_mark{0}; // Dep pool top after Orch-side wiring std::atomic completed_subtasks{0}; // Each core completion increments by 1 @@ -558,23 +558,22 @@ struct alignas(64) ChipTaskSlotState { // Lock-free callers use this as a fast-path hint. A false result is // rechecked by try_mark_dispatch_propagated() while holding fanout_lock. bool has_dispatch_propagated() const { - return (lifecycle_flags.load(std::memory_order_acquire) & PTO2_DISPATCH_PROPAGATED) != 0; + return (lifecycle_flags.load(std::memory_order_acquire) & DISPATCH_PROPAGATED) != 0; } // The propagation owner holds fanout_lock through its fanout snapshot, so // wiring can classify late edges exactly once. bool try_mark_dispatch_propagated() { - return (lifecycle_flags.fetch_or(PTO2_DISPATCH_PROPAGATED, std::memory_order_acq_rel) & - PTO2_DISPATCH_PROPAGATED) == 0; + return (lifecycle_flags.fetch_or(DISPATCH_PROPAGATED, std::memory_order_acq_rel) & DISPATCH_PROPAGATED) == 0; } void mark_completed() { - task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); - lifecycle_flags.fetch_or(PTO2_COMPLETION_DONE, std::memory_order_release); + task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); + lifecycle_flags.fetch_or(COMPLETION_DONE, std::memory_order_release); } bool is_completion_flag_set() const { - return (lifecycle_flags.load(std::memory_order_acquire) & PTO2_COMPLETION_DONE) != 0; + return (lifecycle_flags.load(std::memory_order_acquire) & COMPLETION_DONE) != 0; } // Set by any subtask FIN that pushed deferred-completion CONDITIONs to the @@ -582,10 +581,10 @@ struct alignas(64) ChipTaskSlotState { // needs MPSC-deferred completion or can complete inline on this thread. The // release write is sequenced before on_subtask_complete's acq_rel fetch_add; // the acquire read by the last subtask observes all earlier writes. - void mark_any_subtask_deferred() { lifecycle_flags.fetch_or(PTO2_SUBTASK_DEFERRED, std::memory_order_release); } + void mark_any_subtask_deferred() { lifecycle_flags.fetch_or(SUBTASK_DEFERRED, std::memory_order_release); } bool has_any_subtask_deferred() const { - return (lifecycle_flags.load(std::memory_order_acquire) & PTO2_SUBTASK_DEFERRED) != 0; + return (lifecycle_flags.load(std::memory_order_acquire) & SUBTASK_DEFERRED) != 0; } /** @@ -607,7 +606,7 @@ struct alignas(64) ChipTaskSlotState { fanout_refcount.store(0, std::memory_order_relaxed); completed_subtasks.store(0, std::memory_order_relaxed); next_block_idx.store(0, std::memory_order_relaxed); - lifecycle_flags.store(PTO2_LIFECYCLE_FLAGS_NONE, std::memory_order_relaxed); + lifecycle_flags.store(LIFECYCLE_FLAGS_NONE, std::memory_order_relaxed); // Note: active_mask and task_attrs are per-submit-constant fields // rewritten in prepare_task on every reuse, so they are not reset here. // Payload early-dispatch fields are initialized by TaskPayload::init diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index f91d6daed9..c902864f72 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -504,7 +504,7 @@ struct SchedulerState { while (last_task_alive < current_task_index) { ChipTaskSlotState &slot_state = ring->get_slot_state_by_task_id(last_task_alive); - if (slot_state.task_state.load(std::memory_order_acquire) != PTO2_TASK_CONSUMED) { + if (slot_state.task_state.load(std::memory_order_acquire) != CHIP_TASK_CONSUMED) { break; } last_task_alive++; @@ -640,8 +640,8 @@ struct SchedulerState { bool try_claim_ready_once(ChipTaskSlotState &slot_state) { uint8_t flags = slot_state.lifecycle_flags.load(std::memory_order_acquire); for (;;) { - if ((flags & PTO2_READY_CLAIMED) != 0) return false; - uint8_t desired_flags = flags | PTO2_READY_CLAIMED; + if ((flags & READY_CLAIMED) != 0) return false; + uint8_t desired_flags = flags | READY_CLAIMED; if (slot_state.lifecycle_flags.compare_exchange_weak( flags, desired_flags, std::memory_order_acq_rel, std::memory_order_acquire )) { @@ -662,9 +662,9 @@ struct SchedulerState { bool became_consumed = false; slot_state.lock_fanout(); if (slot_state.fanout_refcount.load(std::memory_order_acquire) == slot_state.fanout_count) { - PTO2TaskState expected = PTO2_TASK_COMPLETED; + ChipTaskState expected = CHIP_TASK_COMPLETED; became_consumed = slot_state.task_state.compare_exchange_strong( - expected, PTO2_TASK_CONSUMED, std::memory_order_acq_rel, std::memory_order_acquire + expected, CHIP_TASK_CONSUMED, std::memory_order_acq_rel, std::memory_order_acquire ); } slot_state.unlock_fanout(); @@ -698,9 +698,9 @@ struct SchedulerState { uint32_t rc = slot_state.fanout_refcount.load(std::memory_order_acquire); atomic_count += 1; // fanout_refcount.load (fanout_count is a plain read under lock) if (rc == fc) { - PTO2TaskState expected = PTO2_TASK_COMPLETED; + ChipTaskState expected = CHIP_TASK_COMPLETED; became_consumed = slot_state.task_state.compare_exchange_strong( - expected, PTO2_TASK_CONSUMED, std::memory_order_acq_rel, std::memory_order_acquire + expected, CHIP_TASK_CONSUMED, std::memory_order_acq_rel, std::memory_order_acquire ); atomic_count += 1; // CAS } @@ -1050,8 +1050,8 @@ struct SchedulerState { uint8_t flags = slot_state.lifecycle_flags.load(std::memory_order_acquire); atomic_count += 1; for (;;) { - if ((flags & PTO2_READY_CLAIMED) != 0) return false; - uint8_t desired_flags = flags | PTO2_READY_CLAIMED; + if ((flags & READY_CLAIMED) != 0) return false; + uint8_t desired_flags = flags | READY_CLAIMED; if (slot_state.lifecycle_flags.compare_exchange_weak( flags, desired_flags, std::memory_order_acq_rel, std::memory_order_acquire )) { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index e9303f4b71..ac68db51c3 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -244,14 +244,14 @@ void SchedulerContext::log_stall_diagnostics( int32_t ring_task_start = ring.fc.last_task_alive.load(std::memory_order_relaxed); for (int32_t si = ring_task_start; si < ring_task_count; si++) { ChipTaskSlotState &slot_state = ring.get_slot_state_by_task_id(si); - PTO2TaskState st = slot_state.task_state.load(std::memory_order_relaxed); + ChipTaskState st = slot_state.task_state.load(std::memory_order_relaxed); int32_t rc = slot_state.fanin_refcount.load(std::memory_order_relaxed); int32_t fi = slot_state.fanin_count; int32_t kid_aic = slot_state.task->kernel_id[0]; int32_t kid_aiv0 = slot_state.task->kernel_id[1]; int32_t kid_aiv1 = slot_state.task->kernel_id[2]; int64_t task_id = static_cast(slot_state.task->task_id.raw); - if (st >= PTO2_TASK_COMPLETED) continue; + if (st >= CHIP_TASK_COMPLETED) continue; // task_state has no intermediate ready/running value — it // stays PENDING until the worker stores COMPLETED. Classify // by the ground truth instead: a slot is RUNNING iff some @@ -380,8 +380,8 @@ SchedulerContext::StallClassification SchedulerContext::classify_stall_reason() int32_t ring_task_start = ring.fc.last_task_alive.load(std::memory_order_relaxed); for (int32_t si = ring_task_start; si < ring_task_count; si++) { ChipTaskSlotState &slot_state = ring.get_slot_state_by_task_id(si); - PTO2TaskState st = slot_state.task_state.load(std::memory_order_relaxed); - if (st >= PTO2_TASK_COMPLETED) continue; + ChipTaskState st = slot_state.task_state.load(std::memory_order_relaxed); + if (st >= CHIP_TASK_COMPLETED) continue; // Same ground truth as log_stall_diagnostics: task_state stays PENDING // until COMPLETED, so RUNNING is read from core ownership, not the slot. int32_t run_core = -1; diff --git a/src/common/host_build_graph/graph_execution.cpp b/src/common/host_build_graph/graph_execution.cpp index 9786afd73b..e0bd7a0731 100644 --- a/src/common/host_build_graph/graph_execution.cpp +++ b/src/common/host_build_graph/graph_execution.cpp @@ -450,7 +450,7 @@ GraphMaterializeResult graph_execution_materialize_slice( task.packed_buffer_end = reinterpret_cast(outer_base + node_offset + output_bytes); slot.reset_for_reuse(); - slot.task_state.store(PTO2_TASK_PENDING, std::memory_order_relaxed); + slot.task_state.store(CHIP_TASK_PENDING, std::memory_order_relaxed); slot.bind_buffers(&payload, &task); slot.active_mask = ActiveMask(source.active_mask); slot.task_attrs = TaskAttrs(source.task_attrs); diff --git a/tests/ut/cpp/a2a3/test_graph_activation.cpp b/tests/ut/cpp/a2a3/test_graph_activation.cpp index 9937218ad5..885c7420dc 100644 --- a/tests/ut/cpp/a2a3/test_graph_activation.cpp +++ b/tests/ut/cpp/a2a3/test_graph_activation.cpp @@ -58,7 +58,7 @@ class GraphActivationTest : public ::testing::Test { // One GraphExecution node whose slot is a routable single-block KERNEL/AIC // task in the given completion state, with its payload wired the way // materialization leaves it for the wake/route path. - static void init_graph_node(GraphNodeStorage &node, int32_t node_index, PTO2TaskState state) { + static void init_graph_node(GraphNodeStorage &node, int32_t node_index, ChipTaskState state) { memset(&node, 0, sizeof(GraphNodeStorage)); node.slot.task_state.store(state); node.slot.graph_node_index = node_index; @@ -75,8 +75,8 @@ class GraphActivationTest : public ::testing::Test { // routes it — it is never lost on the closed wake list. TEST_F(GraphActivationTest, WakeRoutesConsumerWhenProducerCompletedBeforeRegister) { auto nodes = std::make_unique(2); - init_graph_node(nodes[0], 0, PTO2_TASK_COMPLETED); // producer, already completed - init_graph_node(nodes[1], 1, PTO2_TASK_PENDING); // consumer of node 0 + init_graph_node(nodes[0], 0, CHIP_TASK_COMPLETED); // producer, already completed + init_graph_node(nodes[1], 1, CHIP_TASK_PENDING); // consumer of node 0 nodes[0].slot.wake_list_head.store(WAKE_LIST_SENTINEL); // its wake list already drained std::vector fanin_offsets{0, 0, 1}; // node 0 is a root; node 1 <- {0} @@ -99,10 +99,10 @@ TEST_F(GraphActivationTest, WakeRoutesConsumerWhenProducerCompletedBeforeRegiste // routes exactly once that producer completes and drains its wake list. TEST_F(GraphActivationTest, IncrementalPublishRoutesCompletedDepsAndWakeChainsPending) { auto nodes = std::make_unique(4); - init_graph_node(nodes[0], 0, PTO2_TASK_COMPLETED); // root, completed - init_graph_node(nodes[1], 1, PTO2_TASK_PENDING); // root, pending - init_graph_node(nodes[2], 2, PTO2_TASK_PENDING); // consumer of node 0 (completed) - init_graph_node(nodes[3], 3, PTO2_TASK_PENDING); // consumer of node 1 (pending) + init_graph_node(nodes[0], 0, CHIP_TASK_COMPLETED); // root, completed + init_graph_node(nodes[1], 1, CHIP_TASK_PENDING); // root, pending + init_graph_node(nodes[2], 2, CHIP_TASK_PENDING); // consumer of node 0 (completed) + init_graph_node(nodes[3], 3, CHIP_TASK_PENDING); // consumer of node 1 (pending) std::vector fanin_offsets{0, 0, 0, 1, 2}; // node 2 <- {0}, node 3 <- {1} std::vector fanin_indices{0, 1}; @@ -119,7 +119,7 @@ TEST_F(GraphActivationTest, IncrementalPublishRoutesCompletedDepsAndWakeChainsPe << "only the consumer whose producers are all COMPLETED routes at publish time"; EXPECT_EQ(out[0], &nodes[2].slot); - nodes[1].slot.task_state.store(PTO2_TASK_COMPLETED); + nodes[1].slot.task_state.store(CHIP_TASK_COMPLETED); sched.drain_graph_wake_list(exec, nodes[1].slot); ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 4), 1) << "the wake-chained consumer must route once its pending producer completes"; diff --git a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp index 80b89fc1d7..89c6c02f4c 100644 --- a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp @@ -144,8 +144,8 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { // task_state is written at submit (reset_for_reuse skips it): PENDING for a // dispatchable task, COMPLETED for a pre-completed hidden-alloc. Either way a // real enum, never poison. - const PTO2TaskState state = st.task_state.load(std::memory_order_relaxed); - EXPECT_TRUE(state == PTO2_TASK_PENDING || state == PTO2_TASK_COMPLETED); + const ChipTaskState state = st.task_state.load(std::memory_order_relaxed); + EXPECT_TRUE(state == CHIP_TASK_PENDING || state == CHIP_TASK_COMPLETED); // Completion flag is written to a real 0/1 (pending vs pre-completed), not a // poison byte (0xAA). const uint8_t cflag = ring.completion_flags[slot].load(std::memory_order_relaxed); @@ -158,7 +158,7 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { // predicate.op is a dispatch-time field, read only for tasks the device // actually dispatches. submit_task_common writes it (NONE when unset); a // pre-completed hidden-alloc is never dispatched, so it does not. - if (state == PTO2_TASK_PENDING) { + if (state == CHIP_TASK_PENDING) { EXPECT_LE(static_cast(pl.predicate.op), static_cast(PredicateOp::LE)); } } diff --git a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp index 61687a8f9f..3847b48146 100644 --- a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp @@ -207,7 +207,7 @@ TEST_F(OrchestratorFaninTest, AllCompletedFastPathReleasesWaitOnlyPin) { sm_handle->header->rings[producer.task_id().ring()].get_slot_state_by_task_id(producer.task_id().local()); // COMPLETED but not consumed (the open scope still pins it): the consumer takes // the all-completed fast path. - producer_slot.task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + producer_slot.task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); int32_t rc_before = producer_slot.fanout_refcount.load(); TaskId deps[] = {producer.task_id()}; @@ -237,7 +237,7 @@ TEST_F(OrchestratorFaninTest, SubmitPathHeapDeadlockLogReportsRingAndRealHeapSta auto &ring = sm_handle->header->rings[1]; auto &first_slot = ring.get_slot_state_by_task_id(static_cast(first.task_id().local())); orch.end_scope(); - first_slot.task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + first_slot.task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); sched.check_and_handle_consumed(first_slot); ASSERT_EQ(ring.fc.last_task_alive.load(std::memory_order_acquire), 1); diff --git a/tests/ut/cpp/a2a3/test_scheduler_state.cpp b/tests/ut/cpp/a2a3/test_scheduler_state.cpp index 2f0a446c3b..b79b1670b3 100644 --- a/tests/ut/cpp/a2a3/test_scheduler_state.cpp +++ b/tests/ut/cpp/a2a3/test_scheduler_state.cpp @@ -81,7 +81,7 @@ class SchedulerStateTest : public ::testing::Test { } void init_slot( - ChipTaskSlotState &slot, PTO2TaskState state, int32_t fanin_count, int32_t fanout_count, uint8_t ring_id = 0 + ChipTaskSlotState &slot, ChipTaskState state, int32_t fanin_count, int32_t fanout_count, uint8_t ring_id = 0 ) { memset(&slot, 0, sizeof(slot)); slot.task_state.store(state); @@ -102,7 +102,7 @@ class SchedulerStateTest : public ::testing::Test { } void init_ring_slot( - SharedMemoryRingHeader &ring, int32_t task_id, PTO2TaskState state, uint8_t ring_id, uint32_t fanout_count = 1, + SharedMemoryRingHeader &ring, int32_t task_id, ChipTaskState state, uint8_t ring_id, uint32_t fanout_count = 1, uint32_t fanout_refcount = 1 ) { ChipTaskSlotState &slot = ring.get_slot_state_by_task_id(task_id); @@ -123,7 +123,7 @@ class SchedulerStateTest : public ::testing::Test { slot.completed_subtasks.store(0, std::memory_order_relaxed); slot.total_required_subtasks = 1; slot.logical_block_num = 1; - slot.lifecycle_flags.store(PTO2_COMPLETION_DONE, std::memory_order_relaxed); + slot.lifecycle_flags.store(COMPLETION_DONE, std::memory_order_relaxed); slot.payload = &payload; slot.task = &task; } @@ -139,7 +139,7 @@ class SchedulerStateTest : public ::testing::Test { sched.advance_pending_mask.store(0, std::memory_order_release); for (int32_t task_id = 0; task_id < current_task_index; task_id++) { - PTO2TaskState state = task_id < blocked_task_id ? PTO2_TASK_CONSUMED : PTO2_TASK_COMPLETED; + ChipTaskState state = task_id < blocked_task_id ? CHIP_TASK_CONSUMED : CHIP_TASK_COMPLETED; init_ring_slot(ring, task_id, state, static_cast(ring_id)); } } @@ -152,8 +152,8 @@ class SchedulerStateTest : public ::testing::Test { ring_sched.last_task_alive = head_task_id; ring_sched.advance_lock.store(0, std::memory_order_release); sched.advance_pending_mask.store(0, std::memory_order_release); - init_ring_slot(ring, head_task_id, PTO2_TASK_COMPLETED, static_cast(ring_id)); - init_ring_slot(ring, head_task_id + 1, PTO2_TASK_COMPLETED, static_cast(ring_id), 1, 0); + init_ring_slot(ring, head_task_id, CHIP_TASK_COMPLETED, static_cast(ring_id)); + init_ring_slot(ring, head_task_id + 1, CHIP_TASK_COMPLETED, static_cast(ring_id), 1, 0); } }; @@ -163,20 +163,20 @@ class SchedulerStateTest : public ::testing::Test { TEST_F(SchedulerStateTest, ConsumedNotReady) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_COMPLETED, 1, 2); + init_slot(slot, CHIP_TASK_COMPLETED, 1, 2); slot.fanout_refcount.store(1); // 1 != 2 sched.check_and_handle_consumed(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_COMPLETED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_COMPLETED); } TEST_F(SchedulerStateTest, ConsumedTransition) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_COMPLETED, 1, 2); + init_slot(slot, CHIP_TASK_COMPLETED, 1, 2); slot.fanout_refcount.store(2); // matches fanout_count sched.check_and_handle_consumed(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_CONSUMED); } TEST_F(SchedulerStateTest, ConsumedHeadAdvancesAfterContendedAdvanceLock) { @@ -201,28 +201,28 @@ TEST_F(SchedulerStateTest, ConsumedHeadAdvancesAfterContendedAdvanceLock) { unlocker.join(); EXPECT_TRUE(sched.drain_pending_ring_advances()); - EXPECT_EQ(head.task_state.load(std::memory_order_acquire), PTO2_TASK_CONSUMED); + EXPECT_EQ(head.task_state.load(std::memory_order_acquire), CHIP_TASK_CONSUMED); EXPECT_EQ(ring.fc.last_task_alive.load(std::memory_order_acquire), 1) << "a CONSUMED ring head must not remain pinned after advance_lock contention clears"; } TEST_F(SchedulerStateTest, ConsumedNotCompletedState) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.fanout_refcount.store(1); sched.check_and_handle_consumed(slot); // CAS fails because state is PENDING, not COMPLETED - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_PENDING); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_PENDING); } TEST_F(SchedulerStateTest, ConsumedIdempotent) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_CONSUMED, 1, 1); + init_slot(slot, CHIP_TASK_CONSUMED, 1, 1); slot.fanout_refcount.store(1); sched.check_and_handle_consumed(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_CONSUMED); } TEST_F(SchedulerStateTest, ContendedConsumedHeadSetsPendingAndIdleDrainAdvances) { @@ -238,7 +238,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadSetsPendingAndIdleDrainAdvances) ring_sched.advance_lock.store(1, std::memory_order_release); sched.check_and_handle_consumed(head); - EXPECT_EQ(head.task_state.load(std::memory_order_acquire), PTO2_TASK_CONSUMED); + EXPECT_EQ(head.task_state.load(std::memory_order_acquire), CHIP_TASK_CONSUMED); EXPECT_EQ(ring.fc.last_task_alive.load(std::memory_order_acquire), head_task_id); EXPECT_NE(sched.advance_pending_mask.load(std::memory_order_acquire) & pending_bit, 0u); @@ -282,7 +282,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadIdleDrainStress) { TEST_F(SchedulerStateTest, ReleaseProducerIncrements) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_COMPLETED, 1, 3); + init_slot(slot, CHIP_TASK_COMPLETED, 1, 3); sched.release_producer(slot); EXPECT_EQ(slot.fanout_refcount.load(), 1); @@ -293,11 +293,11 @@ TEST_F(SchedulerStateTest, ReleaseProducerIncrements) { TEST_F(SchedulerStateTest, ReleaseProducerTriggersConsumed) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_COMPLETED, 1, 2); + init_slot(slot, CHIP_TASK_COMPLETED, 1, 2); slot.fanout_refcount.store(1); // One away sched.release_producer(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_CONSUMED); } // ============================================================================= @@ -306,7 +306,7 @@ TEST_F(SchedulerStateTest, ReleaseProducerTriggersConsumed) { TEST_F(SchedulerStateTest, SubtaskCompleteSingle) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 1; slot.completed_subtasks.store(0); @@ -315,7 +315,7 @@ TEST_F(SchedulerStateTest, SubtaskCompleteSingle) { TEST_F(SchedulerStateTest, SubtaskCompleteMultiBlock) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 6; // 3 cores * 2 blocks slot.completed_subtasks.store(0); @@ -335,7 +335,7 @@ TEST_F(SchedulerStateTest, ScopeEndBatchRelease) { ChipTaskSlotState *ptrs[N]; for (int i = 0; i < N; i++) { - init_slot(slots[i], PTO2_TASK_COMPLETED, 1, 2); + init_slot(slots[i], CHIP_TASK_COMPLETED, 1, 2); ptrs[i] = &slots[i]; } @@ -356,8 +356,8 @@ TEST_F(SchedulerStateTest, GetReadyTasksBatchDrainsSharedQueue) { alignas(64) ChipTaskSlotState slot_a, slot_b; // fanin_count = 1 so a single release_fanin_and_check_ready call drives each // slot to ready (new_refcount 0->1 == fanin_count) and enqueues it. - init_slot(slot_a, PTO2_TASK_PENDING, 1, 1); - init_slot(slot_b, PTO2_TASK_PENDING, 1, 1); + init_slot(slot_a, CHIP_TASK_PENDING, 1, 1); + init_slot(slot_b, CHIP_TASK_PENDING, 1, 1); // Route both slots into the global ready queue via the src API. ASSERT_TRUE(sched.release_fanin_and_check_ready(slot_a)); diff --git a/tests/ut/cpp/a2a3/test_task_state.cpp b/tests/ut/cpp/a2a3/test_task_state.cpp index 587e89e5b6..90de1e4151 100644 --- a/tests/ut/cpp/a2a3/test_task_state.cpp +++ b/tests/ut/cpp/a2a3/test_task_state.cpp @@ -60,7 +60,7 @@ class TaskStateTest : public ::testing::Test { sm_arena.release(); } - void init_slot(ChipTaskSlotState &slot, PTO2TaskState state, int32_t fanin_count, int32_t fanout_count) { + void init_slot(ChipTaskSlotState &slot, ChipTaskState state, int32_t fanin_count, int32_t fanout_count) { memset(&slot, 0, sizeof(slot)); slot.task_state.store(state); slot.fanin_count = fanin_count; @@ -86,7 +86,7 @@ class TaskStateTest : public ::testing::Test { // ============================================================================= TEST_F(TaskStateTest, FullLifecycleThroughAPI) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 1; slot.completed_subtasks.store(0); @@ -99,11 +99,11 @@ TEST_F(TaskStateTest, FullLifecycleThroughAPI) { EXPECT_TRUE(done); // Manually transition to COMPLETED (normally done by scheduler dispatch loop) - slot.task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + slot.task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); // Fanout released -> CONSUMED sched.release_producer(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_CONSUMED); } // ============================================================================= @@ -115,13 +115,13 @@ TEST_F(TaskStateTest, FullLifecycleThroughAPI) { // ============================================================================= TEST_F(TaskStateTest, ReadyPathStaysPending) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); bool ready = sched.release_fanin_and_check_ready(slot); ASSERT_TRUE(ready) << "Task should be detected as ready via refcount"; // task_state remains PENDING -- there is no intermediate ready/running state. - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_PENDING) << "release_fanin_and_check_ready must not write task_state"; + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_PENDING) << "release_fanin_and_check_ready must not write task_state"; } // ============================================================================= @@ -129,7 +129,7 @@ TEST_F(TaskStateTest, ReadyPathStaysPending) { // ============================================================================= TEST_F(TaskStateTest, MultiFaninPartialNotReady) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 3, 1); + init_slot(slot, CHIP_TASK_PENDING, 3, 1); EXPECT_FALSE(sched.release_fanin_and_check_ready(slot)); EXPECT_FALSE(sched.release_fanin_and_check_ready(slot)); @@ -144,7 +144,7 @@ TEST_F(TaskStateTest, ConcurrentFaninExactlyOneReady) { for (int round = 0; round < ROUNDS; round++) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 3, 1); + init_slot(slot, CHIP_TASK_PENDING, 3, 1); std::atomic ready_count{0}; auto release = [&]() { @@ -170,7 +170,7 @@ TEST_F(TaskStateTest, ConcurrentSubtaskCompletion) { for (int round = 0; round < ROUNDS; round++) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 3; slot.completed_subtasks.store(0); std::atomic done_count{0}; @@ -199,7 +199,7 @@ TEST_F(TaskStateTest, ConcurrentSubtaskCompletion) { // ============================================================================= TEST_F(TaskStateTest, DoubleSubtaskCompletionCounterWeakness) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 2; slot.completed_subtasks.store(0); diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index ea03dd7067..009118fb6d 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -75,7 +75,7 @@ class WiringTest : public ::testing::Test { // Initialize a slot for testing wiring/completion void init_slot( - ChipTaskSlotState &slot, PTO2TaskState state, int32_t fanin_count, int32_t fanout_count, uint8_t ring_id = 0 + ChipTaskSlotState &slot, ChipTaskState state, int32_t fanin_count, int32_t fanout_count, uint8_t ring_id = 0 ) { memset(&slot, 0, sizeof(slot)); slot.task_state.store(state); @@ -127,7 +127,7 @@ TEST_F(WiringTest, NoFaninTaskBecomesReady) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 0; task_slot.payload = &payload; task_slot.task = &desc; @@ -158,11 +158,11 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { // Set up 2 producers that are already COMPLETED for (int i = 0; i < 2; i++) { - init_slot(producer_slots[i], PTO2_TASK_COMPLETED, 1, 2); + init_slot(producer_slots[i], CHIP_TASK_COMPLETED, 1, 2); } // Consumer task with 2 fanins - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&producer_slots[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producer_slots[1], DEP_WAIT | DEP_RETAIN); @@ -196,10 +196,10 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { // Producers are PENDING (not yet completed) for (int i = 0; i < 2; i++) { - init_slot(producer_slots[i], PTO2_TASK_PENDING, 1, 2); + init_slot(producer_slots[i], CHIP_TASK_PENDING, 1, 2); } - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&producer_slots[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producer_slots[1], DEP_WAIT | DEP_RETAIN); @@ -228,7 +228,7 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { TEST_F(WiringTest, DispatchPropagationMarkerPreservesLifecycleFlagsAndResets) { alignas(64) ChipTaskSlotState producer; - init_slot(producer, PTO2_TASK_PENDING, 1, 1); + init_slot(producer, CHIP_TASK_PENDING, 1, 1); EXPECT_FALSE(producer.has_dispatch_propagated()); EXPECT_TRUE(producer.try_mark_dispatch_propagated()); @@ -238,12 +238,12 @@ TEST_F(WiringTest, DispatchPropagationMarkerPreservesLifecycleFlagsAndResets) { EXPECT_FALSE(producer.try_mark_dispatch_propagated()); EXPECT_EQ( producer.lifecycle_flags.load(std::memory_order_relaxed), - PTO2_READY_CLAIMED | PTO2_COMPLETION_DONE | PTO2_SUBTASK_DEFERRED | PTO2_DISPATCH_PROPAGATED + READY_CLAIMED | COMPLETION_DONE | SUBTASK_DEFERRED | DISPATCH_PROPAGATED ); producer.reset_for_reuse(); EXPECT_FALSE(producer.has_dispatch_propagated()); - EXPECT_EQ(producer.lifecycle_flags.load(std::memory_order_relaxed), PTO2_LIFECYCLE_FLAGS_NONE); + EXPECT_EQ(producer.lifecycle_flags.load(std::memory_order_relaxed), LIFECYCLE_FLAGS_NONE); } // ============================================================================= @@ -257,11 +257,11 @@ TEST_F(WiringTest, WireTaskMixedProducerStates) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(producers[0], PTO2_TASK_COMPLETED, 1, 2); // early finished - init_slot(producers[1], PTO2_TASK_PENDING, 1, 2); // in flight (< COMPLETED) - init_slot(producers[2], PTO2_TASK_CONSUMED, 1, 2); // early finished (>= COMPLETED) + init_slot(producers[0], CHIP_TASK_COMPLETED, 1, 2); // early finished + init_slot(producers[1], CHIP_TASK_PENDING, 1, 2); // in flight (< COMPLETED) + init_slot(producers[2], CHIP_TASK_CONSUMED, 1, 2); // early finished (>= COMPLETED) - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 3; for (int i = 0; i < 3; i++) { payload.fanin_inline_edges[i].set(&producers[i], DEP_WAIT | DEP_RETAIN); @@ -300,11 +300,11 @@ TEST_F(WiringTest, WireTaskAllFlaggedPrecompletedSeedsDispatchFanin) { TaskDescriptor desc{}; for (int i = 0; i < 2; i++) { - init_slot(producer_slots[i], PTO2_TASK_COMPLETED, 1, 2); + init_slot(producer_slots[i], CHIP_TASK_COMPLETED, 1, 2); producer_slots[i].task_attrs.set_early_resolve(true); // codegen-flagged } - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&producer_slots[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producer_slots[1], DEP_WAIT | DEP_RETAIN); @@ -329,10 +329,10 @@ TEST_F(WiringTest, WireTaskUnflaggedPrecompletedProducerDoesNotSeed) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(producer, PTO2_TASK_COMPLETED, 1, 1); + init_slot(producer, CHIP_TASK_COMPLETED, 1, 1); producer.task_attrs.set_early_resolve(false); // unflagged - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 1; payload.fanin_inline_edges[0].set(&producer, DEP_WAIT | DEP_RETAIN); task_slot.payload = &payload; @@ -352,12 +352,12 @@ TEST_F(WiringTest, WireTaskOneUnflaggedProducerDisqualifiesSeed) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(producers[0], PTO2_TASK_COMPLETED, 1, 2); + init_slot(producers[0], CHIP_TASK_COMPLETED, 1, 2); producers[0].task_attrs.set_early_resolve(true); // flagged - init_slot(producers[1], PTO2_TASK_COMPLETED, 1, 2); + init_slot(producers[1], CHIP_TASK_COMPLETED, 1, 2); producers[1].task_attrs.set_early_resolve(false); // one unflagged -> disqualifies - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&producers[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producers[1], DEP_WAIT | DEP_RETAIN); @@ -379,11 +379,11 @@ TEST_F(WiringTest, EarlyDispatchWaitsForAllProducerBlocksPublished) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(producer, PTO2_TASK_PENDING, 1, 1); + init_slot(producer, CHIP_TASK_PENDING, 1, 1); producer.task_attrs.set_early_resolve(true); producer.logical_block_num = 3; - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 1; payload.fanin_inline_edges[0].set(&producer, DEP_WAIT | DEP_RETAIN); task_slot.payload = &payload; @@ -408,7 +408,7 @@ TEST_F(WiringTest, EarlyDispatchWaitsForAllProducerBlocksPublished) { TEST_F(WiringTest, BatchPushReportsFullInsteadOfSpinning) { alignas(64) ChipTaskSlotState filler; - init_slot(filler, PTO2_TASK_PENDING, 0, 1); + init_slot(filler, CHIP_TASK_PENDING, 0, 1); auto &queue = sched.early_dispatch_queues[static_cast(filler.active_mask.to_shape())]; for (uint64_t i = 0; i < queue.capacity; i++) { ASSERT_TRUE(queue.push_tagged(&filler, i)); @@ -427,7 +427,7 @@ TEST_F(WiringTest, BatchPushReportsFullInsteadOfSpinning) { TEST_F(WiringTest, BatchPushSucceedsAfterSpaceIsReclaimed) { alignas(64) ChipTaskSlotState filler; - init_slot(filler, PTO2_TASK_PENDING, 0, 1); + init_slot(filler, CHIP_TASK_PENDING, 0, 1); auto &queue = sched.early_dispatch_queues[static_cast(filler.active_mask.to_shape())]; for (uint64_t i = 0; i < queue.capacity; i++) { ASSERT_TRUE(queue.push_tagged(&filler, i)); @@ -443,8 +443,8 @@ TEST_F(WiringTest, BatchPushSucceedsAfterSpaceIsReclaimed) { TEST_F(WiringTest, EarlyDispatchQueueOverflowRollsBackStagingClaim) { alignas(64) ChipTaskSlotState filler, consumer; - init_slot(filler, PTO2_TASK_PENDING, 0, 1); - init_slot(consumer, PTO2_TASK_PENDING, 0, 1); + init_slot(filler, CHIP_TASK_PENDING, 0, 1); + init_slot(consumer, CHIP_TASK_PENDING, 0, 1); auto shape = static_cast(consumer.active_mask.to_shape()); auto &queue = sched.early_dispatch_queues[shape]; @@ -460,8 +460,8 @@ TEST_F(WiringTest, EarlyDispatchQueueOverflowRollsBackStagingClaim) { TEST_F(WiringTest, EarlyDispatchQueueOverflowFallsBackToNormalDispatch) { alignas(64) ChipTaskSlotState filler, consumer; - init_slot(filler, PTO2_TASK_PENDING, 0, 1); - init_slot(consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(filler, CHIP_TASK_PENDING, 0, 1); + init_slot(consumer, CHIP_TASK_PENDING, 1, 1); ResourceShape shape = consumer.active_mask.to_shape(); auto &queue = sched.early_dispatch_queues[static_cast(shape)]; @@ -481,8 +481,8 @@ TEST_F(WiringTest, EarlyDispatchQueueOverflowFallsBackToNormalDispatch) { TEST_F(WiringTest, EarlyDispatchSyncStartQueueOverflowFallsBackToSyncReadyQueue) { alignas(64) ChipTaskSlotState filler, consumer; - init_slot(filler, PTO2_TASK_PENDING, 0, 1); - init_slot(consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(filler, CHIP_TASK_PENDING, 0, 1); + init_slot(consumer, CHIP_TASK_PENDING, 1, 1); consumer.task_attrs.set_sync_start(); ASSERT_TRUE(consumer.task_attrs.requires_sync_start()); @@ -508,13 +508,13 @@ TEST_F(WiringTest, LateWiredFullyPublishedProducerStillSeedsEarlyDispatch) { memset(&consumer_payload, 0, sizeof(consumer_payload)); TaskDescriptor consumer_desc{}; - init_slot(producer, PTO2_TASK_PENDING, 1, 1); + init_slot(producer, CHIP_TASK_PENDING, 1, 1); producer.task_attrs.set_early_resolve(true); sched.record_published_blocks(producer, producer.logical_block_num); sched.propagate_dispatch_fanin(producer); ASSERT_TRUE(producer.has_dispatch_propagated()); - init_slot(consumer, PTO2_TASK_PENDING, 0, 1); + init_slot(consumer, CHIP_TASK_PENDING, 0, 1); consumer_payload.fanin_actual_count = 1; consumer_payload.fanin_inline_edges[0].set(&producer, DEP_WAIT | DEP_RETAIN); consumer.payload = &consumer_payload; @@ -535,14 +535,14 @@ TEST_F(WiringTest, WiringSeedEnqueuesAfterConcurrentPropagation) { memset(&consumer_payload, 0, sizeof(consumer_payload)); TaskDescriptor consumer_desc{}; - init_slot(producers[0], PTO2_TASK_COMPLETED, 1, 1); - init_slot(producers[1], PTO2_TASK_PENDING, 1, 1); - init_slot(producers[2], PTO2_TASK_COMPLETED, 1, 1); + init_slot(producers[0], CHIP_TASK_COMPLETED, 1, 1); + init_slot(producers[1], CHIP_TASK_PENDING, 1, 1); + init_slot(producers[2], CHIP_TASK_COMPLETED, 1, 1); for (auto &producer : producers) producer.task_attrs.set_early_resolve(true); sched.record_published_blocks(producers[1], producers[1].logical_block_num); - init_slot(consumer, PTO2_TASK_PENDING, 0, 1); + init_slot(consumer, CHIP_TASK_PENDING, 0, 1); consumer_payload.fanin_actual_count = 3; for (int i = 0; i < 3; i++) consumer_payload.fanin_inline_edges[i].set(&producers[i], DEP_WAIT | DEP_RETAIN); @@ -575,7 +575,7 @@ TEST_F(WiringTest, WiringSeedEnqueuesAfterConcurrentPropagation) { TEST_F(WiringTest, ConcurrentBlockRangeClaimsDoNotOverlap) { alignas(64) ChipTaskSlotState task_slot; - init_slot(task_slot, PTO2_TASK_PENDING, 1, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 1, 1); task_slot.logical_block_num = 8; struct ClaimedRange { @@ -610,7 +610,7 @@ TEST_F(WiringTest, ConcurrentBlockRangeClaimsDoNotOverlap) { TEST_F(WiringTest, PartialStagedReleaseRoutesRemainderToReadyQueue) { alignas(64) ChipTaskSlotState consumer; - init_slot(consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(consumer, CHIP_TASK_PENDING, 1, 1); consumer.logical_block_num = 5; consumer.next_block_idx.store(2, std::memory_order_relaxed); consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); @@ -684,8 +684,8 @@ TEST_F(WiringTest, EarlyDispatchLaunchHasSingleOwner) { TEST_F(WiringTest, EarlyDispatchFanoutWaitsForDoorbellPass) { alignas(64) ChipTaskSlotState producer, consumer; - init_slot(producer, PTO2_TASK_PENDING, 1, 1); - init_slot(consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(producer, CHIP_TASK_PENDING, 1, 1); + init_slot(consumer, CHIP_TASK_PENDING, 1, 1); producer.task_attrs.set_early_resolve(true); producer.payload->published_block_count.store(1, std::memory_order_relaxed); @@ -734,7 +734,7 @@ TEST_F(WiringTest, EarlyDispatchReleaseConsumesDoorbellMask) { constexpr uint64_t reg_addr = 0x98760000; constexpr uint32_t token = 11; alignas(64) ChipTaskSlotState task; - init_slot(task, PTO2_TASK_PENDING, 1, 1); + init_slot(task, CHIP_TASK_PENDING, 1, 1); task.task_attrs.set_early_resolve(true); task.next_block_idx.store(1, std::memory_order_relaxed); task.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); @@ -799,8 +799,8 @@ TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { alignas(64) ChipTaskSlotState sync_consumer, downstream; - init_slot(sync_consumer, PTO2_TASK_PENDING, 1, 1); - init_slot(downstream, PTO2_TASK_PENDING, 1, 1); + init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); + init_slot(downstream, CHIP_TASK_PENDING, 1, 1); sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); @@ -836,8 +836,8 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { alignas(64) ChipTaskSlotState sync_consumer, downstream; - init_slot(sync_consumer, PTO2_TASK_PENDING, 1, 1); - init_slot(downstream, PTO2_TASK_PENDING, 1, 1); + init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); + init_slot(downstream, CHIP_TASK_PENDING, 1, 1); sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); @@ -867,7 +867,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { TEST_F(WiringTest, ArmedEarlySyncDrainOwnsFinalReadyRoute) { alignas(64) ChipTaskSlotState sync_consumer; - init_slot(sync_consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; @@ -936,7 +936,7 @@ TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) { alignas(64) ChipTaskSlotState sync_consumer; - init_slot(sync_consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; @@ -960,7 +960,7 @@ TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) TEST_F(WiringTest, CancelledEarlySyncDrainRoutesProducerRelease) { alignas(64) ChipTaskSlotState sync_consumer; - init_slot(sync_consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; @@ -978,7 +978,7 @@ TEST_F(WiringTest, CancelledEarlySyncDrainRoutesProducerRelease) { TEST_F(WiringTest, ProducerReleaseTransfersReadyRouteToCancellingDrain) { alignas(64) ChipTaskSlotState sync_consumer; - init_slot(sync_consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; @@ -1002,12 +1002,12 @@ TEST_F(WiringTest, EarlyDispatchBlockedByUnflaggedProducer) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(p_flagged, PTO2_TASK_PENDING, 1, 1); + init_slot(p_flagged, CHIP_TASK_PENDING, 1, 1); p_flagged.task_attrs.set_early_resolve(true); - init_slot(q_unflagged, PTO2_TASK_PENDING, 1, 1); + init_slot(q_unflagged, CHIP_TASK_PENDING, 1, 1); q_unflagged.task_attrs.set_early_resolve(false); - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&p_flagged, DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&q_unflagged, DEP_WAIT | DEP_RETAIN); @@ -1032,11 +1032,11 @@ TEST_F(WiringTest, UnflaggedProducerDoesNotPropagate) { memset(&prod_payload, 0, sizeof(prod_payload)); memset(&cons_payload, 0, sizeof(cons_payload)); - init_slot(producer, PTO2_TASK_PENDING, 1, 1); + init_slot(producer, CHIP_TASK_PENDING, 1, 1); producer.task_attrs.set_early_resolve(false); // unflagged producer.payload = &prod_payload; - init_slot(consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(consumer, CHIP_TASK_PENDING, 1, 1); consumer.payload = &cons_payload; cons_payload.fanin_actual_count = 1; @@ -1063,12 +1063,12 @@ TEST_F(WiringTest, FlaggedPrecompletedCreatorTransparentToEarlyDispatch) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(creator, PTO2_TASK_COMPLETED, 1, 1); + init_slot(creator, CHIP_TASK_COMPLETED, 1, 1); creator.task_attrs.set_early_resolve(true); // alloc creator: flagged -> transparent - init_slot(compute, PTO2_TASK_PENDING, 1, 1); + init_slot(compute, CHIP_TASK_PENDING, 1, 1); compute.task_attrs.set_early_resolve(true); // flagged compute producer - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&creator, DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&compute, DEP_WAIT | DEP_RETAIN); @@ -1095,17 +1095,17 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { TaskDescriptor desc{}; // Producer in flight (PENDING, not yet COMPLETED) with 2 consumers in fanout chain - init_slot(producer, PTO2_TASK_PENDING, 1, 1); + init_slot(producer, CHIP_TASK_PENDING, 1, 1); producer.payload = &prod_payload; producer.task = &desc; // Consumer1: needs 1 more fanin to become ready - init_slot(consumer1, PTO2_TASK_PENDING, 2, 1); + init_slot(consumer1, CHIP_TASK_PENDING, 2, 1); consumer1.fanin_refcount.store(1); // 1 of 2 satisfied consumer1.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); // Consumer2: this release will make it ready - init_slot(consumer2, PTO2_TASK_PENDING, 2, 1); + init_slot(consumer2, CHIP_TASK_PENDING, 2, 1); consumer2.fanin_refcount.store(1); // 1 of 2 satisfied consumer2.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); @@ -1120,7 +1120,7 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { sched.on_task_complete(producer); // Producer should be COMPLETED - EXPECT_EQ(producer.task_state.load(), PTO2_TASK_COMPLETED); + EXPECT_EQ(producer.task_state.load(), CHIP_TASK_COMPLETED); // Both consumers should have fanin_refcount incremented EXPECT_EQ(consumer1.fanin_refcount.load(), 2); @@ -1146,10 +1146,10 @@ TEST_F(WiringTest, OnTaskReleaseReleasesProducers) { // 2 producers, each COMPLETED with fanout_count=1 for (int i = 0; i < 2; i++) { - init_slot(producers[i], PTO2_TASK_COMPLETED, 1, 1); + init_slot(producers[i], CHIP_TASK_COMPLETED, 1, 1); } - init_slot(task_slot, PTO2_TASK_COMPLETED, 3, 1); + init_slot(task_slot, CHIP_TASK_COMPLETED, 3, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&producers[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producers[1], DEP_WAIT | DEP_RETAIN); @@ -1170,8 +1170,8 @@ TEST_F(WiringTest, OnTaskReleaseReleasesProducers) { EXPECT_EQ(producers[1].fanout_refcount.load(), 1); // Producers with fanout_refcount == fanout_count AND COMPLETED -> CONSUMED - EXPECT_EQ(producers[0].task_state.load(), PTO2_TASK_CONSUMED); - EXPECT_EQ(producers[1].task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(producers[0].task_state.load(), CHIP_TASK_CONSUMED); + EXPECT_EQ(producers[1].task_state.load(), CHIP_TASK_CONSUMED); } // ============================================================================= @@ -1189,10 +1189,10 @@ TEST_F(WiringTest, OrderingOnlyReleasedAtWiringRetentionHeldUntilRelease) { TaskDescriptor desc{}; // Both live (PENDING) with a single submit pin (fanout_count = 1). - init_slot(wait_producer, PTO2_TASK_PENDING, 1, 1); - init_slot(retain_producer, PTO2_TASK_PENDING, 1, 1); + init_slot(wait_producer, CHIP_TASK_PENDING, 1, 1); + init_slot(retain_producer, CHIP_TASK_PENDING, 1, 1); - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&wait_producer, DEP_WAIT); payload.fanin_inline_edges[1].set(&retain_producer, DEP_WAIT | DEP_RETAIN); @@ -1231,9 +1231,9 @@ TEST_F(WiringTest, ReleaseHonorsRetainFlagInSpillRegion) { TaskDescriptor desc{}; // filler carries a large fanout_count so releasing it can never consume it. - init_slot(filler, PTO2_TASK_COMPLETED, 1, 100); - init_slot(spill_retain, PTO2_TASK_COMPLETED, 1, 1); - init_slot(task_slot, PTO2_TASK_COMPLETED, 0, 1); + init_slot(filler, CHIP_TASK_COMPLETED, 1, 100); + init_slot(spill_retain, CHIP_TASK_COMPLETED, 1, 1); + init_slot(task_slot, CHIP_TASK_COMPLETED, 0, 1); for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { payload.fanin_inline_edges[i].set(&filler, DEP_WAIT); @@ -1258,7 +1258,7 @@ TEST_F(WiringTest, ReleaseHonorsRetainFlagInSpillRegion) { EXPECT_EQ(filler.fanout_refcount.load(), 0); // The spilled retention edge is released (and consumed: rc == fc, COMPLETED). EXPECT_EQ(spill_retain.fanout_refcount.load(), 1); - EXPECT_EQ(spill_retain.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(spill_retain.task_state.load(), CHIP_TASK_CONSUMED); } // ============================================================================= @@ -1275,7 +1275,7 @@ TEST_F(WiringTest, AdvanceRingPointersScansConsumed) { // Mark all 3 as CONSUMED for (int i = 0; i < 3; i++) { auto &slot = ring->get_slot_state_by_task_id(i); - slot.task_state.store(PTO2_TASK_CONSUMED, std::memory_order_release); + slot.task_state.store(CHIP_TASK_CONSUMED, std::memory_order_release); } EXPECT_EQ(rss.last_task_alive, 0); @@ -1293,9 +1293,9 @@ TEST_F(WiringTest, AdvanceRingPointersStopsAtNonConsumed) { ring->fc.current_task_index.store(5, std::memory_order_release); // Tasks 0,1 CONSUMED; task 2 COMPLETED (not consumed) - ring->get_slot_state_by_task_id(0).task_state.store(PTO2_TASK_CONSUMED); - ring->get_slot_state_by_task_id(1).task_state.store(PTO2_TASK_CONSUMED); - ring->get_slot_state_by_task_id(2).task_state.store(PTO2_TASK_COMPLETED); + ring->get_slot_state_by_task_id(0).task_state.store(CHIP_TASK_CONSUMED); + ring->get_slot_state_by_task_id(1).task_state.store(CHIP_TASK_CONSUMED); + ring->get_slot_state_by_task_id(2).task_state.store(CHIP_TASK_COMPLETED); rss.advance_ring_pointers(); EXPECT_EQ(rss.last_task_alive, 2) << "Should stop at first non-CONSUMED slot"; @@ -1308,7 +1308,7 @@ TEST_F(WiringTest, AdvanceRingPointersResetsSlots) { ring->fc.current_task_index.store(1, std::memory_order_release); auto &slot = ring->get_slot_state_by_task_id(0); - slot.task_state.store(PTO2_TASK_CONSUMED); + slot.task_state.store(CHIP_TASK_CONSUMED); slot.fanout_count = 5; slot.fanin_refcount.store(3); slot.fanout_refcount.store(2); @@ -1331,7 +1331,7 @@ TEST_F(WiringTest, NoEdgePublishRecordsDepPoolMark) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 0; task_slot.payload = &payload; task_slot.task = &desc; diff --git a/tests/ut/cpp/a5/test_graph_activation.cpp b/tests/ut/cpp/a5/test_graph_activation.cpp index 9937218ad5..885c7420dc 100644 --- a/tests/ut/cpp/a5/test_graph_activation.cpp +++ b/tests/ut/cpp/a5/test_graph_activation.cpp @@ -58,7 +58,7 @@ class GraphActivationTest : public ::testing::Test { // One GraphExecution node whose slot is a routable single-block KERNEL/AIC // task in the given completion state, with its payload wired the way // materialization leaves it for the wake/route path. - static void init_graph_node(GraphNodeStorage &node, int32_t node_index, PTO2TaskState state) { + static void init_graph_node(GraphNodeStorage &node, int32_t node_index, ChipTaskState state) { memset(&node, 0, sizeof(GraphNodeStorage)); node.slot.task_state.store(state); node.slot.graph_node_index = node_index; @@ -75,8 +75,8 @@ class GraphActivationTest : public ::testing::Test { // routes it — it is never lost on the closed wake list. TEST_F(GraphActivationTest, WakeRoutesConsumerWhenProducerCompletedBeforeRegister) { auto nodes = std::make_unique(2); - init_graph_node(nodes[0], 0, PTO2_TASK_COMPLETED); // producer, already completed - init_graph_node(nodes[1], 1, PTO2_TASK_PENDING); // consumer of node 0 + init_graph_node(nodes[0], 0, CHIP_TASK_COMPLETED); // producer, already completed + init_graph_node(nodes[1], 1, CHIP_TASK_PENDING); // consumer of node 0 nodes[0].slot.wake_list_head.store(WAKE_LIST_SENTINEL); // its wake list already drained std::vector fanin_offsets{0, 0, 1}; // node 0 is a root; node 1 <- {0} @@ -99,10 +99,10 @@ TEST_F(GraphActivationTest, WakeRoutesConsumerWhenProducerCompletedBeforeRegiste // routes exactly once that producer completes and drains its wake list. TEST_F(GraphActivationTest, IncrementalPublishRoutesCompletedDepsAndWakeChainsPending) { auto nodes = std::make_unique(4); - init_graph_node(nodes[0], 0, PTO2_TASK_COMPLETED); // root, completed - init_graph_node(nodes[1], 1, PTO2_TASK_PENDING); // root, pending - init_graph_node(nodes[2], 2, PTO2_TASK_PENDING); // consumer of node 0 (completed) - init_graph_node(nodes[3], 3, PTO2_TASK_PENDING); // consumer of node 1 (pending) + init_graph_node(nodes[0], 0, CHIP_TASK_COMPLETED); // root, completed + init_graph_node(nodes[1], 1, CHIP_TASK_PENDING); // root, pending + init_graph_node(nodes[2], 2, CHIP_TASK_PENDING); // consumer of node 0 (completed) + init_graph_node(nodes[3], 3, CHIP_TASK_PENDING); // consumer of node 1 (pending) std::vector fanin_offsets{0, 0, 0, 1, 2}; // node 2 <- {0}, node 3 <- {1} std::vector fanin_indices{0, 1}; @@ -119,7 +119,7 @@ TEST_F(GraphActivationTest, IncrementalPublishRoutesCompletedDepsAndWakeChainsPe << "only the consumer whose producers are all COMPLETED routes at publish time"; EXPECT_EQ(out[0], &nodes[2].slot); - nodes[1].slot.task_state.store(PTO2_TASK_COMPLETED); + nodes[1].slot.task_state.store(CHIP_TASK_COMPLETED); sched.drain_graph_wake_list(exec, nodes[1].slot); ASSERT_EQ(sched.get_ready_tasks_batch(sched.ready_queues, ResourceShape::AIC, out, 4), 1) << "the wake-chained consumer must route once its pending producer completes"; diff --git a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp index 80b89fc1d7..89c6c02f4c 100644 --- a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp @@ -144,8 +144,8 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { // task_state is written at submit (reset_for_reuse skips it): PENDING for a // dispatchable task, COMPLETED for a pre-completed hidden-alloc. Either way a // real enum, never poison. - const PTO2TaskState state = st.task_state.load(std::memory_order_relaxed); - EXPECT_TRUE(state == PTO2_TASK_PENDING || state == PTO2_TASK_COMPLETED); + const ChipTaskState state = st.task_state.load(std::memory_order_relaxed); + EXPECT_TRUE(state == CHIP_TASK_PENDING || state == CHIP_TASK_COMPLETED); // Completion flag is written to a real 0/1 (pending vs pre-completed), not a // poison byte (0xAA). const uint8_t cflag = ring.completion_flags[slot].load(std::memory_order_relaxed); @@ -158,7 +158,7 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { // predicate.op is a dispatch-time field, read only for tasks the device // actually dispatches. submit_task_common writes it (NONE when unset); a // pre-completed hidden-alloc is never dispatched, so it does not. - if (state == PTO2_TASK_PENDING) { + if (state == CHIP_TASK_PENDING) { EXPECT_LE(static_cast(pl.predicate.op), static_cast(PredicateOp::LE)); } } diff --git a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp index b9eb2a7c36..bd91cb8398 100644 --- a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp @@ -219,7 +219,7 @@ TEST_F(OrchestratorFaninTest, AllCompletedFastPathReleasesWaitOnlyPin) { sm_handle->header->rings[producer.task_id().ring()].get_slot_state_by_task_id(producer.task_id().local()); // COMPLETED but not consumed (the open scope still pins it): the consumer takes // the all-completed fast path. - producer_slot.task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + producer_slot.task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); int32_t rc_before = producer_slot.fanout_refcount.load(); TaskId deps[] = {producer.task_id()}; @@ -249,7 +249,7 @@ TEST_F(OrchestratorFaninTest, SubmitPathHeapDeadlockLogReportsRingAndRealHeapSta auto &ring = sm_handle->header->rings[1]; auto &first_slot = ring.get_slot_state_by_task_id(static_cast(first.task_id().local())); orch.end_scope(); - first_slot.task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + first_slot.task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); sched.check_and_handle_consumed(first_slot); ASSERT_EQ(ring.fc.last_task_alive.load(std::memory_order_acquire), 1); diff --git a/tests/ut/cpp/a5/test_scheduler_state.cpp b/tests/ut/cpp/a5/test_scheduler_state.cpp index ece7e6ba9a..84dd21601b 100644 --- a/tests/ut/cpp/a5/test_scheduler_state.cpp +++ b/tests/ut/cpp/a5/test_scheduler_state.cpp @@ -81,7 +81,7 @@ class SchedulerStateTest : public ::testing::Test { } void init_slot( - ChipTaskSlotState &slot, PTO2TaskState state, int32_t fanin_count, int32_t fanout_count, uint8_t ring_id = 0 + ChipTaskSlotState &slot, ChipTaskState state, int32_t fanin_count, int32_t fanout_count, uint8_t ring_id = 0 ) { memset(&slot, 0, sizeof(slot)); slot.task_state.store(state); @@ -102,7 +102,7 @@ class SchedulerStateTest : public ::testing::Test { } void init_ring_slot( - SharedMemoryRingHeader &ring, int32_t task_id, PTO2TaskState state, uint8_t ring_id, uint32_t fanout_count = 1, + SharedMemoryRingHeader &ring, int32_t task_id, ChipTaskState state, uint8_t ring_id, uint32_t fanout_count = 1, uint32_t fanout_refcount = 1 ) { ChipTaskSlotState &slot = ring.get_slot_state_by_task_id(task_id); @@ -123,7 +123,7 @@ class SchedulerStateTest : public ::testing::Test { slot.completed_subtasks.store(0, std::memory_order_relaxed); slot.total_required_subtasks = 1; slot.logical_block_num = 1; - slot.lifecycle_flags.store(PTO2_COMPLETION_DONE, std::memory_order_relaxed); + slot.lifecycle_flags.store(COMPLETION_DONE, std::memory_order_relaxed); slot.payload = &payload; slot.task = &task; } @@ -139,7 +139,7 @@ class SchedulerStateTest : public ::testing::Test { sched.advance_pending_mask.store(0, std::memory_order_release); for (int32_t task_id = 0; task_id < current_task_index; task_id++) { - PTO2TaskState state = task_id < blocked_task_id ? PTO2_TASK_CONSUMED : PTO2_TASK_COMPLETED; + ChipTaskState state = task_id < blocked_task_id ? CHIP_TASK_CONSUMED : CHIP_TASK_COMPLETED; init_ring_slot(ring, task_id, state, static_cast(ring_id)); } } @@ -152,8 +152,8 @@ class SchedulerStateTest : public ::testing::Test { ring_sched.last_task_alive = head_task_id; ring_sched.advance_lock.store(0, std::memory_order_release); sched.advance_pending_mask.store(0, std::memory_order_release); - init_ring_slot(ring, head_task_id, PTO2_TASK_COMPLETED, static_cast(ring_id)); - init_ring_slot(ring, head_task_id + 1, PTO2_TASK_COMPLETED, static_cast(ring_id), 1, 0); + init_ring_slot(ring, head_task_id, CHIP_TASK_COMPLETED, static_cast(ring_id)); + init_ring_slot(ring, head_task_id + 1, CHIP_TASK_COMPLETED, static_cast(ring_id), 1, 0); } }; @@ -168,8 +168,8 @@ TEST_F(SchedulerStateTest, UnvalidatedWiringPublishesEveryAdvance) { EXPECT_FALSE(ring_sched.publication_batching_enabled); ring.fc.current_task_index.store(2, std::memory_order_release); - init_ring_slot(ring, 0, PTO2_TASK_CONSUMED, ring_id); - init_ring_slot(ring, 1, PTO2_TASK_PENDING, ring_id); + init_ring_slot(ring, 0, CHIP_TASK_CONSUMED, ring_id); + init_ring_slot(ring, 1, CHIP_TASK_PENDING, ring_id); ring_sched.advance_ring_pointers(); @@ -179,20 +179,20 @@ TEST_F(SchedulerStateTest, UnvalidatedWiringPublishesEveryAdvance) { TEST_F(SchedulerStateTest, ConsumedNotReady) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_COMPLETED, 1, 2); + init_slot(slot, CHIP_TASK_COMPLETED, 1, 2); slot.fanout_refcount.store(1); // 1 != 2 sched.check_and_handle_consumed(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_COMPLETED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_COMPLETED); } TEST_F(SchedulerStateTest, ConsumedTransition) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_COMPLETED, 1, 2); + init_slot(slot, CHIP_TASK_COMPLETED, 1, 2); slot.fanout_refcount.store(2); // matches fanout_count sched.check_and_handle_consumed(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_CONSUMED); } TEST_F(SchedulerStateTest, ConsumedHeadAdvancesAfterContendedAdvanceLock) { @@ -217,28 +217,28 @@ TEST_F(SchedulerStateTest, ConsumedHeadAdvancesAfterContendedAdvanceLock) { unlocker.join(); EXPECT_TRUE(sched.drain_pending_ring_advances()); - EXPECT_EQ(head.task_state.load(std::memory_order_acquire), PTO2_TASK_CONSUMED); + EXPECT_EQ(head.task_state.load(std::memory_order_acquire), CHIP_TASK_CONSUMED); EXPECT_EQ(ring.fc.last_task_alive.load(std::memory_order_acquire), 1) << "a CONSUMED ring head must not remain pinned after advance_lock contention clears"; } TEST_F(SchedulerStateTest, ConsumedNotCompletedState) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.fanout_refcount.store(1); sched.check_and_handle_consumed(slot); // CAS fails because state is PENDING, not COMPLETED - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_PENDING); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_PENDING); } TEST_F(SchedulerStateTest, ConsumedIdempotent) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_CONSUMED, 1, 1); + init_slot(slot, CHIP_TASK_CONSUMED, 1, 1); slot.fanout_refcount.store(1); sched.check_and_handle_consumed(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_CONSUMED); } TEST_F(SchedulerStateTest, ContendedConsumedHeadSetsPendingAndIdleDrainAdvances) { @@ -254,7 +254,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadSetsPendingAndIdleDrainAdvances) ring_sched.advance_lock.store(1, std::memory_order_release); sched.check_and_handle_consumed(head); - EXPECT_EQ(head.task_state.load(std::memory_order_acquire), PTO2_TASK_CONSUMED); + EXPECT_EQ(head.task_state.load(std::memory_order_acquire), CHIP_TASK_CONSUMED); EXPECT_EQ(ring.fc.last_task_alive.load(std::memory_order_acquire), head_task_id); EXPECT_NE(sched.advance_pending_mask.load(std::memory_order_acquire) & pending_bit, 0u); @@ -335,7 +335,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadIdleDrainStress) { TEST_F(SchedulerStateTest, ReleaseProducerIncrements) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_COMPLETED, 1, 3); + init_slot(slot, CHIP_TASK_COMPLETED, 1, 3); sched.release_producer(slot); EXPECT_EQ(slot.fanout_refcount.load(), 1); @@ -346,11 +346,11 @@ TEST_F(SchedulerStateTest, ReleaseProducerIncrements) { TEST_F(SchedulerStateTest, ReleaseProducerTriggersConsumed) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_COMPLETED, 1, 2); + init_slot(slot, CHIP_TASK_COMPLETED, 1, 2); slot.fanout_refcount.store(1); // One away sched.release_producer(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_CONSUMED); } // ============================================================================= @@ -359,7 +359,7 @@ TEST_F(SchedulerStateTest, ReleaseProducerTriggersConsumed) { TEST_F(SchedulerStateTest, SubtaskCompleteSingle) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 1; slot.completed_subtasks.store(0); @@ -368,7 +368,7 @@ TEST_F(SchedulerStateTest, SubtaskCompleteSingle) { TEST_F(SchedulerStateTest, SubtaskCompleteMultiBlock) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 6; // 3 cores * 2 blocks slot.completed_subtasks.store(0); @@ -388,7 +388,7 @@ TEST_F(SchedulerStateTest, ScopeEndBatchRelease) { ChipTaskSlotState *ptrs[N]; for (int i = 0; i < N; i++) { - init_slot(slots[i], PTO2_TASK_COMPLETED, 1, 2); + init_slot(slots[i], CHIP_TASK_COMPLETED, 1, 2); ptrs[i] = &slots[i]; } @@ -409,8 +409,8 @@ TEST_F(SchedulerStateTest, GetReadyTasksBatchDrainsSharedQueue) { alignas(64) ChipTaskSlotState slot_a, slot_b; // fanin_count = 1 so a single release_fanin_and_check_ready call drives each // slot to ready (new_refcount 0->1 == fanin_count) and enqueues it. - init_slot(slot_a, PTO2_TASK_PENDING, 1, 1); - init_slot(slot_b, PTO2_TASK_PENDING, 1, 1); + init_slot(slot_a, CHIP_TASK_PENDING, 1, 1); + init_slot(slot_b, CHIP_TASK_PENDING, 1, 1); // Route both slots into the global ready queue via the src API. ASSERT_TRUE(sched.release_fanin_and_check_ready(slot_a)); @@ -427,7 +427,7 @@ TEST_F(SchedulerStateTest, GetReadyTasksBatchDrainsSharedQueue) { TEST_F(SchedulerStateTest, SyncStartRoutesToDedicatedReadyQueue) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.task_attrs.set_sync_start(); ASSERT_TRUE(sched.release_fanin_and_check_ready(slot)); diff --git a/tests/ut/cpp/a5/test_task_state.cpp b/tests/ut/cpp/a5/test_task_state.cpp index 587e89e5b6..90de1e4151 100644 --- a/tests/ut/cpp/a5/test_task_state.cpp +++ b/tests/ut/cpp/a5/test_task_state.cpp @@ -60,7 +60,7 @@ class TaskStateTest : public ::testing::Test { sm_arena.release(); } - void init_slot(ChipTaskSlotState &slot, PTO2TaskState state, int32_t fanin_count, int32_t fanout_count) { + void init_slot(ChipTaskSlotState &slot, ChipTaskState state, int32_t fanin_count, int32_t fanout_count) { memset(&slot, 0, sizeof(slot)); slot.task_state.store(state); slot.fanin_count = fanin_count; @@ -86,7 +86,7 @@ class TaskStateTest : public ::testing::Test { // ============================================================================= TEST_F(TaskStateTest, FullLifecycleThroughAPI) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 1; slot.completed_subtasks.store(0); @@ -99,11 +99,11 @@ TEST_F(TaskStateTest, FullLifecycleThroughAPI) { EXPECT_TRUE(done); // Manually transition to COMPLETED (normally done by scheduler dispatch loop) - slot.task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + slot.task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); // Fanout released -> CONSUMED sched.release_producer(slot); - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_CONSUMED); } // ============================================================================= @@ -115,13 +115,13 @@ TEST_F(TaskStateTest, FullLifecycleThroughAPI) { // ============================================================================= TEST_F(TaskStateTest, ReadyPathStaysPending) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); bool ready = sched.release_fanin_and_check_ready(slot); ASSERT_TRUE(ready) << "Task should be detected as ready via refcount"; // task_state remains PENDING -- there is no intermediate ready/running state. - EXPECT_EQ(slot.task_state.load(), PTO2_TASK_PENDING) << "release_fanin_and_check_ready must not write task_state"; + EXPECT_EQ(slot.task_state.load(), CHIP_TASK_PENDING) << "release_fanin_and_check_ready must not write task_state"; } // ============================================================================= @@ -129,7 +129,7 @@ TEST_F(TaskStateTest, ReadyPathStaysPending) { // ============================================================================= TEST_F(TaskStateTest, MultiFaninPartialNotReady) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 3, 1); + init_slot(slot, CHIP_TASK_PENDING, 3, 1); EXPECT_FALSE(sched.release_fanin_and_check_ready(slot)); EXPECT_FALSE(sched.release_fanin_and_check_ready(slot)); @@ -144,7 +144,7 @@ TEST_F(TaskStateTest, ConcurrentFaninExactlyOneReady) { for (int round = 0; round < ROUNDS; round++) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 3, 1); + init_slot(slot, CHIP_TASK_PENDING, 3, 1); std::atomic ready_count{0}; auto release = [&]() { @@ -170,7 +170,7 @@ TEST_F(TaskStateTest, ConcurrentSubtaskCompletion) { for (int round = 0; round < ROUNDS; round++) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 3; slot.completed_subtasks.store(0); std::atomic done_count{0}; @@ -199,7 +199,7 @@ TEST_F(TaskStateTest, ConcurrentSubtaskCompletion) { // ============================================================================= TEST_F(TaskStateTest, DoubleSubtaskCompletionCounterWeakness) { alignas(64) ChipTaskSlotState slot; - init_slot(slot, PTO2_TASK_PENDING, 1, 1); + init_slot(slot, CHIP_TASK_PENDING, 1, 1); slot.total_required_subtasks = 2; slot.completed_subtasks.store(0); diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index 00f24011c4..a21eaef512 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -72,7 +72,7 @@ class WiringTest : public ::testing::Test { // Initialize a slot for testing wiring/completion void init_slot( - ChipTaskSlotState &slot, PTO2TaskState state, int32_t fanin_count, int32_t fanout_count, uint8_t ring_id = 0 + ChipTaskSlotState &slot, ChipTaskState state, int32_t fanin_count, int32_t fanout_count, uint8_t ring_id = 0 ) { memset(&slot, 0, sizeof(slot)); slot.task_state.store(state); @@ -143,7 +143,7 @@ TEST_F(WiringTest, NoFaninTaskBecomesReady) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 0; task_slot.payload = &payload; task_slot.task = &desc; @@ -174,11 +174,11 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { // Set up 2 producers that are already COMPLETED for (int i = 0; i < 2; i++) { - init_slot(producer_slots[i], PTO2_TASK_COMPLETED, 1, 2); + init_slot(producer_slots[i], CHIP_TASK_COMPLETED, 1, 2); } // Consumer task with 2 fanins - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&producer_slots[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producer_slots[1], DEP_WAIT | DEP_RETAIN); @@ -212,10 +212,10 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { // Producers are PENDING (not yet completed) for (int i = 0; i < 2; i++) { - init_slot(producer_slots[i], PTO2_TASK_PENDING, 1, 2); + init_slot(producer_slots[i], CHIP_TASK_PENDING, 1, 2); } - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&producer_slots[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producer_slots[1], DEP_WAIT | DEP_RETAIN); @@ -253,11 +253,11 @@ TEST_F(WiringTest, WireTaskMixedProducerStates) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(producers[0], PTO2_TASK_COMPLETED, 1, 2); // early finished - init_slot(producers[1], PTO2_TASK_PENDING, 1, 2); // in flight (< COMPLETED) - init_slot(producers[2], PTO2_TASK_CONSUMED, 1, 2); // early finished (>= COMPLETED) + init_slot(producers[0], CHIP_TASK_COMPLETED, 1, 2); // early finished + init_slot(producers[1], CHIP_TASK_PENDING, 1, 2); // in flight (< COMPLETED) + init_slot(producers[2], CHIP_TASK_CONSUMED, 1, 2); // early finished (>= COMPLETED) - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 3; for (int i = 0; i < 3; i++) { payload.fanin_inline_edges[i].set(&producers[i], DEP_WAIT | DEP_RETAIN); @@ -310,8 +310,8 @@ TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { alignas(64) ChipTaskSlotState sync_consumer, downstream; - init_slot(sync_consumer, PTO2_TASK_PENDING, 1, 1); - init_slot(downstream, PTO2_TASK_PENDING, 1, 1); + init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); + init_slot(downstream, CHIP_TASK_PENDING, 1, 1); sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); @@ -345,8 +345,8 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { alignas(64) ChipTaskSlotState sync_consumer, downstream; - init_slot(sync_consumer, PTO2_TASK_PENDING, 1, 1); - init_slot(downstream, PTO2_TASK_PENDING, 1, 1); + init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); + init_slot(downstream, CHIP_TASK_PENDING, 1, 1); sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); @@ -376,7 +376,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) { alignas(64) ChipTaskSlotState sync_consumer; - init_slot(sync_consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; @@ -409,17 +409,17 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { TaskDescriptor desc{}; // Producer in flight (PENDING, not yet COMPLETED) with 2 consumers in fanout chain - init_slot(producer, PTO2_TASK_PENDING, 1, 1); + init_slot(producer, CHIP_TASK_PENDING, 1, 1); producer.payload = &prod_payload; producer.task = &desc; // Consumer1: needs 1 more fanin to become ready - init_slot(consumer1, PTO2_TASK_PENDING, 2, 1); + init_slot(consumer1, CHIP_TASK_PENDING, 2, 1); consumer1.fanin_refcount.store(1); // 1 of 2 satisfied consumer1.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); // Consumer2: this release will make it ready - init_slot(consumer2, PTO2_TASK_PENDING, 2, 1); + init_slot(consumer2, CHIP_TASK_PENDING, 2, 1); consumer2.fanin_refcount.store(1); // 1 of 2 satisfied consumer2.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); @@ -439,7 +439,7 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { #endif // Producer should be COMPLETED - EXPECT_EQ(producer.task_state.load(), PTO2_TASK_COMPLETED); + EXPECT_EQ(producer.task_state.load(), CHIP_TASK_COMPLETED); // Both consumers should have fanin_refcount incremented EXPECT_EQ(consumer1.fanin_refcount.load(), 2); @@ -465,10 +465,10 @@ TEST_F(WiringTest, OnTaskReleaseReleasesProducers) { // 2 producers, each COMPLETED with fanout_count=1 for (int i = 0; i < 2; i++) { - init_slot(producers[i], PTO2_TASK_COMPLETED, 1, 1); + init_slot(producers[i], CHIP_TASK_COMPLETED, 1, 1); } - init_slot(task_slot, PTO2_TASK_COMPLETED, 3, 1); + init_slot(task_slot, CHIP_TASK_COMPLETED, 3, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&producers[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producers[1], DEP_WAIT | DEP_RETAIN); @@ -489,8 +489,8 @@ TEST_F(WiringTest, OnTaskReleaseReleasesProducers) { EXPECT_EQ(producers[1].fanout_refcount.load(), 1); // Producers with fanout_refcount == fanout_count AND COMPLETED -> CONSUMED - EXPECT_EQ(producers[0].task_state.load(), PTO2_TASK_CONSUMED); - EXPECT_EQ(producers[1].task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(producers[0].task_state.load(), CHIP_TASK_CONSUMED); + EXPECT_EQ(producers[1].task_state.load(), CHIP_TASK_CONSUMED); } // ============================================================================= @@ -508,10 +508,10 @@ TEST_F(WiringTest, OrderingOnlyReleasedAtWiringRetentionHeldUntilRelease) { TaskDescriptor desc{}; // Both live (PENDING) with a single submit pin (fanout_count = 1). - init_slot(wait_producer, PTO2_TASK_PENDING, 1, 1); - init_slot(retain_producer, PTO2_TASK_PENDING, 1, 1); + init_slot(wait_producer, CHIP_TASK_PENDING, 1, 1); + init_slot(retain_producer, CHIP_TASK_PENDING, 1, 1); - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&wait_producer, DEP_WAIT); payload.fanin_inline_edges[1].set(&retain_producer, DEP_WAIT | DEP_RETAIN); @@ -550,9 +550,9 @@ TEST_F(WiringTest, ReleaseHonorsRetainFlagInSpillRegion) { TaskDescriptor desc{}; // filler carries a large fanout_count so releasing it can never consume it. - init_slot(filler, PTO2_TASK_COMPLETED, 1, 100); - init_slot(spill_retain, PTO2_TASK_COMPLETED, 1, 1); - init_slot(task_slot, PTO2_TASK_COMPLETED, 0, 1); + init_slot(filler, CHIP_TASK_COMPLETED, 1, 100); + init_slot(spill_retain, CHIP_TASK_COMPLETED, 1, 1); + init_slot(task_slot, CHIP_TASK_COMPLETED, 0, 1); for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { payload.fanin_inline_edges[i].set(&filler, DEP_WAIT); @@ -577,7 +577,7 @@ TEST_F(WiringTest, ReleaseHonorsRetainFlagInSpillRegion) { EXPECT_EQ(filler.fanout_refcount.load(), 0); // The spilled retention edge is released (and consumed: rc == fc, COMPLETED). EXPECT_EQ(spill_retain.fanout_refcount.load(), 1); - EXPECT_EQ(spill_retain.task_state.load(), PTO2_TASK_CONSUMED); + EXPECT_EQ(spill_retain.task_state.load(), CHIP_TASK_CONSUMED); } // ============================================================================= @@ -594,7 +594,7 @@ TEST_F(WiringTest, AdvanceRingPointersScansConsumed) { // Mark all 3 as CONSUMED for (int i = 0; i < 3; i++) { auto &slot = ring->get_slot_state_by_task_id(i); - slot.task_state.store(PTO2_TASK_CONSUMED, std::memory_order_release); + slot.task_state.store(CHIP_TASK_CONSUMED, std::memory_order_release); } EXPECT_EQ(rss.last_task_alive, 0); @@ -611,7 +611,7 @@ TEST_F(WiringTest, AdvanceRingPointersBatchesSharedMemoryPublication) { ring->fc.current_task_index.store(18, std::memory_order_release); for (int i = 0; i < 17; i++) { - ring->get_slot_state_by_task_id(i).task_state.store(PTO2_TASK_CONSUMED); + ring->get_slot_state_by_task_id(i).task_state.store(CHIP_TASK_CONSUMED); } rss.advance_ring_pointers(); @@ -623,9 +623,9 @@ TEST_F(WiringTest, AdvanceRingPointersBatchesSharedMemoryPublication) { rss.last_published_to_sm = 0; for (int advances : {1, 15, 16, 17}) { for (int i = 0; i < advances; i++) { - ring->get_slot_state_by_task_id(i).task_state.store(PTO2_TASK_CONSUMED); + ring->get_slot_state_by_task_id(i).task_state.store(CHIP_TASK_CONSUMED); } - ring->get_slot_state_by_task_id(advances).task_state.store(PTO2_TASK_COMPLETED); + ring->get_slot_state_by_task_id(advances).task_state.store(CHIP_TASK_COMPLETED); rss.advance_ring_pointers(); EXPECT_EQ(ring->fc.last_task_alive.load(), advances >= 16 ? advances : 0); @@ -642,7 +642,7 @@ TEST_F(WiringTest, AdvanceRingPointersPublishesDrainedTail) { for (int advances : {1, 15, 16, 17}) { ring->fc.current_task_index.store(advances, std::memory_order_release); for (int i = 0; i < advances; i++) { - ring->get_slot_state_by_task_id(i).task_state.store(PTO2_TASK_CONSUMED); + ring->get_slot_state_by_task_id(i).task_state.store(CHIP_TASK_CONSUMED); } rss.advance_ring_pointers(); @@ -710,7 +710,7 @@ TEST_F(WiringTest, DrainPublicationRequestsPublishesWithheldProgress) { auto *ring = rss.ring; ring->fc.current_task_index.store(129, std::memory_order_release); - ring->get_slot_state_by_task_id(128).task_state.store(PTO2_TASK_PENDING); + ring->get_slot_state_by_task_id(128).task_state.store(CHIP_TASK_PENDING); rss.last_task_alive = 128; rss.last_published_to_sm = 113; ring->fc.last_task_alive.store(113, std::memory_order_release); @@ -742,8 +742,8 @@ TEST_F(WiringTest, TaskWindowOpensOnceWithheldProgressIsPublished) { alignas(64) char heap[64] = {}; ring->fc.current_task_index.store(3, std::memory_order_release); ring->get_task_by_task_id(0).packed_buffer_end = heap; - ring->get_slot_state_by_task_id(0).task_state.store(PTO2_TASK_CONSUMED); - ring->get_slot_state_by_task_id(1).task_state.store(PTO2_TASK_PENDING); + ring->get_slot_state_by_task_id(0).task_state.store(CHIP_TASK_CONSUMED); + ring->get_slot_state_by_task_id(1).task_state.store(CHIP_TASK_PENDING); rss.advance_ring_pointers(); ASSERT_EQ(rss.last_task_alive, 1); @@ -794,8 +794,8 @@ TEST_F(WiringTest, HeapReclaimsOnceWithheldProgressIsPublished) { ASSERT_FALSE(live_tail.failed()); ring->get_task_by_task_id(live_tail.task_id).packed_buffer_end = live_tail.packed_end; - ring->get_slot_state_by_task_id(0).task_state.store(PTO2_TASK_CONSUMED); - ring->get_slot_state_by_task_id(1).task_state.store(PTO2_TASK_PENDING); + ring->get_slot_state_by_task_id(0).task_state.store(CHIP_TASK_CONSUMED); + ring->get_slot_state_by_task_id(1).task_state.store(CHIP_TASK_PENDING); rss.advance_ring_pointers(); ASSERT_EQ(rss.last_task_alive, 1); @@ -824,7 +824,7 @@ TEST_F(WiringTest, DependencyPoolReclaimsOnceWithheldProgressIsPublished) { ring->fc.current_task_index.store(129, std::memory_order_release); ring->fc.last_task_alive.store(113, std::memory_order_release); ring->get_slot_state_by_task_id(127).dep_pool_mark = 5; - ring->get_slot_state_by_task_id(128).task_state.store(PTO2_TASK_PENDING); + ring->get_slot_state_by_task_id(128).task_state.store(CHIP_TASK_PENDING); rss.last_task_alive = 128; rss.last_published_to_sm = 113; rss.dep_pool.capacity = 8; @@ -851,8 +851,8 @@ TEST_F(WiringTest, FaninPoolReclaimsOnceWithheldProgressIsPublished) { auto *ring = rss.ring; ring->fc.current_task_index.store(2, std::memory_order_release); - ring->get_slot_state_by_task_id(0).task_state.store(PTO2_TASK_CONSUMED); - ring->get_slot_state_by_task_id(1).task_state.store(PTO2_TASK_PENDING); + ring->get_slot_state_by_task_id(0).task_state.store(CHIP_TASK_CONSUMED); + ring->get_slot_state_by_task_id(1).task_state.store(CHIP_TASK_PENDING); rss.advance_ring_pointers(); ASSERT_EQ(rss.last_task_alive, 1); ASSERT_EQ(ring->fc.last_task_alive.load(), 0); @@ -905,9 +905,9 @@ TEST_F(WiringTest, AdvanceRingPointersStopsAtNonConsumed) { ring->fc.current_task_index.store(5, std::memory_order_release); // Tasks 0,1 CONSUMED; task 2 COMPLETED (not consumed) - ring->get_slot_state_by_task_id(0).task_state.store(PTO2_TASK_CONSUMED); - ring->get_slot_state_by_task_id(1).task_state.store(PTO2_TASK_CONSUMED); - ring->get_slot_state_by_task_id(2).task_state.store(PTO2_TASK_COMPLETED); + ring->get_slot_state_by_task_id(0).task_state.store(CHIP_TASK_CONSUMED); + ring->get_slot_state_by_task_id(1).task_state.store(CHIP_TASK_CONSUMED); + ring->get_slot_state_by_task_id(2).task_state.store(CHIP_TASK_COMPLETED); rss.advance_ring_pointers(); EXPECT_EQ(rss.last_task_alive, 2) << "Should stop at first non-CONSUMED slot"; @@ -920,7 +920,7 @@ TEST_F(WiringTest, AdvanceRingPointersResetsSlots) { ring->fc.current_task_index.store(1, std::memory_order_release); auto &slot = ring->get_slot_state_by_task_id(0); - slot.task_state.store(PTO2_TASK_CONSUMED); + slot.task_state.store(CHIP_TASK_CONSUMED); slot.fanout_count = 5; slot.fanin_refcount.store(3); slot.fanout_refcount.store(2); @@ -943,7 +943,7 @@ TEST_F(WiringTest, NoEdgePublishRecordsDepPoolMark) { memset(&payload, 0, sizeof(payload)); TaskDescriptor desc{}; - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); + init_slot(task_slot, CHIP_TASK_PENDING, 0, 1); payload.fanin_actual_count = 0; task_slot.payload = &payload; task_slot.task = &desc; @@ -956,7 +956,7 @@ TEST_F(WiringTest, NoEdgePublishRecordsDepPoolMark) { TEST_F(WiringTest, BatchPushReportsFullInsteadOfSpinning) { alignas(64) ChipTaskSlotState filler; - init_slot(filler, PTO2_TASK_PENDING, 0, 1); + init_slot(filler, CHIP_TASK_PENDING, 0, 1); auto &queue = sched.early_dispatch_queues[static_cast(filler.active_mask.to_shape())]; for (uint64_t i = 0; i < queue.capacity; i++) { ASSERT_TRUE(queue.push_tagged(&filler, i)); @@ -975,7 +975,7 @@ TEST_F(WiringTest, BatchPushReportsFullInsteadOfSpinning) { TEST_F(WiringTest, BatchPushSucceedsAfterSpaceIsReclaimed) { alignas(64) ChipTaskSlotState filler; - init_slot(filler, PTO2_TASK_PENDING, 0, 1); + init_slot(filler, CHIP_TASK_PENDING, 0, 1); auto &queue = sched.early_dispatch_queues[static_cast(filler.active_mask.to_shape())]; for (uint64_t i = 0; i < queue.capacity; i++) { ASSERT_TRUE(queue.push_tagged(&filler, i)); @@ -991,8 +991,8 @@ TEST_F(WiringTest, BatchPushSucceedsAfterSpaceIsReclaimed) { TEST_F(WiringTest, EarlyDispatchQueueOverflowFallsBackToNormalDispatch) { alignas(64) ChipTaskSlotState filler, consumer; - init_slot(filler, PTO2_TASK_PENDING, 0, 1); - init_slot(consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(filler, CHIP_TASK_PENDING, 0, 1); + init_slot(consumer, CHIP_TASK_PENDING, 1, 1); ResourceShape shape = consumer.active_mask.to_shape(); auto &queue = sched.early_dispatch_queues[static_cast(shape)]; @@ -1012,8 +1012,8 @@ TEST_F(WiringTest, EarlyDispatchQueueOverflowFallsBackToNormalDispatch) { TEST_F(WiringTest, EarlyDispatchSyncStartQueueOverflowFallsBackToSyncReadyQueue) { alignas(64) ChipTaskSlotState filler, consumer; - init_slot(filler, PTO2_TASK_PENDING, 0, 1); - init_slot(consumer, PTO2_TASK_PENDING, 1, 1); + init_slot(filler, CHIP_TASK_PENDING, 0, 1); + init_slot(consumer, CHIP_TASK_PENDING, 1, 1); consumer.task_attrs.set_sync_start(); ASSERT_TRUE(consumer.task_attrs.requires_sync_start()); diff --git a/tests/ut/cpp/common/test_hbg_graph_cache.cpp b/tests/ut/cpp/common/test_hbg_graph_cache.cpp index 63c2ed7b7f..9e107a9232 100644 --- a/tests/ut/cpp/common/test_hbg_graph_cache.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_cache.cpp @@ -752,7 +752,7 @@ TEST(GraphExecutionMaterialize, DirtyStorageYieldsValidExecution) { // values only the device side wrote. for (int32_t i = 0; i < execution->node_count; ++i) { const GraphNodeStorage &node = execution->node_at(i); - ASSERT_EQ(node.slot.task_state.load(std::memory_order_relaxed), PTO2_TASK_PENDING); + ASSERT_EQ(node.slot.task_state.load(std::memory_order_relaxed), CHIP_TASK_PENDING); ASSERT_EQ(node.slot.task_kind, TaskKind::GRAPH_NODE); ASSERT_EQ(node.slot.completed_subtasks.load(std::memory_order_relaxed), 0); ASSERT_EQ(node.payload.dispatch_fanin.load(std::memory_order_relaxed), 0); diff --git a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp index ae987ff199..ea526e461f 100644 --- a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp +++ b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp @@ -27,7 +27,7 @@ void make_head_match_old_structural_predicate( ) { descriptor.task_id = TaskId::make(ring_id, local_task_id); head.task = &descriptor; - head.task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + head.task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); head.fanout_count = PTO2_FANOUT_SCOPE_BIT; head.fanout_refcount.store(0, std::memory_order_release); } From 4aff540979725ebfef12d12aaebea3b42cee53e7 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:22:55 -0700 Subject: [PATCH 09/22] Refactor: name the chip runtime's ring types by their context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The five ring types take the `Chip` prefix as one cluster: `ChipTaskRing`, `ChipHeapRing`, `ChipRingSet`, `ChipRingFlowControl`, `ChipRingSegmentOffsets`. `HeapRing` is the name that forces it — `src/common/hierarchical/ring.h:151` declares a `struct HeapRing` inside the host orchestrator's `class Ring`, which owns `mmap`ed host slabs with a mutex and condition variable. The chip runtime's rings live in the device shared-memory image and are advanced by atomic counters. Rule 13 gives the bare names to the L3+ side. The other four do not collide on their own, but they are the same ring: splitting the cluster would leave `ChipHeapRing` and `RingFlowControl` as fields of one struct, which reads as two subsystems rather than one. 85 occurrences across 25 files. Co-Authored-By: Claude Opus 5 (1M context) --- .../host_build_graph/host/runtime_maker.cpp | 2 +- .../host_build_graph/runtime/ring_buffer.h | 2 +- .../runtime/shared/shared_memory.cpp | 2 +- .../host_build_graph/runtime/shared_memory.h | 20 +++++++++---------- .../docs/MULTI_RING.md | 18 ++++++++--------- .../docs/RUNTIME_LOGIC.md | 6 +++--- .../runtime/orchestrator.cpp | 2 +- .../runtime/orchestrator.h | 2 +- .../runtime/ring_buffer.h | 4 ++-- .../runtime/shared/shared_memory.cpp | 2 +- .../runtime/shared_memory.h | 16 +++++++-------- .../host_build_graph/host/runtime_maker.cpp | 2 +- .../host_build_graph/runtime/ring_buffer.h | 2 +- .../runtime/shared/shared_memory.cpp | 2 +- .../host_build_graph/runtime/shared_memory.h | 20 +++++++++---------- .../docs/MULTI_RING.md | 18 ++++++++--------- .../docs/RUNTIME_LOGIC.md | 6 +++--- .../runtime/orchestrator.cpp | 2 +- .../runtime/orchestrator.h | 2 +- .../runtime/ring_buffer.h | 4 ++-- .../runtime/shared/shared_memory.cpp | 2 +- .../runtime/shared_memory.h | 16 +++++++-------- tests/ut/cpp/a2a3/test_task_allocator.cpp | 2 +- tests/ut/cpp/a5/test_task_allocator.cpp | 2 +- .../ut/cpp/common/test_hbg_sm_compaction.cpp | 2 +- 25 files changed, 79 insertions(+), 79 deletions(-) diff --git a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp index 65ea3c00c1..42bc0c1928 100644 --- a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp @@ -554,7 +554,7 @@ int32_t run_host_orchestration( // each written per task at submit and read only for [0, total_tasks). Zero // only the fixed-size header here; the per-slot segments are initialized in // orch::prepare_task and shipped bounded to total_tasks below. - const sm_layout::PTO2RingSegmentOffsets sm_segs = sm_layout::ring_segment_offsets(eff_task_window_size); + const sm_layout::ChipRingSegmentOffsets sm_segs = sm_layout::ring_segment_offsets(eff_task_window_size); // Over-allocated and rounded up: every segment offset is a multiple of // PTO2_ALIGN_SIZE and ChipTaskSlotState is alignas(64), which a plain // new uint8_t[] does not guarantee. diff --git a/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h b/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h index 6b96d82441..956b3d3c9a 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h +++ b/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h @@ -60,7 +60,7 @@ class PTO2TaskAllocator { * from host code that constructs a prebuilt arena image. * * The ring starts at task id 0, matching the SM flow-control counter that - * current_index_ptr points at (PTO2RingFlowControl::init() runs on the AICPU + * current_index_ptr points at (ChipRingFlowControl::init() runs on the AICPU * during SM reset), so local_task_id_ stays in sync without reading the SM. * Because ids are never reclaimed, alloc() caps them at window_size — they * cannot run away toward INT32_MAX. diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 91b187b733..7cc4fd7308 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -211,7 +211,7 @@ bool SharedMemoryHandle::validate() { return true; } -bool PTO2RingFlowControl::validate(SharedMemoryHandle *handle) const { +bool ChipRingFlowControl::validate(SharedMemoryHandle *handle) const { if (!handle) return false; if (!handle->header) return false; diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h index 3a76a415fa..92b1a2b871 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h @@ -50,7 +50,7 @@ struct SharedMemoryHandle; * Per-ring flow control state in shared memory. * Written/read by Orchestrator and Scheduler for synchronization. */ -struct alignas(64) PTO2RingFlowControl { +struct alignas(64) ChipRingFlowControl { // Written by Orchestrator, read by Scheduler. There is no reverse channel: // the ring is whole-graph-resident, so the scheduler never reclaims task // slots and has nothing to publish back. @@ -67,7 +67,7 @@ struct alignas(64) PTO2RingFlowControl { bool validate(SharedMemoryHandle *handle) const; }; -static_assert(sizeof(PTO2RingFlowControl) == 64, "PTO2RingFlowControl must be exactly one cache line (64B)"); +static_assert(sizeof(ChipRingFlowControl) == 64, "ChipRingFlowControl must be exactly one cache line (64B)"); /** * Per-ring shared memory header section. @@ -76,7 +76,7 @@ static_assert(sizeof(PTO2RingFlowControl) == 64, "PTO2RingFlowControl must be ex * Pointers are host-side only (set by setup_pointers, invalid on device). */ struct alignas(64) SharedMemoryRingHeader { - PTO2RingFlowControl fc; + ChipRingFlowControl fc; // Highest task_id such that every task with id in [0, completed_watermark] // has its completion_flags byte set. Advanced over the full contiguous @@ -293,7 +293,7 @@ inline SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base) noexcept { inline std::atomic *ring_current_task_index_addr(void *sm_dev_base) noexcept { return reinterpret_cast *>( reinterpret_cast(ring_header_addr(sm_dev_base)) + offsetof(SharedMemoryRingHeader, fc) + - offsetof(PTO2RingFlowControl, current_task_index) + offsetof(ChipRingFlowControl, current_task_index) ); } @@ -306,7 +306,7 @@ inline std::atomic *ring_current_task_index_addr(void *sm_dev_base) noe // The pools sit last because nothing on the device resolves a segment past // completion_flags: a payload names its argument regions by delta, so the four // slot-pitched offsets are all the attach path computes. -struct PTO2RingSegmentOffsets { +struct ChipRingSegmentOffsets { uint64_t descriptors; uint64_t payloads; uint64_t slot_states; @@ -342,9 +342,9 @@ inline RingImageExtents mirror_extents(uint64_t task_window_size) noexcept { // `sm_base`) and the device-address helpers below (which add `sm_dev_base`). Adding // or reordering a segment is a one-line edit here; every consumer follows // automatically, so the layout walk can never silently disagree across call sites. -inline PTO2RingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) noexcept { +inline ChipRingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) noexcept { uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); - PTO2RingSegmentOffsets o{}; + ChipRingSegmentOffsets o{}; o.descriptors = off; off += PTO2_ALIGN_UP(e.slots * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); o.payloads = off; @@ -363,7 +363,7 @@ inline PTO2RingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) no return o; } -inline PTO2RingSegmentOffsets ring_segment_offsets(uint64_t task_window_size) noexcept { +inline ChipRingSegmentOffsets ring_segment_offsets(uint64_t task_window_size) noexcept { return ring_segment_offsets(mirror_extents(task_window_size)); } @@ -473,8 +473,8 @@ inline uint64_t compact_live_image( always_assert(used.fanin_elems <= mirror.fanin_elems); always_assert(used.tensor_elems <= mirror.tensor_elems); always_assert(used.scalar_elems <= mirror.scalar_elems); - const PTO2RingSegmentOffsets from = ring_segment_offsets(mirror); - const PTO2RingSegmentOffsets to = ring_segment_offsets(image_extents(used)); + const ChipRingSegmentOffsets from = ring_segment_offsets(mirror); + const ChipRingSegmentOffsets to = ring_segment_offsets(image_extents(used)); // The header and the descriptors offset are pitch-independent, so the header // lands where it already was. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index fe6f3404fe..61e850f330 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -46,14 +46,14 @@ Type changes: ## 4. Data Structures -### 4.1 PTO2RingSet (new) +### 4.1 ChipRingSet (new) Bundles the three per-ring resources into a single aggregate (`ring_buffer.h`): ```cpp -struct PTO2RingSet { - PTO2HeapRing heap_ring; - PTO2TaskRing task_ring; +struct ChipRingSet { + ChipHeapRing heap_ring; + ChipTaskRing task_ring; PTO2FaninPool fanin_pool; }; ``` @@ -62,12 +62,12 @@ struct PTO2RingSet { ```cpp // Before: single ring -PTO2HeapRing heap_ring; -PTO2TaskRing task_ring; +ChipHeapRing heap_ring; +ChipTaskRing task_ring; PTO2DepListPool dep_pool; // After: per-ring array (dep_pool moved to scheduler, see §4.5) -PTO2RingSet rings[CHIP_MAX_RING_DEPTH]; +ChipRingSet rings[CHIP_MAX_RING_DEPTH]; ``` Ring selection: `current_ring_id() = min(scope_stack_top, CHIP_MAX_RING_DEPTH - 1)`. @@ -77,7 +77,7 @@ Ring selection: `current_ring_id() = min(scope_stack_top, CHIP_MAX_RING_DEPTH - Per-ring flow control and per-ring layout info are grouped together: ```cpp -struct PTO2RingFlowControl { +struct ChipRingFlowControl { std::atomic current_task_index; // task ring head std::atomic last_task_alive; // task ring tail std::atomic heap_top; // heap alloc pointer @@ -85,7 +85,7 @@ struct PTO2RingFlowControl { }; struct alignas(64) SharedMemoryRingHeader { - PTO2RingFlowControl fc; + ChipRingFlowControl fc; // Layout metadata (set once at init) uint64_t task_window_size; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 4176c4ec9e..d5abcd1527 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -189,7 +189,7 @@ Alignment is 64 bytes (`PTO2_ALIGN_SIZE`). The task ring manages task slot allocation with back-pressure flow control. -**Structure** (`PTO2TaskRing`): +**Structure** (`ChipTaskRing`): - `descriptors`: pointer to `TaskDescriptor[]` in shared memory - `window_size`: number of slots (power of 2) @@ -216,7 +216,7 @@ else: The heap ring manages output buffer allocation from a circular GM heap. -**Structure** (`PTO2HeapRing`): +**Structure** (`ChipHeapRing`): - `base`: GM heap base address - `size`: total heap size (default 1 GB) @@ -421,7 +421,7 @@ The orchestrator runs on AICPU Thread 3 and builds the task graph by calling the Key members: -- `rings[CHIP_MAX_RING_DEPTH]`: per-ring `PTO2RingSet` (HeapRing + TaskRing + FaninPool). See [MULTI_RING.md §4.2](MULTI_RING.md). +- `rings[CHIP_MAX_RING_DEPTH]`: per-ring `ChipRingSet` (HeapRing + TaskRing + FaninPool). See [MULTI_RING.md §4.2](MULTI_RING.md). - `tensor_map`, `tensor_pool`: dependency tracking - `scope_tasks[]`, `scope_begins[]`, `scope_stack_top`: scope nesting stack (flat buffer partitioned by level) - `scheduler`: pointer to scheduler state (for Orch-side wiring helpers and ready queue access) diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index f9cf95a24b..e6e8cd2e89 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -894,7 +894,7 @@ static TaskOutputTensors submit_task_common( } uint8_t ring_id = prepared.task_id.ring(); SchedulerState *sched = orch->scheduler; - PTO2RingFlowControl &fc = orch->sm_header->rings[ring_id].fc; + ChipRingFlowControl &fc = orch->sm_header->rings[ring_id].fc; TaskId task_id = prepared.task_id; ChipTaskSlotState &cur_slot_state = *prepared.slot_state; TaskDescriptor &task = *prepared.task; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 0b5949dab4..cb91daa77c 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -67,7 +67,7 @@ struct OrchestratorState { SharedMemoryHeader *sm_header; // === PER-RING RESOURCES === - PTO2RingSet rings[CHIP_MAX_RING_DEPTH]; + ChipRingSet rings[CHIP_MAX_RING_DEPTH]; uint32_t *fanin_seen_epoch[CHIP_MAX_RING_DEPTH]; uint32_t fanin_seen_current_epoch{1}; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index b2fac8e344..af24e51231 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -84,7 +84,7 @@ class PTO2TaskAllocator { * * Production callers leave `initial_local_task_id` at 0: the SM ring * flow-control counters that current_index_ptr / last_alive_ptr point at - * start at zero (PTO2RingFlowControl::init() runs on the AICPU during SM + * start at zero (ChipRingFlowControl::init() runs on the AICPU during SM * reset), so we keep local_task_id_ aligned with that without reading the * SM. Tests that drive SM state directly may pass a non-zero seed to * exercise corner cases like task IDs near INT32_MAX. @@ -809,7 +809,7 @@ struct PTO2DepListPool { * Groups a TaskAllocator and DepPool into one per-depth unit. * CHIP_MAX_RING_DEPTH instances provide independent reclamation per scope depth. */ -struct PTO2RingSet { +struct ChipRingSet { PTO2TaskAllocator task_allocator; PTO2FaninPool fanin_pool; }; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index 0912b8aa53..b1e17c0a43 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -223,7 +223,7 @@ bool SharedMemoryHandle::validate() { return true; } -bool PTO2RingFlowControl::validate(SharedMemoryHandle *handle, int32_t ring_id) const { +bool ChipRingFlowControl::validate(SharedMemoryHandle *handle, int32_t ring_id) const { if (!handle) return false; if (!handle->header) return false; if (ring_id < 0 || ring_id >= CHIP_MAX_RING_DEPTH) return false; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 815bc09875..66fe450524 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -53,7 +53,7 @@ struct SharedMemoryHandle; * Per-ring flow control state in shared memory. * Written/read by Orchestrator and Scheduler for synchronization. */ -struct alignas(64) PTO2RingFlowControl { +struct alignas(64) ChipRingFlowControl { // === Cache Line 0: Written by Orchestrator, Read by Scheduler === alignas(64) std::atomic current_task_index; // Task ring head (next to allocate) @@ -75,7 +75,7 @@ struct alignas(64) PTO2RingFlowControl { bool validate(SharedMemoryHandle *handle, int32_t ring_id) const; }; -static_assert(sizeof(PTO2RingFlowControl) == 128, "PTO2RingFlowControl must be exactly 2 cache lines (128B)"); +static_assert(sizeof(ChipRingFlowControl) == 128, "ChipRingFlowControl must be exactly 2 cache lines (128B)"); /** * Per-ring shared memory header section. @@ -84,7 +84,7 @@ static_assert(sizeof(PTO2RingFlowControl) == 128, "PTO2RingFlowControl must be e * Pointers are host-side only (set by setup_pointers, invalid on device). */ struct alignas(64) SharedMemoryRingHeader { - PTO2RingFlowControl fc; + ChipRingFlowControl fc; // Layout metadata (set once at init) uint64_t task_window_size; @@ -255,21 +255,21 @@ inline SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base, int ring_id) inline std::atomic *ring_current_task_index_addr(void *sm_dev_base, int ring_id) noexcept { return reinterpret_cast *>( reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(SharedMemoryRingHeader, fc) + - offsetof(PTO2RingFlowControl, current_task_index) + offsetof(ChipRingFlowControl, current_task_index) ); } inline std::atomic *ring_last_task_alive_addr(void *sm_dev_base, int ring_id) noexcept { return reinterpret_cast *>( reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(SharedMemoryRingHeader, fc) + - offsetof(PTO2RingFlowControl, last_task_alive) + offsetof(ChipRingFlowControl, last_task_alive) ); } // Byte offsets (from the SM base) of one ring's three segments. The per-ring // layout is: header, then for each ring descriptors -> payloads -> slot_states, // every segment PTO2_ALIGN_UP-padded. -struct PTO2RingSegmentOffsets { +struct ChipRingSegmentOffsets { uint64_t descriptors; uint64_t payloads; uint64_t slot_states; @@ -282,7 +282,7 @@ struct PTO2RingSegmentOffsets { // helpers below (which add `sm_dev_base`). Adding or reordering a per-ring // segment is a one-line edit here; every consumer follows automatically, so the // layout walk can never silently disagree across call sites. -inline PTO2RingSegmentOffsets +inline ChipRingSegmentOffsets ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int ring_id) noexcept { assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "sm_layout: ring_id out of range"); uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); @@ -291,7 +291,7 @@ ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); } - PTO2RingSegmentOffsets o{}; + ChipRingSegmentOffsets o{}; o.descriptors = off; off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); o.payloads = off; diff --git a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp index 50821a6e7f..c59547b2b7 100644 --- a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp @@ -560,7 +560,7 @@ int32_t run_host_orchestration( // each written per task at submit and read only for [0, total_tasks). Zero // only the fixed-size header here; the per-slot segments are initialized in // orch::prepare_task and shipped bounded to total_tasks below. - const sm_layout::PTO2RingSegmentOffsets sm_segs = sm_layout::ring_segment_offsets(eff_task_window_size); + const sm_layout::ChipRingSegmentOffsets sm_segs = sm_layout::ring_segment_offsets(eff_task_window_size); // Over-allocated and rounded up: every segment offset is a multiple of // PTO2_ALIGN_SIZE and ChipTaskSlotState is alignas(64), which a plain // new uint8_t[] does not guarantee. diff --git a/src/a5/runtime/host_build_graph/runtime/ring_buffer.h b/src/a5/runtime/host_build_graph/runtime/ring_buffer.h index 6b96d82441..956b3d3c9a 100644 --- a/src/a5/runtime/host_build_graph/runtime/ring_buffer.h +++ b/src/a5/runtime/host_build_graph/runtime/ring_buffer.h @@ -60,7 +60,7 @@ class PTO2TaskAllocator { * from host code that constructs a prebuilt arena image. * * The ring starts at task id 0, matching the SM flow-control counter that - * current_index_ptr points at (PTO2RingFlowControl::init() runs on the AICPU + * current_index_ptr points at (ChipRingFlowControl::init() runs on the AICPU * during SM reset), so local_task_id_ stays in sync without reading the SM. * Because ids are never reclaimed, alloc() caps them at window_size — they * cannot run away toward INT32_MAX. diff --git a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 91b187b733..7cc4fd7308 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -211,7 +211,7 @@ bool SharedMemoryHandle::validate() { return true; } -bool PTO2RingFlowControl::validate(SharedMemoryHandle *handle) const { +bool ChipRingFlowControl::validate(SharedMemoryHandle *handle) const { if (!handle) return false; if (!handle->header) return false; diff --git a/src/a5/runtime/host_build_graph/runtime/shared_memory.h b/src/a5/runtime/host_build_graph/runtime/shared_memory.h index 3a76a415fa..92b1a2b871 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a5/runtime/host_build_graph/runtime/shared_memory.h @@ -50,7 +50,7 @@ struct SharedMemoryHandle; * Per-ring flow control state in shared memory. * Written/read by Orchestrator and Scheduler for synchronization. */ -struct alignas(64) PTO2RingFlowControl { +struct alignas(64) ChipRingFlowControl { // Written by Orchestrator, read by Scheduler. There is no reverse channel: // the ring is whole-graph-resident, so the scheduler never reclaims task // slots and has nothing to publish back. @@ -67,7 +67,7 @@ struct alignas(64) PTO2RingFlowControl { bool validate(SharedMemoryHandle *handle) const; }; -static_assert(sizeof(PTO2RingFlowControl) == 64, "PTO2RingFlowControl must be exactly one cache line (64B)"); +static_assert(sizeof(ChipRingFlowControl) == 64, "ChipRingFlowControl must be exactly one cache line (64B)"); /** * Per-ring shared memory header section. @@ -76,7 +76,7 @@ static_assert(sizeof(PTO2RingFlowControl) == 64, "PTO2RingFlowControl must be ex * Pointers are host-side only (set by setup_pointers, invalid on device). */ struct alignas(64) SharedMemoryRingHeader { - PTO2RingFlowControl fc; + ChipRingFlowControl fc; // Highest task_id such that every task with id in [0, completed_watermark] // has its completion_flags byte set. Advanced over the full contiguous @@ -293,7 +293,7 @@ inline SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base) noexcept { inline std::atomic *ring_current_task_index_addr(void *sm_dev_base) noexcept { return reinterpret_cast *>( reinterpret_cast(ring_header_addr(sm_dev_base)) + offsetof(SharedMemoryRingHeader, fc) + - offsetof(PTO2RingFlowControl, current_task_index) + offsetof(ChipRingFlowControl, current_task_index) ); } @@ -306,7 +306,7 @@ inline std::atomic *ring_current_task_index_addr(void *sm_dev_base) noe // The pools sit last because nothing on the device resolves a segment past // completion_flags: a payload names its argument regions by delta, so the four // slot-pitched offsets are all the attach path computes. -struct PTO2RingSegmentOffsets { +struct ChipRingSegmentOffsets { uint64_t descriptors; uint64_t payloads; uint64_t slot_states; @@ -342,9 +342,9 @@ inline RingImageExtents mirror_extents(uint64_t task_window_size) noexcept { // `sm_base`) and the device-address helpers below (which add `sm_dev_base`). Adding // or reordering a segment is a one-line edit here; every consumer follows // automatically, so the layout walk can never silently disagree across call sites. -inline PTO2RingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) noexcept { +inline ChipRingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) noexcept { uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); - PTO2RingSegmentOffsets o{}; + ChipRingSegmentOffsets o{}; o.descriptors = off; off += PTO2_ALIGN_UP(e.slots * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); o.payloads = off; @@ -363,7 +363,7 @@ inline PTO2RingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) no return o; } -inline PTO2RingSegmentOffsets ring_segment_offsets(uint64_t task_window_size) noexcept { +inline ChipRingSegmentOffsets ring_segment_offsets(uint64_t task_window_size) noexcept { return ring_segment_offsets(mirror_extents(task_window_size)); } @@ -473,8 +473,8 @@ inline uint64_t compact_live_image( always_assert(used.fanin_elems <= mirror.fanin_elems); always_assert(used.tensor_elems <= mirror.tensor_elems); always_assert(used.scalar_elems <= mirror.scalar_elems); - const PTO2RingSegmentOffsets from = ring_segment_offsets(mirror); - const PTO2RingSegmentOffsets to = ring_segment_offsets(image_extents(used)); + const ChipRingSegmentOffsets from = ring_segment_offsets(mirror); + const ChipRingSegmentOffsets to = ring_segment_offsets(image_extents(used)); // The header and the descriptors offset are pitch-independent, so the header // lands where it already was. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index d92cf4f5ff..ad6f04b56d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -46,14 +46,14 @@ Type changes: ## 4. Data Structures -### 4.1 PTO2RingSet (new) +### 4.1 ChipRingSet (new) Bundles the three per-ring resources into a single aggregate (`ring_buffer.h`): ```cpp -struct PTO2RingSet { - PTO2HeapRing heap_ring; - PTO2TaskRing task_ring; +struct ChipRingSet { + ChipHeapRing heap_ring; + ChipTaskRing task_ring; PTO2FaninPool fanin_pool; }; ``` @@ -62,12 +62,12 @@ struct PTO2RingSet { ```cpp // Before: single ring -PTO2HeapRing heap_ring; -PTO2TaskRing task_ring; +ChipHeapRing heap_ring; +ChipTaskRing task_ring; PTO2DepListPool dep_pool; // After: per-ring array (dep_pool moved to scheduler, see §4.5) -PTO2RingSet rings[CHIP_MAX_RING_DEPTH]; +ChipRingSet rings[CHIP_MAX_RING_DEPTH]; ``` Ring selection: `current_ring_id() = min(scope_stack_top, CHIP_MAX_RING_DEPTH - 1)`. @@ -77,7 +77,7 @@ Ring selection: `current_ring_id() = min(scope_stack_top, CHIP_MAX_RING_DEPTH - Per-ring flow control and per-ring layout info are grouped together: ```cpp -struct PTO2RingFlowControl { +struct ChipRingFlowControl { std::atomic current_task_index; // task ring head std::atomic last_task_alive; // task ring tail std::atomic heap_top; // heap alloc pointer @@ -85,7 +85,7 @@ struct PTO2RingFlowControl { }; struct alignas(64) SharedMemoryRingHeader { - PTO2RingFlowControl fc; + ChipRingFlowControl fc; // Layout metadata (set once at init) uint64_t task_window_size; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index f4f3eb4a7a..2c075a57d8 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -190,7 +190,7 @@ Alignment is 64 bytes (`PTO2_ALIGN_SIZE`). The task ring manages task slot allocation with back-pressure flow control. -**Structure** (`PTO2TaskRing`): +**Structure** (`ChipTaskRing`): - `descriptors`: pointer to `TaskDescriptor[]` in shared memory - `window_size`: number of slots (power of 2) @@ -217,7 +217,7 @@ else: The heap ring manages output buffer allocation from a circular GM heap. -**Structure** (`PTO2HeapRing`): +**Structure** (`ChipHeapRing`): - `base`: GM heap base address - `size`: total heap size (default 1 GB) @@ -431,7 +431,7 @@ The orchestrator runs on AICPU Thread 3 and builds the task graph by calling the Key members: -- `rings[CHIP_MAX_RING_DEPTH]`: per-ring `PTO2RingSet` (HeapRing + TaskRing + FaninPool). See [MULTI_RING.md §4.2](MULTI_RING.md). +- `rings[CHIP_MAX_RING_DEPTH]`: per-ring `ChipRingSet` (HeapRing + TaskRing + FaninPool). See [MULTI_RING.md §4.2](MULTI_RING.md). - `tensor_map`, `tensor_pool`: dependency tracking - `scope_tasks[]`, `scope_begins[]`, `scope_stack_top`: scope nesting stack (flat buffer partitioned by level) - `scheduler`: pointer to scheduler state (for Orch-side wiring helpers and ready queue access) diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index c6be03176c..1dc53becda 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -896,7 +896,7 @@ static TaskOutputTensors submit_task_common( } uint8_t ring_id = prepared.task_id.ring(); SchedulerState *sched = orch->scheduler; - PTO2RingFlowControl &fc = orch->sm_header->rings[ring_id].fc; + ChipRingFlowControl &fc = orch->sm_header->rings[ring_id].fc; TaskId task_id = prepared.task_id; ChipTaskSlotState &cur_slot_state = *prepared.slot_state; TaskDescriptor &task = *prepared.task; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 0b5949dab4..cb91daa77c 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -67,7 +67,7 @@ struct OrchestratorState { SharedMemoryHeader *sm_header; // === PER-RING RESOURCES === - PTO2RingSet rings[CHIP_MAX_RING_DEPTH]; + ChipRingSet rings[CHIP_MAX_RING_DEPTH]; uint32_t *fanin_seen_epoch[CHIP_MAX_RING_DEPTH]; uint32_t fanin_seen_current_epoch{1}; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 59bd39d743..4e4ad1f7f9 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -128,7 +128,7 @@ class PTO2TaskAllocator { * * Production callers leave `initial_local_task_id` at 0: the SM ring * flow-control counters that current_index_ptr / last_alive_ptr point at - * start at zero (PTO2RingFlowControl::init() runs on the AICPU during SM + * start at zero (ChipRingFlowControl::init() runs on the AICPU during SM * reset), so we keep local_task_id_ aligned with that without reading the * SM. Tests that drive SM state directly may pass a non-zero seed to * exercise corner cases like task IDs near INT32_MAX. @@ -923,7 +923,7 @@ struct PTO2DepListPool { * Groups a TaskAllocator and DepPool into one per-depth unit. * CHIP_MAX_RING_DEPTH instances provide independent reclamation per scope depth. */ -struct PTO2RingSet { +struct ChipRingSet { PTO2TaskAllocator task_allocator; PTO2FaninPool fanin_pool; }; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index 0912b8aa53..b1e17c0a43 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -223,7 +223,7 @@ bool SharedMemoryHandle::validate() { return true; } -bool PTO2RingFlowControl::validate(SharedMemoryHandle *handle, int32_t ring_id) const { +bool ChipRingFlowControl::validate(SharedMemoryHandle *handle, int32_t ring_id) const { if (!handle) return false; if (!handle->header) return false; if (ring_id < 0 || ring_id >= CHIP_MAX_RING_DEPTH) return false; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 815bc09875..66fe450524 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -53,7 +53,7 @@ struct SharedMemoryHandle; * Per-ring flow control state in shared memory. * Written/read by Orchestrator and Scheduler for synchronization. */ -struct alignas(64) PTO2RingFlowControl { +struct alignas(64) ChipRingFlowControl { // === Cache Line 0: Written by Orchestrator, Read by Scheduler === alignas(64) std::atomic current_task_index; // Task ring head (next to allocate) @@ -75,7 +75,7 @@ struct alignas(64) PTO2RingFlowControl { bool validate(SharedMemoryHandle *handle, int32_t ring_id) const; }; -static_assert(sizeof(PTO2RingFlowControl) == 128, "PTO2RingFlowControl must be exactly 2 cache lines (128B)"); +static_assert(sizeof(ChipRingFlowControl) == 128, "ChipRingFlowControl must be exactly 2 cache lines (128B)"); /** * Per-ring shared memory header section. @@ -84,7 +84,7 @@ static_assert(sizeof(PTO2RingFlowControl) == 128, "PTO2RingFlowControl must be e * Pointers are host-side only (set by setup_pointers, invalid on device). */ struct alignas(64) SharedMemoryRingHeader { - PTO2RingFlowControl fc; + ChipRingFlowControl fc; // Layout metadata (set once at init) uint64_t task_window_size; @@ -255,21 +255,21 @@ inline SharedMemoryRingHeader *ring_header_addr(void *sm_dev_base, int ring_id) inline std::atomic *ring_current_task_index_addr(void *sm_dev_base, int ring_id) noexcept { return reinterpret_cast *>( reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(SharedMemoryRingHeader, fc) + - offsetof(PTO2RingFlowControl, current_task_index) + offsetof(ChipRingFlowControl, current_task_index) ); } inline std::atomic *ring_last_task_alive_addr(void *sm_dev_base, int ring_id) noexcept { return reinterpret_cast *>( reinterpret_cast(ring_header_addr(sm_dev_base, ring_id)) + offsetof(SharedMemoryRingHeader, fc) + - offsetof(PTO2RingFlowControl, last_task_alive) + offsetof(ChipRingFlowControl, last_task_alive) ); } // Byte offsets (from the SM base) of one ring's three segments. The per-ring // layout is: header, then for each ring descriptors -> payloads -> slot_states, // every segment PTO2_ALIGN_UP-padded. -struct PTO2RingSegmentOffsets { +struct ChipRingSegmentOffsets { uint64_t descriptors; uint64_t payloads; uint64_t slot_states; @@ -282,7 +282,7 @@ struct PTO2RingSegmentOffsets { // helpers below (which add `sm_dev_base`). Adding or reordering a per-ring // segment is a one-line edit here; every consumer follows automatically, so the // layout walk can never silently disagree across call sites. -inline PTO2RingSegmentOffsets +inline ChipRingSegmentOffsets ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int ring_id) noexcept { assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "sm_layout: ring_id out of range"); uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); @@ -291,7 +291,7 @@ ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); } - PTO2RingSegmentOffsets o{}; + ChipRingSegmentOffsets o{}; o.descriptors = off; off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); o.payloads = off; diff --git a/tests/ut/cpp/a2a3/test_task_allocator.cpp b/tests/ut/cpp/a2a3/test_task_allocator.cpp index b693bb4c1a..7ad6d26f16 100644 --- a/tests/ut/cpp/a2a3/test_task_allocator.cpp +++ b/tests/ut/cpp/a2a3/test_task_allocator.cpp @@ -16,7 +16,7 @@ * * The allocator is single-threaded (orchestrator thread), so no concurrency * tests are needed. The unified PTO2TaskAllocator replaces the previous - * separate PTO2HeapRing + PTO2TaskRing. + * separate ChipHeapRing + ChipTaskRing. * * Design contracts (try_bump_heap): * diff --git a/tests/ut/cpp/a5/test_task_allocator.cpp b/tests/ut/cpp/a5/test_task_allocator.cpp index b693bb4c1a..7ad6d26f16 100644 --- a/tests/ut/cpp/a5/test_task_allocator.cpp +++ b/tests/ut/cpp/a5/test_task_allocator.cpp @@ -16,7 +16,7 @@ * * The allocator is single-threaded (orchestrator thread), so no concurrency * tests are needed. The unified PTO2TaskAllocator replaces the previous - * separate PTO2HeapRing + PTO2TaskRing. + * separate ChipHeapRing + ChipTaskRing. * * Design contracts (try_bump_heap): * diff --git a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp index cfb4d389ea..d9152017f0 100644 --- a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp +++ b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp @@ -192,7 +192,7 @@ struct Compacted { bytes = sm_layout::compact_live_image(mirror.base(), WINDOW, used, rebase, image.base()); } - sm_layout::PTO2RingSegmentOffsets off(uint64_t submitted = SUBMITTED) const { + sm_layout::ChipRingSegmentOffsets off(uint64_t submitted = SUBMITTED) const { return sm_layout::ring_segment_offsets(sm_layout::image_extents(usage_for(submitted))); } From c82420db59877e4eaa950adb6559d14bad8d7a30 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:23:49 -0700 Subject: [PATCH 10/22] Refactor: retire the PTO2 prefix on the task allocator `PTO2TaskAllocator` -> `TaskAllocator`, `PTO2TaskAllocResult` -> `TaskAllocResult`, `PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES` -> `ALLOC_DEADLOCK_TIMEOUT_CYCLES`. 84 occurrences across 33 files. All three are plain strips, and `TaskAllocResult` is the one worth naming: the host orchestrator has a `struct AllocResult` in `src/common/hierarchical/ring.h:86`, so dropping `Task` as well as the prefix would have collided. Keeping it distinguishes the chip runtime's ring allocation from the host's heap-slab allocation without needing the `Chip` prefix. The timeout is the reclaim backstop that allocator enforces, so it moves with it. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/rules/running-onboard.md | 2 +- ...2026-08-hbg-graph-definition-single-upload.md | 2 +- .../host_build_graph/runtime/orchestrator.h | 2 +- .../runtime/orchestrator_core/orchestrator.cpp | 4 ++-- .../runtime/orchestrator_core/ring_buffer.cpp | 2 +- .../host_build_graph/runtime/ring_buffer.h | 4 ++-- .../host_build_graph/runtime/runtime_types.h | 9 ++++----- .../host_build_graph/runtime/shared_memory.h | 4 ++-- .../docs/RUNTIME_LOGIC.md | 8 ++++---- .../runtime/orchestrator.cpp | 6 +++--- .../runtime/ring_buffer.cpp | 4 ++-- .../runtime/ring_buffer.h | 16 ++++++++-------- .../runtime/runtime_types.h | 7 +++---- .../runtime/shared_memory.h | 4 ++-- .../host_build_graph/runtime/orchestrator.h | 2 +- .../runtime/orchestrator_core/orchestrator.cpp | 4 ++-- .../runtime/orchestrator_core/ring_buffer.cpp | 2 +- .../host_build_graph/runtime/ring_buffer.h | 4 ++-- .../host_build_graph/runtime/runtime_types.h | 9 ++++----- .../host_build_graph/runtime/shared_memory.h | 4 ++-- .../docs/RUNTIME_LOGIC.md | 8 ++++---- .../runtime/orchestrator.cpp | 6 +++--- .../runtime/ring_buffer.cpp | 4 ++-- .../runtime/ring_buffer.h | 16 ++++++++-------- .../runtime/runtime_types.h | 7 +++---- .../runtime/shared_memory.h | 4 ++-- src/common/hierarchical/ring.h | 2 +- .../host_build_graph/docs/GRAPH_EXECUTION.md | 4 ++-- tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp | 10 +++++----- tests/ut/cpp/a2a3/test_task_allocator.cpp | 6 +++--- tests/ut/cpp/a5/test_task_allocator.cpp | 6 +++--- tests/ut/cpp/a5/test_wiring.cpp | 8 ++++---- .../cpp/common/test_hbg_graph_submit_failure.cpp | 2 +- 33 files changed, 89 insertions(+), 93 deletions(-) diff --git a/.claude/rules/running-onboard.md b/.claude/rules/running-onboard.md index f5ab090a32..19ef3a1e97 100644 --- a/.claude/rules/running-onboard.md +++ b/.claude/rules/running-onboard.md @@ -174,7 +174,7 @@ for the signature that actually fired:** | -------------------- | --------- | ---- | | `FATAL: Task Allocator Deadlock` / `FATAL: Dependency Pool Deadlock` / `FATAL: Fanin Spill Pool Deadlock` | an allocator or pool could not reclaim enough space | This header identifies the blocked resource, not the root cause. Classify it using the structural/timeout line that follows. | | `Provable head-of-line deadlock` | **proven open-scope structural deadlock** | TRB only: the reclaim head is the oldest task owned by an open scope on that ring. The blocked orchestrator cannot end that scope, so the head cannot become consumed. On A5, classification waits for at least 10 ms without reclaim progress and an exact-watermark publication acknowledgment. | -| `No reclaim progress for ~500 ms` / `cannot reclaim space after ~500 ms` | allocator/pool **reclaim timeout** | TRB only — the 500ms backstop (`PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES`) lives in its reclaiming allocators. It proves prolonged lack of reclaim progress, not why progress stopped; check capacity, the dumped head, consumers and scheduler state. | +| `No reclaim progress for ~500 ms` / `cannot reclaim space after ~500 ms` | allocator/pool **reclaim timeout** | TRB only — the 500ms backstop (`ALLOC_DEADLOCK_TIMEOUT_CYCLES`) lives in its reclaiming allocators. It proves prolonged lack of reclaim progress, not why progress stopped; check capacity, the dumped head, consumers and scheduler state. | | `Task Window Exhausted` / `Fanin Capacity Exhausted` / `TensorMap Entry Pool Exhausted` | **capacity**, HBG only | HBG is whole-graph-resident and reclaims nothing mid-run, so this is a sizing verdict reached immediately, not a stall. The line carries used/capacity and the requested amount; raise `PTO2_RING_TASK_WINDOW` / `CHIP_TENSORMAP_POOL_SIZE` or shrink the graph. Inline fanin is hard-capped at `PTO2_MAX_FANIN=128`; HBG has no `PTO2_RING_DEP_POOL`. HBG's graph heap has no knob and cannot exhaust: it is committed after orchestration at the measured size, so a graph too large for the device fails host-side at that commit. | | `Timeout (N cycles): producer/consumers ...` | **SPIN** wait on a specific producer/consumer | `runtime_core.cpp`. | | `HandleTaskTimeout` / `kill aicpu-sd` | **OS op-execute timeout** | STARS/tsdaemon, default 45s (`PLATFORM_OP_EXECUTE_TIMEOUT_US`). **A 45s kill ≠ deadlock** — the op was merely long or stalled. Raise this constant to measure true on-device duration. | diff --git a/docs/investigations/2026-08-hbg-graph-definition-single-upload.md b/docs/investigations/2026-08-hbg-graph-definition-single-upload.md index d170c7bdb7..a89d2f5795 100644 --- a/docs/investigations/2026-08-hbg-graph-definition-single-upload.md +++ b/docs/investigations/2026-08-hbg-graph-definition-single-upload.md @@ -136,7 +136,7 @@ cost is excluded. Worth doing, but it is not where the 12 ms lives. **Where the 12 ms does live.** The one-time 12.19 ms is the 53 MB of execution-storage allocation and zeroing. Its lifetime is exactly that of the outer GRAPH task's packed output buffer, so it can come from the same -`PTO2TaskAllocator::alloc` call as the outputs instead of a separate retained +`TaskAllocator::alloc` call as the outputs instead of a separate retained `rtMalloc` — removing both the allocations and the memsets, and making cold start converge with steady state. That is a separate change; this amendment only records why it, and not batching, is the one that moves the 12 ms. diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h index 245dc41ffb..f56aa75cfd 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h @@ -62,7 +62,7 @@ struct OrchestratorState { // === TASK / HEAP ALLOCATION === // hbg is single-ring, so one allocator covers the whole graph. - PTO2TaskAllocator task_allocator; + TaskAllocator task_allocator; std::unique_ptr fanin_seen_epoch; uint32_t fanin_seen_current_epoch{1}; diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index e5609a1825..95a30969e8 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1218,7 +1218,7 @@ static bool append_fanin_or_fail( struct PreparedTask { TaskId task_id = TaskId::invalid(); - PTO2TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; + TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; TaskDescriptor *task = nullptr; TaskPayload *payload = nullptr; ChipTaskSlotState *slot_state = nullptr; @@ -1790,7 +1790,7 @@ bool graph_submit_outer( }; const int32_t tensormap_needed = count_registrable_outputs(boundary_inputs, orch->in_manual_scope()); if (tensormap_needed > 0 && !ensure_tensormap_capacity(orch, tensormap_needed)) return false; - const PTO2TaskAllocResult allocation = allocator.alloc(defer_heap ? 0 : owned_heap); + const TaskAllocResult allocation = allocator.alloc(defer_heap ? 0 : owned_heap); if (allocation.failed()) { orch_mark_fatal(orch, SIMPLER_ERROR_HEAP_RING_DEADLOCK); return false; diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp index b0fa706558..b321cedce4 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp @@ -13,5 +13,5 @@ // dependencies are position-independent local-id integers on the payload and // readiness is derived from the ring completion_flags. The pool ring buffers // (PTO2FaninPool / PTO2DepListPool) that once lived here are gone, so this -// translation unit carries no definitions. PTO2TaskAllocator's methods are +// translation unit carries no definitions. TaskAllocator's methods are // defined inline in ring_buffer.h. diff --git a/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h b/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h index 956b3d3c9a..3bbe149f71 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h +++ b/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h @@ -50,7 +50,7 @@ * request that does not fit can never become satisfiable by waiting — alloc() * reports the exhausted resource and fails on the spot. */ -class PTO2TaskAllocator { +class TaskAllocator { public: /** * Initialize the allocator with task ring and heap ring resources. @@ -100,7 +100,7 @@ class PTO2TaskAllocator { * @param output_size Total packed output size in bytes (0 = no heap needed) * @return Allocation result; check failed() for errors */ - PTO2TaskAllocResult alloc(int32_t output_size) { + TaskAllocResult alloc(int32_t output_size) { uint64_t aligned_size = output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index 5d74582380..b250e2abf2 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -105,7 +105,7 @@ inline constexpr uint64_t HEAP_VIRTUAL_BASE = 1ULL << 62; // graph_classify_tensor can tell an internal producer's output from a boundary // tensor by address-range containment alone, and the Definition stores them as // offsets. That classification is only sound while the range is disjoint from -// every graph-heap address, which PTO2TaskAllocator::init() asserts, and from +// every graph-heap address, which TaskAllocator::init() asserts, and from // every real device address, which the two asserts in the host's bind path // (the acquired heap base, and each caller tensor as it enters device_args) // keep below HEAP_VIRTUAL_BASE. @@ -191,7 +191,7 @@ typedef enum { /** * Result of a unified task allocation. */ -struct PTO2TaskAllocResult { +struct TaskAllocResult { int32_t task_id; // Absolute task ID (not wrapped) int32_t slot; // task_id & (window_size - 1) void *packed_base; // Heap allocation result (nullptr if failure) @@ -428,9 +428,8 @@ struct TaskPayload { * @param args Task arguments (tensors + scalars) * @param result Materialized output tensors (from TensorCreateInfo path) */ - void init( - const CoreTaskArgs &args, TaskOutputTensors &result, PTO2TaskAllocResult &alloc_result, PTO2OutputLayout &layout - ) { + void + init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, PTO2OutputLayout &layout) { tensor_count = args.tensor_count(); scalar_count = args.scalar_count(); diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h index 92b1a2b871..8eddaf82c4 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h @@ -56,11 +56,11 @@ struct alignas(64) ChipRingFlowControl { // slots and has nothing to publish back. alignas(64) std::atomic current_task_index; // Task ring head (next to allocate) - // Per-boot SM reset. PTO2TaskAllocator::init() seeds its private + // Per-boot SM reset. TaskAllocator::init() seeds its private // local_task_id_ to 0 *without* dereferencing current_task_index — it // relies on this reset running on every AICPU boot so 0 stays in sync. If // you ever change the initial fc value or the boot ordering, update - // PTO2TaskAllocator::init (ring_buffer.h) in the same change, or + // TaskAllocator::init (ring_buffer.h) in the same change, or // submit IDs will be off by the divergence. void init() { current_task_index.store(0, std::memory_order_relaxed); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index d5abcd1527..bc9281ed68 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -198,7 +198,7 @@ The task ring manages task slot allocation with back-pressure flow control. **Slot mapping**: `slot = task_id & (window_size - 1)` -**Allocation** (`PTO2TaskAllocator::alloc`): +**Allocation** (`TaskAllocator::alloc`): ```text active_count = current_index - *last_alive_ptr @@ -244,9 +244,9 @@ A simple bump allocator for `PTO2DepListEntry` nodes used in fanin/fanout linked The ring buffer mechanism provides **flow control** between the orchestrator (producer) and the scheduler (consumer). When a ring is exhausted, the orchestrator **blocks** — it cannot submit new tasks or allocate more output memory until the scheduler reclaims slots/space by advancing the watermarks. -**Task Ring back-pressure**: When `active_count = current_index - last_task_alive >= window_size - 1`, `PTO2TaskAllocator::alloc` spin-waits until the scheduler completes tasks and advances `last_task_alive`. +**Task Ring back-pressure**: When `active_count = current_index - last_task_alive >= window_size - 1`, `TaskAllocator::alloc` spin-waits until the scheduler completes tasks and advances `last_task_alive`. -**Heap Ring back-pressure**: When the heap has insufficient contiguous space, `PTO2TaskAllocator::alloc` spin-waits until the scheduler advances `heap_tail` past completed tasks' output buffers. +**Heap Ring back-pressure**: When the heap has insufficient contiguous space, `TaskAllocator::alloc` spin-waits until the scheduler advances `heap_tail` past completed tasks' output buffers. **TensorMap pool back-pressure**: Before STEP 4 registers a task's outputs, the orchestrator's `ensure_tensormap_capacity` reserves pool space for the inserts. When the shared entry pool is exhausted, it reclaims retired entries across all rings and spin-waits until reclaim actually frees entries, with a 500 ms wall-clock deadlock backstop (see Section 5.4). @@ -432,7 +432,7 @@ Key members: | Step | Operation | | ---- | --------- | | 0 | `ChipTensorMap::sync_tensormap` — prune stale TensorMap entries | -| 1 | `PTO2TaskAllocator::alloc` — allocate task slot (may block on flow control) | +| 1 | `TaskAllocator::alloc` — allocate task slot (may block on flow control) | | 2 | Initialize task descriptor + slot state, copy parameters | | 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `PTO2FaninBuilder` | | 4 | **Insert**: register OUTPUT/INOUT args in TensorMap | diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index e6e8cd2e89..2335788894 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -539,7 +539,7 @@ static void scope_tasks_push(OrchestratorState *orch, ChipTaskSlotState *task_sl struct PreparedTask { TaskId task_id = TaskId::invalid(); - PTO2TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; + TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; TaskDescriptor *task = nullptr; TaskPayload *payload = nullptr; ChipTaskSlotState *slot_state = nullptr; @@ -559,7 +559,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { return layout; } -static bool check_scope_can_accept_task(OrchestratorState *orch, PTO2TaskAllocator &allocator, uint8_t ring_id) { +static bool check_scope_can_accept_task(OrchestratorState *orch, TaskAllocator &allocator, uint8_t ring_id) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit task outside a scope"); int32_t scope_task_count = orch->scope_tasks_size - orch->scope_begins[orch->scope_stack_top]; @@ -852,7 +852,7 @@ static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { if (!block_timing) { block_cycle0 = now; block_timing = true; - } else if (now - block_cycle0 >= PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES) { + } else if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { LOG_ERROR("========================================"); LOG_ERROR("FATAL: TensorMap Entry Pool Deadlock Detected!"); LOG_ERROR("========================================"); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index d6e1ae427a..ff725df2d1 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -85,7 +85,7 @@ bool PTO2FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { if (!block_timing) { block_cycle0 = now; block_timing = true; - } else if (now - block_cycle0 >= PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES) { + } else if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { int32_t current = ring.fc.current_task_index.load(std::memory_order_acquire); LOG_ERROR("========================================"); LOG_ERROR("FATAL: Fanin Spill Pool Deadlock Detected!"); @@ -217,7 +217,7 @@ bool PTO2DepListPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed, if (!block_timing) { block_cycle0 = now; block_timing = true; - } else if (now - block_cycle0 >= PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES) { + } else if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { report_deadlock(ring, needed, cur_last_alive, /*scope_gated=*/false); latch_pool_error(error_code_ptr, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index af24e51231..753ff2ec71 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -57,7 +57,7 @@ // oldest task owned by an open scope on the blocked ring. This wall-clock value // is the backstop for all other cases; it is an ABSOLUTE TIME (not a spin // count), so it is stable across chips/contention. -#define PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES (PLATFORM_PROF_SYS_CNT_FREQ / 2) // 500 ms +#define ALLOC_DEADLOCK_TIMEOUT_CYCLES (PLATFORM_PROF_SYS_CNT_FREQ / 2) // 500 ms // ============================================================================= // Task Allocator (unified task slot + heap buffer allocation) @@ -73,7 +73,7 @@ * The alloc() method checks both resources BEFORE committing to either, * eliminating the need for rollback on partial failure. */ -class PTO2TaskAllocator { +class TaskAllocator { public: /** * Initialize the allocator with task ring and heap ring resources. @@ -122,7 +122,7 @@ class PTO2TaskAllocator { * @param oldest_open_task Oldest task owned by any open scope on this ring * @return Allocation result; check failed() for errors */ - PTO2TaskAllocResult alloc(int32_t output_size, ChipTaskSlotState *oldest_open_task = nullptr) { + TaskAllocResult alloc(int32_t output_size, ChipTaskSlotState *oldest_open_task = nullptr) { uint64_t aligned_size = output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; @@ -196,7 +196,7 @@ class PTO2TaskAllocator { if (!block_timing) { block_cycle0 = now; block_timing = true; - } else if (now - block_cycle0 >= PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES) { + } else if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { report_deadlock(output_size, blocked_on_heap, /*scope_gated=*/false); return {-1, -1, nullptr, nullptr}; } @@ -435,7 +435,7 @@ class PTO2TaskAllocator { } else { LOG_ERROR( "No reclaim progress for ~500 ms (%" PRIu64 " cycles wall clock).", - (uint64_t)PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES + (uint64_t)ALLOC_DEADLOCK_TIMEOUT_CYCLES ); } LOG_ERROR( @@ -725,13 +725,13 @@ struct PTO2DepListPool { * Ensure dep pool for a specific ring has at least `needed` entries available. * Spin-waits for reclamation under pressure. The dep pool shares * last_task_alive with the heap and task rings, so it detects a wedged - * reclaim watermark the same way PTO2TaskAllocator::alloc does: a structural + * reclaim watermark the same way TaskAllocator::alloc does: a structural * head-of-line check plus a wall-clock backstop, each emitting report_deadlock. */ bool ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task = nullptr); /** - * Structured dep-pool deadlock report, mirroring PTO2TaskAllocator::report_deadlock. + * Structured dep-pool deadlock report, mirroring TaskAllocator::report_deadlock. * scope_gated marks the provable head-of-line case where the head is pinned * by an open scope on this ring, as opposed to the wall-clock backstop. */ @@ -810,7 +810,7 @@ struct PTO2DepListPool { * CHIP_MAX_RING_DEPTH instances provide independent reclamation per scope depth. */ struct ChipRingSet { - PTO2TaskAllocator task_allocator; + TaskAllocator task_allocator; PTO2FaninPool fanin_pool; }; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 7fb88fde8d..62502d458d 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -143,7 +143,7 @@ typedef enum { /** * Result of a unified task allocation. */ -struct PTO2TaskAllocResult { +struct TaskAllocResult { int32_t task_id; // Absolute task ID (not wrapped) int32_t slot; // task_id & (window_size - 1) void *packed_base; // Heap allocation result (nullptr if failure) @@ -374,9 +374,8 @@ struct TaskPayload { * @param args Task arguments (tensors + scalars) * @param result Materialized output tensors (from TensorCreateInfo path) */ - void init( - const CoreTaskArgs &args, TaskOutputTensors &result, PTO2TaskAllocResult &alloc_result, PTO2OutputLayout &layout - ) { + void + init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, PTO2OutputLayout &layout) { tensor_count = args.tensor_count(); scalar_count = args.scalar_count(); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 66fe450524..8a8692acf7 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -60,12 +60,12 @@ struct alignas(64) ChipRingFlowControl { // === Cache Line 1: Written by Scheduler, Read by Orchestrator (for back-pressure) === alignas(64) std::atomic last_task_alive; // Task ring tail (oldest active task) - // Per-boot SM reset. PTO2TaskAllocator::init() seeds its private + // Per-boot SM reset. TaskAllocator::init() seeds its private // local_task_id_ from initial_local_task_id (default 0 in production) // *without* dereferencing current_task_index — it relies on this reset // running on every AICPU boot so 0 stays in sync. If you ever change // the initial fc value or the boot ordering, update the default in - // PTO2TaskAllocator::init (ring_buffer.h) in the same change, or + // TaskAllocator::init (ring_buffer.h) in the same change, or // submit IDs will be off by the divergence. void init() { current_task_index.store(0, std::memory_order_relaxed); diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator.h b/src/a5/runtime/host_build_graph/runtime/orchestrator.h index ccf5d15098..a256314067 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator.h @@ -62,7 +62,7 @@ struct OrchestratorState { // === TASK / HEAP ALLOCATION === // hbg is single-ring, so one allocator covers the whole graph. - PTO2TaskAllocator task_allocator; + TaskAllocator task_allocator; std::unique_ptr fanin_seen_epoch; uint32_t fanin_seen_current_epoch{1}; diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index e5609a1825..95a30969e8 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1218,7 +1218,7 @@ static bool append_fanin_or_fail( struct PreparedTask { TaskId task_id = TaskId::invalid(); - PTO2TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; + TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; TaskDescriptor *task = nullptr; TaskPayload *payload = nullptr; ChipTaskSlotState *slot_state = nullptr; @@ -1790,7 +1790,7 @@ bool graph_submit_outer( }; const int32_t tensormap_needed = count_registrable_outputs(boundary_inputs, orch->in_manual_scope()); if (tensormap_needed > 0 && !ensure_tensormap_capacity(orch, tensormap_needed)) return false; - const PTO2TaskAllocResult allocation = allocator.alloc(defer_heap ? 0 : owned_heap); + const TaskAllocResult allocation = allocator.alloc(defer_heap ? 0 : owned_heap); if (allocation.failed()) { orch_mark_fatal(orch, SIMPLER_ERROR_HEAP_RING_DEADLOCK); return false; diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp index b0fa706558..b321cedce4 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp @@ -13,5 +13,5 @@ // dependencies are position-independent local-id integers on the payload and // readiness is derived from the ring completion_flags. The pool ring buffers // (PTO2FaninPool / PTO2DepListPool) that once lived here are gone, so this -// translation unit carries no definitions. PTO2TaskAllocator's methods are +// translation unit carries no definitions. TaskAllocator's methods are // defined inline in ring_buffer.h. diff --git a/src/a5/runtime/host_build_graph/runtime/ring_buffer.h b/src/a5/runtime/host_build_graph/runtime/ring_buffer.h index 956b3d3c9a..3bbe149f71 100644 --- a/src/a5/runtime/host_build_graph/runtime/ring_buffer.h +++ b/src/a5/runtime/host_build_graph/runtime/ring_buffer.h @@ -50,7 +50,7 @@ * request that does not fit can never become satisfiable by waiting — alloc() * reports the exhausted resource and fails on the spot. */ -class PTO2TaskAllocator { +class TaskAllocator { public: /** * Initialize the allocator with task ring and heap ring resources. @@ -100,7 +100,7 @@ class PTO2TaskAllocator { * @param output_size Total packed output size in bytes (0 = no heap needed) * @return Allocation result; check failed() for errors */ - PTO2TaskAllocResult alloc(int32_t output_size) { + TaskAllocResult alloc(int32_t output_size) { uint64_t aligned_size = output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index cffb8ea6c0..039cf02c73 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -106,7 +106,7 @@ inline constexpr uint64_t HEAP_VIRTUAL_BASE = 1ULL << 62; // graph_classify_tensor can tell an internal producer's output from a boundary // tensor by address-range containment alone, and the Definition stores them as // offsets. That classification is only sound while the range is disjoint from -// every graph-heap address, which PTO2TaskAllocator::init() asserts, and from +// every graph-heap address, which TaskAllocator::init() asserts, and from // every real device address, which the two asserts in the host's bind path // (the acquired heap base, and each caller tensor as it enters device_args) // keep below HEAP_VIRTUAL_BASE. @@ -192,7 +192,7 @@ typedef enum { /** * Result of a unified task allocation. */ -struct PTO2TaskAllocResult { +struct TaskAllocResult { int32_t task_id; // Absolute task ID (not wrapped) int32_t slot; // task_id & (window_size - 1) void *packed_base; // Heap allocation result (nullptr if failure) @@ -429,9 +429,8 @@ struct TaskPayload { * @param args Task arguments (tensors + scalars) * @param result Materialized output tensors (from TensorCreateInfo path) */ - void init( - const CoreTaskArgs &args, TaskOutputTensors &result, PTO2TaskAllocResult &alloc_result, PTO2OutputLayout &layout - ) { + void + init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, PTO2OutputLayout &layout) { tensor_count = args.tensor_count(); scalar_count = args.scalar_count(); diff --git a/src/a5/runtime/host_build_graph/runtime/shared_memory.h b/src/a5/runtime/host_build_graph/runtime/shared_memory.h index 92b1a2b871..8eddaf82c4 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a5/runtime/host_build_graph/runtime/shared_memory.h @@ -56,11 +56,11 @@ struct alignas(64) ChipRingFlowControl { // slots and has nothing to publish back. alignas(64) std::atomic current_task_index; // Task ring head (next to allocate) - // Per-boot SM reset. PTO2TaskAllocator::init() seeds its private + // Per-boot SM reset. TaskAllocator::init() seeds its private // local_task_id_ to 0 *without* dereferencing current_task_index — it // relies on this reset running on every AICPU boot so 0 stays in sync. If // you ever change the initial fc value or the boot ordering, update - // PTO2TaskAllocator::init (ring_buffer.h) in the same change, or + // TaskAllocator::init (ring_buffer.h) in the same change, or // submit IDs will be off by the divergence. void init() { current_task_index.store(0, std::memory_order_relaxed); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 2c075a57d8..d41763eb91 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -199,7 +199,7 @@ The task ring manages task slot allocation with back-pressure flow control. **Slot mapping**: `slot = task_id & (window_size - 1)` -**Allocation** (`PTO2TaskAllocator::alloc`): +**Allocation** (`TaskAllocator::alloc`): ```text active_count = current_index - *last_alive_ptr @@ -245,9 +245,9 @@ A simple bump allocator for `PTO2DepListEntry` nodes used in fanin/fanout linked The ring buffer mechanism provides **flow control** between the orchestrator (producer) and the scheduler (consumer). When a ring is exhausted, the orchestrator **blocks** — it cannot submit new tasks or allocate more output memory until the scheduler reclaims slots/space by advancing the watermarks. -**Task Ring back-pressure**: When `active_count = current_index - last_task_alive >= window_size - 1`, `PTO2TaskAllocator::alloc` spin-waits until the scheduler completes tasks and advances `last_task_alive`. +**Task Ring back-pressure**: When `active_count = current_index - last_task_alive >= window_size - 1`, `TaskAllocator::alloc` spin-waits until the scheduler completes tasks and advances `last_task_alive`. -**Heap Ring back-pressure**: When the heap has insufficient contiguous space, `PTO2TaskAllocator::alloc` spin-waits until the scheduler advances `heap_tail` past completed tasks' output buffers. +**Heap Ring back-pressure**: When the heap has insufficient contiguous space, `TaskAllocator::alloc` spin-waits until the scheduler advances `heap_tail` past completed tasks' output buffers. **TensorMap pool back-pressure**: Before STEP 4 registers a task's outputs, the orchestrator's `ensure_tensormap_capacity` reserves pool space for the inserts. When the shared entry pool is exhausted, it reclaims retired entries across all rings and spin-waits until reclaim actually frees entries, with a 500 ms wall-clock deadlock backstop (see Section 5.4). @@ -442,7 +442,7 @@ Key members: | Step | Operation | | ---- | --------- | | 0 | `ChipTensorMap::sync_tensormap` — prune stale TensorMap entries | -| 1 | `PTO2TaskAllocator::alloc` — allocate task slot (may block on flow control) | +| 1 | `TaskAllocator::alloc` — allocate task slot (may block on flow control) | | 2 | Initialize task descriptor + slot state, copy parameters | | 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `PTO2FaninBuilder` | | 4 | **Insert**: register OUTPUT/INOUT args in TensorMap | diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 1dc53becda..0f13084ebe 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -531,7 +531,7 @@ static void scope_tasks_push(OrchestratorState *orch, ChipTaskSlotState *task_sl struct PreparedTask { TaskId task_id = TaskId::invalid(); - PTO2TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; + TaskAllocResult alloc_result = {-1, 0, nullptr, nullptr}; TaskDescriptor *task = nullptr; TaskPayload *payload = nullptr; ChipTaskSlotState *slot_state = nullptr; @@ -551,7 +551,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { return layout; } -static bool check_scope_can_accept_task(OrchestratorState *orch, PTO2TaskAllocator &allocator, uint8_t ring_id) { +static bool check_scope_can_accept_task(OrchestratorState *orch, TaskAllocator &allocator, uint8_t ring_id) { always_assert(orch->scope_stack_top >= 0 && "Cannot submit task outside a scope"); int32_t scope_task_count = orch->scope_tasks_size - orch->scope_begins[orch->scope_stack_top]; @@ -854,7 +854,7 @@ static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { } else if (now - block_cycle0 >= PTO2_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { request_reclaim_publication(); } - if (now - block_cycle0 >= PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES) { + if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { LOG_ERROR("========================================"); LOG_ERROR("FATAL: TensorMap Entry Pool Deadlock Detected!"); LOG_ERROR("========================================"); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index 60d35b05c4..5534a7867b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -97,7 +97,7 @@ bool PTO2FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { } else if (!watermark_synchronized && now - block_cycle0 >= PTO2_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { publication_request.request(); } - if (now - block_cycle0 >= PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES) { + if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { int32_t current = ring.fc.current_task_index.load(std::memory_order_acquire); LOG_ERROR("========================================"); LOG_ERROR("FATAL: Fanin Spill Pool Deadlock Detected!"); @@ -244,7 +244,7 @@ bool PTO2DepListPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed, // (2) Wall-clock backstop for the residual case the head test cannot // prove. Absolute time (get_sys_cnt_aicpu is an MMIO read), sampled // once per 1024 spins. - if (now - block_cycle0 >= PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES) { + if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { report_deadlock(ring, needed, cur_last_alive, /*scope_gated=*/false); latch_pool_error(error_code_ptr, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 4e4ad1f7f9..1bdab7a9e1 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -60,7 +60,7 @@ // oldest task owned by an open scope on the blocked ring. This wall-clock value // is the backstop for all other cases; it is an ABSOLUTE TIME (not a spin // count), so it is stable across chips/contention. -#define PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES (PLATFORM_PROF_SYS_CNT_FREQ / 2) // 500 ms +#define ALLOC_DEADLOCK_TIMEOUT_CYCLES (PLATFORM_PROF_SYS_CNT_FREQ / 2) // 500 ms constexpr uint32_t ring_mask_bit(int32_t ring_id) { static_assert(CHIP_MAX_RING_DEPTH <= 32, "ring masks use one uint32_t bit per ring"); @@ -117,7 +117,7 @@ class ReclaimPublicationRequest { * The alloc() method checks both resources BEFORE committing to either, * eliminating the need for rollback on partial failure. */ -class PTO2TaskAllocator { +class TaskAllocator { public: /** * Initialize the allocator with task ring and heap ring resources. @@ -179,7 +179,7 @@ class PTO2TaskAllocator { * @param oldest_open_task Oldest task owned by any open scope on this ring * @return Allocation result; check failed() for errors */ - PTO2TaskAllocResult alloc(int32_t output_size, ChipTaskSlotState *oldest_open_task = nullptr) { + TaskAllocResult alloc(int32_t output_size, ChipTaskSlotState *oldest_open_task = nullptr) { uint64_t aligned_size = output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; @@ -267,7 +267,7 @@ class PTO2TaskAllocator { // (2) Wall-clock backstop for the residual case the local head // test can't prove (e.g. a closed sibling whose consumer is // deferred). Absolute time, not a spin count. - if (now - block_cycle0 >= PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES) { + if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { report_deadlock(output_size, blocked_on_heap, /*scope_gated=*/false); return {-1, -1, nullptr, nullptr}; } @@ -508,7 +508,7 @@ class PTO2TaskAllocator { } else { LOG_ERROR( "No reclaim progress for ~500 ms (%" PRIu64 " cycles wall clock).", - (uint64_t)PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES + (uint64_t)ALLOC_DEADLOCK_TIMEOUT_CYCLES ); } LOG_ERROR( @@ -838,14 +838,14 @@ struct PTO2DepListPool { * Ensure dep pool for a specific ring has at least `needed` entries available. * Spin-waits for reclamation under pressure. The dep pool shares * last_task_alive with the heap and task rings, so it detects a wedged - * reclaim watermark the same way PTO2TaskAllocator::alloc does: request an + * reclaim watermark the same way TaskAllocator::alloc does: request an * exact publication, then use a structural head-of-line check plus a * wall-clock backstop, each emitting report_deadlock. */ bool ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task = nullptr); /** - * Structured dep-pool deadlock report, mirroring PTO2TaskAllocator::report_deadlock. + * Structured dep-pool deadlock report, mirroring TaskAllocator::report_deadlock. * scope_gated marks the provable head-of-line case where the head is pinned * by an open scope on this ring, as opposed to the wall-clock backstop. */ @@ -924,7 +924,7 @@ struct PTO2DepListPool { * CHIP_MAX_RING_DEPTH instances provide independent reclamation per scope depth. */ struct ChipRingSet { - PTO2TaskAllocator task_allocator; + TaskAllocator task_allocator; PTO2FaninPool fanin_pool; }; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index d07f6273c8..2eebda93da 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -143,7 +143,7 @@ typedef enum { /** * Result of a unified task allocation. */ -struct PTO2TaskAllocResult { +struct TaskAllocResult { int32_t task_id; // Absolute task ID (not wrapped) int32_t slot; // task_id & (window_size - 1) void *packed_base; // Heap allocation result (nullptr if failure) @@ -374,9 +374,8 @@ struct TaskPayload { * @param args Task arguments (tensors + scalars) * @param result Materialized output tensors (from TensorCreateInfo path) */ - void init( - const CoreTaskArgs &args, TaskOutputTensors &result, PTO2TaskAllocResult &alloc_result, PTO2OutputLayout &layout - ) { + void + init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, PTO2OutputLayout &layout) { tensor_count = args.tensor_count(); scalar_count = args.scalar_count(); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 66fe450524..8a8692acf7 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -60,12 +60,12 @@ struct alignas(64) ChipRingFlowControl { // === Cache Line 1: Written by Scheduler, Read by Orchestrator (for back-pressure) === alignas(64) std::atomic last_task_alive; // Task ring tail (oldest active task) - // Per-boot SM reset. PTO2TaskAllocator::init() seeds its private + // Per-boot SM reset. TaskAllocator::init() seeds its private // local_task_id_ from initial_local_task_id (default 0 in production) // *without* dereferencing current_task_index — it relies on this reset // running on every AICPU boot so 0 stays in sync. If you ever change // the initial fc value or the boot ordering, update the default in - // PTO2TaskAllocator::init (ring_buffer.h) in the same change, or + // TaskAllocator::init (ring_buffer.h) in the same change, or // submit IDs will be off by the divergence. void init() { current_task_index.store(0, std::memory_order_relaxed); diff --git a/src/common/hierarchical/ring.h b/src/common/hierarchical/ring.h index aaa6eb3f88..a36fe3db60 100644 --- a/src/common/hierarchical/ring.h +++ b/src/common/hierarchical/ring.h @@ -15,7 +15,7 @@ * A single structure owns three correlated per-task resources: * * 1. A monotonic task id (`next_task_id_`), allocated by the Orchestrator. - * Unlike L2's `PTO2TaskAllocator` the id is NOT masked into a fixed-size + * Unlike L2's `TaskAllocator` the id is NOT masked into a fixed-size * window — slot state lives in parent-process heap (never crossed into * child workers), so a ring index buys us nothing at L3 (see the plan's * L2 Consistency Audit, allowed exception #6). diff --git a/src/common/host_build_graph/docs/GRAPH_EXECUTION.md b/src/common/host_build_graph/docs/GRAPH_EXECUTION.md index d293b71fb2..b2d05ef74e 100644 --- a/src/common/host_build_graph/docs/GRAPH_EXECUTION.md +++ b/src/common/host_build_graph/docs/GRAPH_EXECUTION.md @@ -225,7 +225,7 @@ They exist only to classify `OWN_OUTPUT` and `INTERNAL` Tensor sources and are converted to offsets in the Definition; they are never dereferenced or placed on the wire. Classification is by address-range containment alone, so it is only sound while no real heap address can fall in that range: -`PTO2TaskAllocator::init()` asserts the whole configured heap lies below +`TaskAllocator::init()` asserts the whole configured heap lies below `GRAPH_RECORD_VIRTUAL_BASE`, which makes an overlapping device GM heap a loud failure at setup instead of a silently misclassified Tensor source. Recording therefore leaves the shared task allocator unchanged. @@ -371,7 +371,7 @@ records on either side of H2D. Internal nodes consume no ring task-window slots. Their descriptor, payload, slot state, and argument pools live in the tail of the outer `GRAPH` task's own heap block, past `required_heap`: `[GraphExecution][GraphNodeStorage...][tensor pool][scalar pool]`. One -`PTO2TaskAllocator::alloc` covers both the packed outputs and this execution storage, +`TaskAllocator::alloc` covers both the packed outputs and this execution storage, so they are reclaimed together without a separate device allocation or release path. A node payload holds no argument array of its own — it names each region by a delta, diff --git a/tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp b/tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp index 0bd89a9a37..7b9fd2176d 100644 --- a/tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for host_build_graph's PTO2TaskAllocator (ring_buffer.h). + * Unit tests for host_build_graph's TaskAllocator (ring_buffer.h). * * host_build_graph is whole-graph-resident: the device runs only after the host * has built the whole graph, so neither the task ring nor the graph heap is ever @@ -46,7 +46,7 @@ class HbgTaskAllocatorTest : public ::testing::Test { alignas(64) uint8_t heap_buf[HEAP_SIZE]{}; std::atomic current_index{0}; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2TaskAllocator allocator{}; + TaskAllocator allocator{}; void SetUp() override { std::memset(heap_buf, 0, sizeof(heap_buf)); @@ -282,11 +282,11 @@ TEST_F(HbgTaskAllocatorTest, LatchedFatalShortCircuitsReserveDeferredHeap) { // sources by address-range containment alone. A real heap that reached into that // range would silently misclassify, so init() refuses it. TEST_F(HbgTaskAllocatorTest, InitRejectsAHeapOverlappingTheRecordingVirtualRange) { - PTO2TaskAllocator overlapping{}; + TaskAllocator overlapping{}; auto *base = reinterpret_cast(GRAPH_RECORD_VIRTUAL_BASE); EXPECT_THROW(overlapping.init(WINDOW_SIZE, ¤t_index, base, HEAP_SIZE, &error_code), AssertionError); - PTO2TaskAllocator straddling{}; + TaskAllocator straddling{}; auto *just_below = reinterpret_cast(GRAPH_RECORD_VIRTUAL_BASE - 64); EXPECT_THROW(straddling.init(WINDOW_SIZE, ¤t_index, just_below, HEAP_SIZE, &error_code), AssertionError); } @@ -299,7 +299,7 @@ TEST_F(HbgTaskAllocatorTest, InitRejectsAHeapOverlappingTheRecordingVirtualRange TEST_F(HbgTaskAllocatorTest, AcceptsTheVirtualHeapWindow) { EXPECT_EQ(HEAP_VIRTUAL_BASE + HEAP_VIRTUAL_CAPACITY, GRAPH_RECORD_VIRTUAL_BASE); - PTO2TaskAllocator virtual_heap{}; + TaskAllocator virtual_heap{}; auto *base = reinterpret_cast(HEAP_VIRTUAL_BASE); virtual_heap.init(WINDOW_SIZE, ¤t_index, base, HEAP_VIRTUAL_CAPACITY, &error_code); EXPECT_EQ(virtual_heap.heap_capacity(), HEAP_VIRTUAL_CAPACITY); diff --git a/tests/ut/cpp/a2a3/test_task_allocator.cpp b/tests/ut/cpp/a2a3/test_task_allocator.cpp index 7ad6d26f16..7cd9622975 100644 --- a/tests/ut/cpp/a2a3/test_task_allocator.cpp +++ b/tests/ut/cpp/a2a3/test_task_allocator.cpp @@ -9,13 +9,13 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2TaskAllocator from ring_buffer.h + * Unit tests for TaskAllocator from ring_buffer.h * * Tests ring buffer allocation, heap bump logic, wrap-around, alignment, * task window flow control, and heap_available semantics. * * The allocator is single-threaded (orchestrator thread), so no concurrency - * tests are needed. The unified PTO2TaskAllocator replaces the previous + * tests are needed. The unified TaskAllocator replaces the previous * separate ChipHeapRing + ChipTaskRing. * * Design contracts (try_bump_heap): @@ -79,7 +79,7 @@ class TaskAllocatorTest : public ::testing::Test { std::atomic current_index{0}; std::atomic last_alive{0}; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2TaskAllocator allocator{}; + TaskAllocator allocator{}; void SetUp() override { descriptors.assign(WINDOW_SIZE, TaskDescriptor{}); diff --git a/tests/ut/cpp/a5/test_task_allocator.cpp b/tests/ut/cpp/a5/test_task_allocator.cpp index 7ad6d26f16..7cd9622975 100644 --- a/tests/ut/cpp/a5/test_task_allocator.cpp +++ b/tests/ut/cpp/a5/test_task_allocator.cpp @@ -9,13 +9,13 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2TaskAllocator from ring_buffer.h + * Unit tests for TaskAllocator from ring_buffer.h * * Tests ring buffer allocation, heap bump logic, wrap-around, alignment, * task window flow control, and heap_available semantics. * * The allocator is single-threaded (orchestrator thread), so no concurrency - * tests are needed. The unified PTO2TaskAllocator replaces the previous + * tests are needed. The unified TaskAllocator replaces the previous * separate ChipHeapRing + ChipTaskRing. * * Design contracts (try_bump_heap): @@ -79,7 +79,7 @@ class TaskAllocatorTest : public ::testing::Test { std::atomic current_index{0}; std::atomic last_alive{0}; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2TaskAllocator allocator{}; + TaskAllocator allocator{}; void SetUp() override { descriptors.assign(WINDOW_SIZE, TaskDescriptor{}); diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index a21eaef512..dc3e82b914 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -749,7 +749,7 @@ TEST_F(WiringTest, TaskWindowOpensOnceWithheldProgressIsPublished) { ASSERT_EQ(rss.last_task_alive, 1); ASSERT_EQ(ring->fc.last_task_alive.load(), 0); - PTO2TaskAllocator allocator; + TaskAllocator allocator; auto *orch_err = sm_layout::orch_error_code_addr(sm_handle->sm_base); allocator.init( ring->task_descriptors, 4, &ring->fc.current_task_index, &ring->fc.last_task_alive, heap, sizeof(heap), @@ -767,7 +767,7 @@ TEST_F(WiringTest, TaskWindowOpensOnceWithheldProgressIsPublished) { request_and_drain_publication(); ASSERT_EQ(allocator.task_tail(), 1); - PTO2TaskAllocResult result = allocator.alloc(0, &ring->get_slot_state_by_task_id(0)); + TaskAllocResult result = allocator.alloc(0, &ring->get_slot_state_by_task_id(0)); EXPECT_FALSE(result.failed()); EXPECT_EQ(result.task_id, 3); @@ -779,7 +779,7 @@ TEST_F(WiringTest, HeapReclaimsOnceWithheldProgressIsPublished) { auto *ring = rss.ring; alignas(64) char heap[64] = {}; - PTO2TaskAllocator allocator; + TaskAllocator allocator; auto *orch_err = sm_layout::orch_error_code_addr(sm_handle->sm_base); allocator.init( ring->task_descriptors, 8, &ring->fc.current_task_index, &ring->fc.last_task_alive, heap, sizeof(heap), @@ -809,7 +809,7 @@ TEST_F(WiringTest, HeapReclaimsOnceWithheldProgressIsPublished) { request_and_drain_publication(); ASSERT_EQ(allocator.task_tail(), 1); - PTO2TaskAllocResult result = allocator.alloc(8); + TaskAllocResult result = allocator.alloc(8); EXPECT_FALSE(result.failed()); EXPECT_EQ(result.task_id, 2); diff --git a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp index a70c08b9cd..51c7c491ec 100644 --- a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp @@ -426,7 +426,7 @@ TEST_F(HbgGraphSubmitFailureTest, CachedGraphUsesFinalTaskWindowSlot) { ASSERT_TRUE(orch.graph_end()); ASSERT_EQ(orch.task_allocator.active_count(), 1); - PTO2TaskAllocator &allocator = orch.task_allocator; + TaskAllocator &allocator = orch.task_allocator; while (allocator.active_count() < allocator.window_size() - 1) { ASSERT_FALSE(allocator.alloc(0).failed()); } From 789a46269e0a745ad9bcd9ae061ecaa80115b406 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:44:11 -0700 Subject: [PATCH 11/22] Refactor: retire the PTO2 prefix on the fanin and fanout types `PTO2FaninPool` -> `FaninPool`, `PTO2FaninSpillEntry` -> `FaninSpillEntry`, `PTO2FaninBuilder` -> `FaninBuilder`, `PTO2FaninForEachReturn` -> `FaninForEachReturn`, `PTO2FaninCallbackResult` -> `FaninCallbackResult`, and the `FANOUT_SCOPE_BIT` constant. 327 occurrences across 53 files. One dependency-edge mechanism, so it moves as one commit: the pool that holds the spill entries, the builder that fills them, the traversal's return type, and the bit that marks a scope reference. The two caps stay prefixed as `CHIP_FANIN_INLINE_CAP` and `CHIP_MAX_FANIN`: both are object-like macros in `runtime/runtime_types.h`, which the whole tree includes. A macro captures its identifier textually rather than declaring anything, so a bare `MAX_FANIN` would silently rewrite any unrelated `MAX_FANIN` a caller happens to declare, instead of colliding where the compiler can say so. `FANOUT_SCOPE_BIT` is an `inline constexpr` and needs no such protection. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/rules/running-onboard.md | 2 +- docs/dfx/dep-gen.md | 2 +- .../device-error-codes/capacity.md | 2 +- .../host/dep_gen_host_graph.cpp | 2 +- .../runtime/dep_gen_host_graph.h | 2 +- .../host_build_graph/runtime/orchestrator.h | 2 +- .../orchestrator_core/orchestrator.cpp | 20 ++++---- .../runtime/orchestrator_core/ring_buffer.cpp | 2 +- .../host_build_graph/runtime/runtime_types.h | 8 ++-- .../host_build_graph/runtime/shared_memory.h | 4 +- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 2 +- .../host/dep_gen_replay.cpp | 4 +- .../host/dep_gen_replay.h | 2 +- .../runtime/orchestrator.cpp | 48 +++++++++---------- .../runtime/ring_buffer.cpp | 10 ++-- .../runtime/ring_buffer.h | 36 +++++++------- .../runtime/runtime_core.cpp | 4 +- .../runtime/runtime_types.h | 32 ++++++------- .../runtime/scheduler/scheduler.h | 6 +-- .../runtime/shared/runtime_init.cpp | 8 ++-- .../host/dep_gen_host_graph.cpp | 2 +- .../runtime/dep_gen_host_graph.h | 2 +- .../host_build_graph/runtime/orchestrator.h | 2 +- .../orchestrator_core/orchestrator.cpp | 20 ++++---- .../runtime/orchestrator_core/ring_buffer.cpp | 2 +- .../host_build_graph/runtime/runtime_types.h | 8 ++-- .../host_build_graph/runtime/shared_memory.h | 4 +- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 2 +- .../host/dep_gen_replay.cpp | 4 +- .../host/dep_gen_replay.h | 2 +- .../runtime/orchestrator.cpp | 46 +++++++++--------- .../runtime/ring_buffer.cpp | 10 ++-- .../runtime/ring_buffer.h | 36 +++++++------- .../runtime/runtime_core.cpp | 4 +- .../runtime/runtime_types.h | 32 ++++++------- .../runtime/scheduler/scheduler.h | 6 +-- .../runtime/shared/runtime_init.cpp | 8 ++-- src/common/runtime_status/error_names.h | 2 +- .../orchestration/dep_pool_overflow_orch.cpp | 6 +-- tests/ut/cpp/a2a3/test_fanin_pool.cpp | 48 +++++++++---------- tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp | 2 +- tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp | 14 +++--- tests/ut/cpp/a2a3/test_scheduler_state.cpp | 4 +- tests/ut/cpp/a2a3/test_wiring.cpp | 20 ++++---- tests/ut/cpp/a5/test_fanin_pool.cpp | 48 +++++++++---------- tests/ut/cpp/a5/test_hbg_submit_poison.cpp | 2 +- tests/ut/cpp/a5/test_orchestrator_fanin.cpp | 14 +++--- tests/ut/cpp/a5/test_scheduler_state.cpp | 4 +- tests/ut/cpp/a5/test_wiring.cpp | 26 +++++----- .../common/test_hbg_graph_submit_failure.cpp | 2 +- .../common/test_scope_deadlock_detection.cpp | 2 +- 53 files changed, 293 insertions(+), 293 deletions(-) diff --git a/.claude/rules/running-onboard.md b/.claude/rules/running-onboard.md index 19ef3a1e97..d1fc1aceab 100644 --- a/.claude/rules/running-onboard.md +++ b/.claude/rules/running-onboard.md @@ -175,7 +175,7 @@ for the signature that actually fired:** | `FATAL: Task Allocator Deadlock` / `FATAL: Dependency Pool Deadlock` / `FATAL: Fanin Spill Pool Deadlock` | an allocator or pool could not reclaim enough space | This header identifies the blocked resource, not the root cause. Classify it using the structural/timeout line that follows. | | `Provable head-of-line deadlock` | **proven open-scope structural deadlock** | TRB only: the reclaim head is the oldest task owned by an open scope on that ring. The blocked orchestrator cannot end that scope, so the head cannot become consumed. On A5, classification waits for at least 10 ms without reclaim progress and an exact-watermark publication acknowledgment. | | `No reclaim progress for ~500 ms` / `cannot reclaim space after ~500 ms` | allocator/pool **reclaim timeout** | TRB only — the 500ms backstop (`ALLOC_DEADLOCK_TIMEOUT_CYCLES`) lives in its reclaiming allocators. It proves prolonged lack of reclaim progress, not why progress stopped; check capacity, the dumped head, consumers and scheduler state. | -| `Task Window Exhausted` / `Fanin Capacity Exhausted` / `TensorMap Entry Pool Exhausted` | **capacity**, HBG only | HBG is whole-graph-resident and reclaims nothing mid-run, so this is a sizing verdict reached immediately, not a stall. The line carries used/capacity and the requested amount; raise `PTO2_RING_TASK_WINDOW` / `CHIP_TENSORMAP_POOL_SIZE` or shrink the graph. Inline fanin is hard-capped at `PTO2_MAX_FANIN=128`; HBG has no `PTO2_RING_DEP_POOL`. HBG's graph heap has no knob and cannot exhaust: it is committed after orchestration at the measured size, so a graph too large for the device fails host-side at that commit. | +| `Task Window Exhausted` / `Fanin Capacity Exhausted` / `TensorMap Entry Pool Exhausted` | **capacity**, HBG only | HBG is whole-graph-resident and reclaims nothing mid-run, so this is a sizing verdict reached immediately, not a stall. The line carries used/capacity and the requested amount; raise `PTO2_RING_TASK_WINDOW` / `CHIP_TENSORMAP_POOL_SIZE` or shrink the graph. Inline fanin is hard-capped at `CHIP_MAX_FANIN=128`; HBG has no `PTO2_RING_DEP_POOL`. HBG's graph heap has no knob and cannot exhaust: it is committed after orchestration at the measured size, so a graph too large for the device fails host-side at that commit. | | `Timeout (N cycles): producer/consumers ...` | **SPIN** wait on a specific producer/consumer | `runtime_core.cpp`. | | `HandleTaskTimeout` / `kill aicpu-sd` | **OS op-execute timeout** | STARS/tsdaemon, default 45s (`PLATFORM_OP_EXECUTE_TIMEOUT_US`). **A 45s kill ≠ deadlock** — the op was merely long or stalled. Raise this constant to measure true on-device duration. | | `log_stall_diagnostics` (cores idle + tasks `state=WAIT fanin 0/N` + `completed` frozen) | **forward-progress stall** | No dedicated detector — intermittent races (often contention-triggered) land here and are reaped only by the op-timeout above. | diff --git a/docs/dfx/dep-gen.md b/docs/dfx/dep-gen.md index c3c095b278..96548e0b08 100644 --- a/docs/dfx/dep-gen.md +++ b/docs/dfx/dep-gen.md @@ -66,7 +66,7 @@ inputs to each submit are captured and the graph is reconstructed afterwards. Per-record semantics mirror runtime `submit_task` exactly: STEP 1 (explicit deps), STEP 3 (creator retention + tensormap lookup), STEP 4 (register outputs). Per-successor dedup matches - `PTO2FaninBuilder::append_fanin_or_fail`. After both passes finish per + `FaninBuilder::append_fanin_or_fail`. After both passes finish per record, the replay asserts the two producer-id → `DepFlags` mappings are equal (same producers and same per-producer flags); if they diverge, `deps.json` is not written and the function returns non-zero. diff --git a/docs/troubleshooting/device-error-codes/capacity.md b/docs/troubleshooting/device-error-codes/capacity.md index a15619cd1e..8955e71005 100644 --- a/docs/troubleshooting/device-error-codes/capacity.md +++ b/docs/troubleshooting/device-error-codes/capacity.md @@ -43,7 +43,7 @@ that trips the code. | Runtime | Bottleneck resource | Code | Fix | | ------- | ------------------- | ---- | --- | | HBG | task window | 3 | raise `ring_task_window` (`PTO2_RING_TASK_WINDOW`), or shrink the graph | -| HBG | inline fanin | 4 | reduce distinct producers to `PTO2_MAX_FANIN` (currently 128) or less; HBG has no `PTO2_RING_DEP_POOL` | +| HBG | inline fanin | 4 | reduce distinct producers to `CHIP_MAX_FANIN` (currently 128) or less; HBG has no `PTO2_RING_DEP_POOL` | | HBG | TensorMap entries | 11 | increase `CHIP_TENSORMAP_POOL_SIZE`, or reduce registered outputs | | TRB | open-scope task window | 1 or 3 | raise `PTO2_RING_TASK_WINDOW`, split the scope, or diagnose stalled reclaim | | TRB | heap | 2 | raise `PTO2_RING_HEAP`, shrink allocations, or diagnose stalled reclaim | diff --git a/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp b/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp index 9450b2cdaa..f57e115dc0 100644 --- a/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp +++ b/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp @@ -19,7 +19,7 @@ * tensormap — a producer whose written slice overlaps what this task reads * (STEP 3 Step B); carries both slices' geometry. * - * Per-task producer dedup mirrors PTO2FaninBuilder::append_fanin_or_fail, which + * Per-task producer dedup mirrors FaninBuilder::append_fanin_or_fail, which * collapses all three sources into one fanin list: the first edge to name a * producer is kept. tensormap edges are exempt — a second producer slice for the * same task is a distinct fact about the data flow, and viewers rely on seeing diff --git a/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h b/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h index 97c4178a4f..9bab78c0d8 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h +++ b/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h @@ -43,7 +43,7 @@ * progress loop satisfies this by being single-threaded; emit returns -3 if the * invariant is ever broken. * - * Per-task producer dedup mirrors PTO2FaninBuilder, which keys on (ring, slot); + * Per-task producer dedup mirrors FaninBuilder, which keys on (ring, slot); * this keys on producer task id. The two agree only because host_build_graph is * whole-graph-resident and never reuses a task slot at build time (see * append_fanin_or_fail in orchestrator.cpp). A runtime that recycles slots diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h index f56aa75cfd..d4b433e7c2 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h @@ -109,7 +109,7 @@ struct OrchestratorState { // case (task_window tasks each at their full cap), so a bump cannot overflow. // The bases live here, not in the ring header: nothing on the device resolves one. // - // The fanin cursor does not advance at bind time. PTO2FaninBuilder appends and + // The fanin cursor does not advance at bind time. FaninBuilder appends and // dedups producers afterwards, so the region's length is known only when the count // is published, and it advances there. int32_t *fanin_pool{nullptr}; diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 95a30969e8..90983af1b4 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1137,11 +1137,11 @@ static uint32_t next_fanin_seen_epoch(OrchestratorState *orch) { // Polling: fanin is a flat array of position-independent producer local ids on // the payload (no dep-pool spill, no producer pointers). The builder writes them // directly into the payload's fanin region as producers are appended, deduping by -// slot and hard-capping at PTO2_MAX_FANIN. self_local is this task's own local id +// slot and hard-capping at CHIP_MAX_FANIN. self_local is this task's own local id // (the consumer), used to bump each producer's last_consumer_local_id (the // reclaim gate the host wait_for_consumers polls via completed_watermark). -struct PTO2FaninBuilder { - PTO2FaninBuilder(OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : +struct FaninBuilder { + FaninBuilder(OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : count(0), orch(orch), seen_epoch(seen_epoch), @@ -1179,7 +1179,7 @@ struct PTO2FaninBuilder { static bool append_fanin_or_fail( OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, - TaskId producer_task_id, PTO2FaninBuilder *fanin_builder + TaskId producer_task_id, FaninBuilder *fanin_builder ) { // Skip a stale/reused producer slot: the cached owner id no longer resolves // to this producer (defensive — whole-graph-resident hbg does not reuse slots @@ -1192,15 +1192,15 @@ static bool append_fanin_or_fail( if (fanin_builder->mark_seen(prod_ring, prod_slot)) { return true; } - if (fanin_builder->count >= PTO2_MAX_FANIN) { + if (fanin_builder->count >= CHIP_MAX_FANIN) { LOG_ERROR("========================================"); LOG_ERROR("FATAL: Fanin Capacity Exhausted!"); LOG_ERROR("========================================"); LOG_ERROR("HBG stores every producer dependency in the consumer task's fanin region."); - LOG_ERROR(" Fanin: used=%d/%d", fanin_builder->count, PTO2_MAX_FANIN); + LOG_ERROR(" Fanin: used=%d/%d", fanin_builder->count, CHIP_MAX_FANIN); LOG_ERROR(" Requested: at least %d distinct producer dependencies", fanin_builder->count + 1); LOG_ERROR("Solution:"); - LOG_ERROR(" Reduce the task fanin to at most PTO2_MAX_FANIN=%d.", PTO2_MAX_FANIN); + LOG_ERROR(" Reduce the task fanin to at most CHIP_MAX_FANIN=%d.", CHIP_MAX_FANIN); LOG_ERROR(" HBG has no dependency spill pool; PTO2_RING_DEP_POOL does not apply."); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); @@ -1279,7 +1279,7 @@ static bool prepare_task( const int32_t scalar_span = PTO2_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); debug_assert(static_cast(orch->tensor_pool_cursor) + args.tensor_count() <= window * MAX_TENSOR_ARGS); debug_assert(static_cast(orch->scalar_pool_cursor) + scalar_span <= window * MAX_SCALAR_ARGS); - debug_assert(static_cast(orch->fanin_pool_cursor) + PTO2_MAX_FANIN <= window * PTO2_MAX_FANIN); + debug_assert(static_cast(orch->fanin_pool_cursor) + CHIP_MAX_FANIN <= window * CHIP_MAX_FANIN); out->payload->bind_regions( orch->tensor_pool + orch->tensor_pool_cursor, orch->scalar_pool + orch->scalar_pool_cursor, orch->fanin_pool + orch->fanin_pool_cursor @@ -1484,7 +1484,7 @@ static TaskOutputTensors submit_task_common( ); } - PTO2FaninBuilder fanin_builder(orch, &payload, static_cast(task_id.local()), next_fanin_seen_epoch(orch)); + FaninBuilder fanin_builder(orch, &payload, static_cast(task_id.local()), next_fanin_seen_epoch(orch)); CYCLE_COUNT_LAP(g_orch_alloc_cycle); @@ -1845,7 +1845,7 @@ bool graph_submit_outer( ); } - PTO2FaninBuilder fanin_builder(orch, &payload, static_cast(task_id.local()), next_fanin_seen_epoch(orch)); + FaninBuilder fanin_builder(orch, &payload, static_cast(task_id.local()), next_fanin_seen_epoch(orch)); auto emit = [&](TaskId producer_id) -> bool { const int32_t producer_local = static_cast(producer_id.local()); const int32_t producer_slot = ring.get_slot_by_task_id(producer_local); diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp index b321cedce4..1645a4061c 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp @@ -12,6 +12,6 @@ // The polling completion scheduler has no dep-pool / fanin-spill pool: producer // dependencies are position-independent local-id integers on the payload and // readiness is derived from the ring completion_flags. The pool ring buffers -// (PTO2FaninPool / PTO2DepListPool) that once lived here are gone, so this +// (FaninPool / PTO2DepListPool) that once lived here are gone, so this // translation unit carries no definitions. TaskAllocator's methods are // defined inline in ring_buffer.h. diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index b250e2abf2..344f8a9776 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -131,13 +131,13 @@ inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HE #define PTO2_EARLY_DISPATCH_QUEUE_SIZE 64 // Fanin storage -#define PTO2_FANIN_INLINE_CAP 64 +#define CHIP_FANIN_INLINE_CAP 64 // Polling-scheduler fanin cap. The polling model stores producer dependencies as // flat position-independent local-id integers in the fanin pool (no dep-pool // spill), so a task's fanin degree is hard-capped here. Must cover the worst-case // fanin of any workload (paged_attention is the densest). -#define PTO2_MAX_FANIN 128 +#define CHIP_MAX_FANIN 128 // Alignment of every per-task region inside an argument pool. Each region starts // and ends on a cache line so TaskPayload::init's round-up scalar memcpy stays @@ -293,7 +293,7 @@ inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; struct TaskPayload { // === Cache line 0 (64B) — the dispatch path's own line === - // sizeof is independent of PTO2_MAX_FANIN / MAX_TENSOR_ARGS / MAX_SCALAR_ARGS: + // sizeof is independent of CHIP_MAX_FANIN / MAX_TENSOR_ARGS / MAX_SCALAR_ARGS: // widening a cap costs pool bytes for the tasks that need them, not a control // block on every task. int32_t tensor_count{0}; @@ -309,7 +309,7 @@ struct TaskPayload { // fanin holds flat position-independent producer local task ids. Single-ring // hbg: every producer is ring 0, so no per-edge ring id is stored. Scanned by // classify_fanin_state against the ring completion_flags. Hard-capped at - // PTO2_MAX_FANIN (no dep-pool spill). Unbound on a Graph node, whose + // CHIP_MAX_FANIN (no dep-pool spill). Unbound on a Graph node, whose // dependencies live in the Definition's fanin CSR instead. simpler::hbg::SelfRelativePtr tensors; simpler::hbg::SelfRelativePtr scalars; diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h index 8eddaf82c4..018e9e5056 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h @@ -331,7 +331,7 @@ struct RingImageExtents { inline RingImageExtents mirror_extents(uint64_t task_window_size) noexcept { return RingImageExtents{ task_window_size, - task_window_size * PTO2_MAX_FANIN, + task_window_size * CHIP_MAX_FANIN, task_window_size * MAX_TENSOR_ARGS, task_window_size * MAX_SCALAR_ARGS, }; @@ -371,7 +371,7 @@ inline ChipRingSegmentOffsets ring_segment_offsets(uint64_t task_window_size) no // round-up scalar memcpy relies on. ChipTensor is 2 cache lines, so a tensor region // is aligned for any count; the fanin and scalar strides need it stated. static_assert( - (PTO2_MAX_FANIN * sizeof(int32_t)) % ARG_POOL_ALIGN == 0, + (CHIP_MAX_FANIN * sizeof(int32_t)) % ARG_POOL_ALIGN == 0, "the fanin region's cap must be a whole number of cache lines" ); static_assert( diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 61e850f330..f79dc0c687 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -54,7 +54,7 @@ Bundles the three per-ring resources into a single aggregate (`ring_buffer.h`): struct ChipRingSet { ChipHeapRing heap_ring; ChipTaskRing task_ring; - PTO2FaninPool fanin_pool; + FaninPool fanin_pool; }; ``` diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index bc9281ed68..7ab3360317 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -434,7 +434,7 @@ Key members: | 0 | `ChipTensorMap::sync_tensormap` — prune stale TensorMap entries | | 1 | `TaskAllocator::alloc` — allocate task slot (may block on flow control) | | 2 | Initialize task descriptor + slot state, copy parameters | -| 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `PTO2FaninBuilder` | +| 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `FaninBuilder` | | 4 | **Insert**: register OUTPUT/INOUT args in TensorMap | | 5 | **Record fanin metadata**: store producer edges (slot pointer + `DepFlags` packed in the low bits) in `payload->fanin_inline_edges[]` (+ spill pool if >64); claim each live producer by incrementing `fanout_count` under that producer's `fanout_lock`. Creator edges are `DEP_WAIT\|DEP_RETAIN`, tensormap-modifier edges `DEP_WAIT`. This step runs **before** `payload.init()`. | | 6 | **Orch-side wiring / ready publish**: the orchestrator wires live fanout edges into the per-ring dep_pool; zero-fanin and already-completed fanin tasks publish directly to ready queues. Only `DEP_WAIT` edges gate readiness — they count toward `fanin_count` and are linked onto the producer's `fanout_head` for completion notification. A `DEP_WAIT`-only edge releases its submit→wire retention pin **at wiring** (and on the already-completed fast path), so its producer can be CONSUMED without waiting for this consumer; a `DEP_RETAIN` edge keeps the pin until this consumer's `on_task_release`. A hypothetical `RETAIN`-only edge (none exist yet) would neither gate readiness nor link a fanout node — it only holds the lifetime pin. | diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp index 0e3ce5513f..f3bd97ea67 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp @@ -536,7 +536,7 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c TensorArgType atype_buf[CORE_MAX_TENSOR_ARGS]; // Per-record producer ID -> accumulated DepFlags — must match runtime's - // PTO2FaninBuilder::append_fanin_or_fail semantics, which collapses STEP 1 + // FaninBuilder::append_fanin_or_fail semantics, which collapses STEP 1 // (explicit_deps) + STEP A (creator retention) + STEP B (tensormap lookup) // into a single per-task fanin edge and OR-accumulates its flags. Both oracle // and annot use this same semantics so the divergence check compares the @@ -761,7 +761,7 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c // the same producer (different version), or two different // producers, both yield their own edges. The producer-id-set // comparison below uses annot_preds, which dedups by pred - // only, matching runtime PTO2FaninBuilder semantics. + // only, matching runtime FaninBuilder semantics. annot_preds[producer.raw] |= DEP_WAIT; EdgeAnnot e{}; e.pred = producer.raw; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h b/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h index 3bb89ee315..c2f70292b2 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h @@ -54,7 +54,7 @@ * are carried per-arg / per-edge alongside ``start_offset`` + ``strides``. * - Distinct producers / arg indices / sources keep their own edges; per-record * deduplication of producer ids mirrors the runtime - * ``PTO2FaninBuilder::append_fanin_or_fail`` semantics so the set of + * ``FaninBuilder::append_fanin_or_fail`` semantics so the set of * ``(pred, succ)`` pairs is identical to what the runtime would have * recorded. * diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 2335788894..196c00283b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -234,8 +234,8 @@ static uint32_t next_fanin_seen_epoch(OrchestratorState *orch) { return next; } -struct PTO2FaninBuilder { - PTO2FaninBuilder(OrchestratorState *orch, PTO2FaninPool &spill_pool, uint32_t seen_epoch) : +struct FaninBuilder { + FaninBuilder(OrchestratorState *orch, FaninPool &spill_pool, uint32_t seen_epoch) : count(0), wait_count(0), spill_start(0), @@ -247,11 +247,11 @@ struct PTO2FaninBuilder { int32_t spill_start{0}; OrchestratorState *orch{nullptr}; uint32_t seen_epoch{0}; - PTO2FaninPool &spill_pool; - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP]; + FaninPool &spill_pool; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP]; template - PTO2FaninForEachReturn for_each(Fn &&fn) const { + FaninForEachReturn for_each(Fn &&fn) const { return for_each_fanin_storage(inline_slots, count, spill_start, spill_pool, static_cast(fn)); } @@ -269,7 +269,7 @@ struct PTO2FaninBuilder { } // Append a new edge (caller has already claimed the producer's fanout pin). - void push_edge(PTO2FaninSpillEntry &entry, ChipTaskSlotState *prod_state, DepFlags kind) { + void push_edge(FaninSpillEntry &entry, ChipTaskSlotState *prod_state, DepFlags kind) { entry.set(prod_state, kind); count++; if (dep_has_wait(kind)) { @@ -282,7 +282,7 @@ struct PTO2FaninBuilder { // edge instead of adding a second one — no extra fanout pin is claimed. void or_flags_into_existing(ChipTaskSlotState *prod_state, DepFlags kind) { for (int32_t i = 0; i < count; i++) { - PTO2FaninSpillEntry &entry = entry_at(i); + FaninSpillEntry &entry = entry_at(i); if (entry.slot_state() == prod_state) { accumulate_flags(entry, kind); return; @@ -295,18 +295,18 @@ struct PTO2FaninBuilder { } private: - // The i-th appended fanin edge: inline for i < PTO2_FANIN_INLINE_CAP, else in + // The i-th appended fanin edge: inline for i < CHIP_FANIN_INLINE_CAP, else in // the spill ring at the same linear->physical position for_each_fanin_storage // walks. Single source of the wrap arithmetic. - PTO2FaninSpillEntry &entry_at(int32_t i) { - if (i < PTO2_FANIN_INLINE_CAP) { + FaninSpillEntry &entry_at(int32_t i) { + if (i < CHIP_FANIN_INLINE_CAP) { return inline_slots[i]; } - int32_t spill_idx = i - PTO2_FANIN_INLINE_CAP; + int32_t spill_idx = i - CHIP_FANIN_INLINE_CAP; return spill_pool.base[(spill_start % spill_pool.capacity + spill_idx) % spill_pool.capacity]; } - void accumulate_flags(PTO2FaninSpillEntry &entry, DepFlags kind) { + void accumulate_flags(FaninSpillEntry &entry, DepFlags kind) { bool had_wait = dep_has_wait(entry.flags()); entry.add_flags(kind); if (!had_wait && dep_has_wait(kind)) { @@ -317,7 +317,7 @@ struct PTO2FaninBuilder { static bool append_fanin_or_fail( OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, - TaskId producer_task_id, PTO2FaninBuilder *fanin_builder, uint8_t ring_id, DepFlags kind + TaskId producer_task_id, FaninBuilder *fanin_builder, uint8_t ring_id, DepFlags kind ) { // Decide-and-claim under the producer's fanout_lock. Two conditions make this // resolved slot a non-dependency, and both must be checked together with the @@ -360,11 +360,11 @@ static bool append_fanin_or_fail( // count must never carry into bit31 (would corrupt the scope-release // flag) — true for any sane fanout (<< 2^31). assert( - (prod_state->fanout_count & ~PTO2_FANOUT_SCOPE_BIT) < (PTO2_FANOUT_SCOPE_BIT - 1) && + (prod_state->fanout_count & ~FANOUT_SCOPE_BIT) < (FANOUT_SCOPE_BIT - 1) && "fanout consumer count overflow into scope bit" ); prod_state->fanout_count++; - fanout_now = static_cast(prod_state->fanout_count & ~PTO2_FANOUT_SCOPE_BIT); + fanout_now = static_cast(prod_state->fanout_count & ~FANOUT_SCOPE_BIT); } prod_state->unlock_fanout(); #if SIMPLER_ORCH_PROFILING @@ -390,30 +390,30 @@ static bool append_fanin_or_fail( return true; } - if (fanin_builder->count < PTO2_FANIN_INLINE_CAP) { + if (fanin_builder->count < CHIP_FANIN_INLINE_CAP) { fanin_builder->push_edge(fanin_builder->inline_slots[fanin_builder->count], prod_state, kind); return true; } - PTO2FaninPool &fanin_pool = fanin_builder->spill_pool; + FaninPool &fanin_pool = fanin_builder->spill_pool; if (!fanin_pool.ensure_space(orch->sm_header->rings[ring_id], 1)) { orch_mark_fatal(orch, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; } int32_t spill_idx = fanin_pool.top; - PTO2FaninSpillEntry *entry = fanin_pool.alloc(); + FaninSpillEntry *entry = fanin_pool.alloc(); if (entry == nullptr) { orch_mark_fatal(orch, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; } - if (fanin_builder->count == PTO2_FANIN_INLINE_CAP) { + if (fanin_builder->count == CHIP_FANIN_INLINE_CAP) { fanin_builder->spill_start = spill_idx; } fanin_builder->push_edge(*entry, prod_state, kind); return true; } -static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) { +static bool all_claimed_fanin_completed(const FaninBuilder &fanin_builder) { if (fanin_builder.count == 0) return true; // Only DEP_WAIT edges gate readiness; a retention-only edge never blocks // dispatch, so it is treated as satisfied here. @@ -423,7 +423,7 @@ static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) { }); } -static bool all_claimed_fanin_allow_early_resolve(const PTO2FaninBuilder &fanin_builder) { +static bool all_claimed_fanin_allow_early_resolve(const FaninBuilder &fanin_builder) { if (fanin_builder.count == 0) return true; return fanin_builder.for_each([](ChipTaskSlotState *producer, DepFlags flags) -> bool { if (!dep_has_wait(flags)) return true; @@ -645,7 +645,7 @@ static bool prepare_task( // single payload-init point, which runs before Orch-side wiring publish. // Fields already reset by advance_ring_pointers (eager reset after CONSUMED): - // fanout_lock=0, fanout_count=PTO2_FANOUT_SCOPE_BIT, fanout_head=nullptr, + // fanout_lock=0, fanout_count=FANOUT_SCOPE_BIT, fanout_head=nullptr, // fanin_refcount=0, fanout_refcount=0, completed_subtasks=0, next_block_idx=0 // Fields immutable after RingSchedState::init(): // ring_id @@ -937,7 +937,7 @@ static TaskOutputTensors submit_task_common( } #endif - PTO2FaninBuilder fanin_builder(orch, orch->rings[ring_id].fanin_pool, next_fanin_seen_epoch(orch)); + FaninBuilder fanin_builder(orch, orch->rings[ring_id].fanin_pool, next_fanin_seen_epoch(orch)); CYCLE_COUNT_LAP(g_orch_alloc_cycle); @@ -1032,7 +1032,7 @@ static TaskOutputTensors submit_task_common( // the producer's fanout_lock. Doing it there (rather than a separate pass // here) is what prevents a producer from transitioning to CONSUMED between // the dependency decision and the claim. - int32_t inline_count = std::min(fanin_builder.count, PTO2_FANIN_INLINE_CAP); + int32_t inline_count = std::min(fanin_builder.count, CHIP_FANIN_INLINE_CAP); // Every fanin edge produced here carries DEP_WAIT (creator = WAIT|RETAIN, // modifier = WAIT, explicit defaults to WAIT|RETAIN or opts into WAIT), so // wait_count == count. fanin_actual_count therefore doubles as the WAIT-edge diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index ff725df2d1..19cbdf473d 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -34,7 +34,7 @@ static void latch_pool_error(std::atomic *error_code_ptr, int32_t error // ============================================================================= // Fanin Spill Pool Implementation // ============================================================================= -void PTO2FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { +void FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { if (sm_last_task_alive <= reclaim_task_cursor) return; int32_t scan_end = sm_last_task_alive; @@ -44,7 +44,7 @@ void PTO2FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_a continue; } - int32_t inline_count = std::min(payload.fanin_actual_count, PTO2_FANIN_INLINE_CAP); + int32_t inline_count = std::min(payload.fanin_actual_count, CHIP_FANIN_INLINE_CAP); int32_t spill_edge_count = payload.fanin_actual_count - inline_count; if (spill_edge_count > 0) { advance_tail(payload.fanin_spill_start + spill_edge_count); @@ -53,7 +53,7 @@ void PTO2FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_a reclaim_task_cursor = scan_end; } -bool PTO2FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { +bool FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { if (available() >= needed) return true; int spin_count = 0; @@ -165,8 +165,8 @@ void PTO2DepListPool::report_deadlock( uint32_t rc = h.fanout_refcount.load(std::memory_order_acquire); LOG_ERROR( " Head task %d: state=%d, consumers=%u/%u, scope_released=%d", last_alive, - static_cast(h.task_state.load(std::memory_order_acquire)), rc & ~PTO2_FANOUT_SCOPE_BIT, - fc & ~PTO2_FANOUT_SCOPE_BIT, (rc & PTO2_FANOUT_SCOPE_BIT) ? 1 : 0 + static_cast(h.task_state.load(std::memory_order_acquire)), rc & ~FANOUT_SCOPE_BIT, + fc & ~FANOUT_SCOPE_BIT, (rc & FANOUT_SCOPE_BIT) ? 1 : 0 ); } LOG_ERROR("Solution:"); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 753ff2ec71..284eb21a29 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -456,8 +456,8 @@ class TaskAllocator { uint32_t rc = h.fanout_refcount.load(std::memory_order_acquire); LOG_ERROR( " Head task %d: state=%d, consumers=%u/%u, scope_released=%d", last_alive, - static_cast(h.task_state.load(std::memory_order_acquire)), rc & ~PTO2_FANOUT_SCOPE_BIT, - fc & ~PTO2_FANOUT_SCOPE_BIT, (rc & PTO2_FANOUT_SCOPE_BIT) ? 1 : 0 + static_cast(h.task_state.load(std::memory_order_acquire)), rc & ~FANOUT_SCOPE_BIT, + fc & ~FANOUT_SCOPE_BIT, (rc & FANOUT_SCOPE_BIT) ? 1 : 0 ); } LOG_ERROR("Solution:"); @@ -503,8 +503,8 @@ class TaskAllocator { * Linear counters (top, tail) grow monotonically; the physical index * is obtained via modulo: base[linear_index % capacity]. */ -struct PTO2FaninPool { - PTO2FaninSpillEntry *base; // Pool base address +struct FaninPool { + FaninSpillEntry *base; // Pool base address int32_t capacity; // Total number of entries int32_t top; // Linear next-allocation counter (starts from 1) int32_t tail; // Linear first-alive counter (entries before this are dead) @@ -513,7 +513,7 @@ struct PTO2FaninPool { std::atomic *error_code_ptr = nullptr; - void init(PTO2FaninSpillEntry *in_base, int32_t in_capacity, std::atomic *in_error_code_ptr) { + void init(FaninSpillEntry *in_base, int32_t in_capacity, std::atomic *in_error_code_ptr) { base = in_base; capacity = in_capacity; top = 1; @@ -537,7 +537,7 @@ struct PTO2FaninPool { bool ensure_space(SharedMemoryRingHeader &ring, int32_t needed); - PTO2FaninSpillEntry *alloc() { + FaninSpillEntry *alloc() { int32_t used = top - tail; if (used >= capacity) { LOG_ERROR("========================================"); @@ -576,25 +576,25 @@ struct PTO2FaninPool { }; template -using PTO2FaninCallbackResult = std::invoke_result_t; +using FaninCallbackResult = std::invoke_result_t; template -using PTO2FaninForEachReturn = std::conditional_t, void>, void, bool>; +using FaninForEachReturn = std::conditional_t, void>, void, bool>; // Visit each fanin edge as (producer slot, DepFlags). Inline and spill entries -// share the packed PTO2FaninSpillEntry layout, so both are unpacked the same way. +// share the packed FaninSpillEntry layout, so both are unpacked the same way. template -inline PTO2FaninForEachReturn for_each_fanin_storage( - InlineSlots &&inline_edges, int32_t fanin_count, int32_t spill_start, PTO2FaninPool &spill_pool, Fn &&fn +inline FaninForEachReturn for_each_fanin_storage( + InlineSlots &&inline_edges, int32_t fanin_count, int32_t spill_start, FaninPool &spill_pool, Fn &&fn ) { - using FaninCallbackResult = PTO2FaninCallbackResult; + using FaninCallbackResult = FaninCallbackResult; static_assert( std::is_same_v || std::is_same_v, "fanin callback must return void or bool" ); if constexpr (std::is_void_v) { - int32_t inline_count = std::min(fanin_count, PTO2_FANIN_INLINE_CAP); + int32_t inline_count = std::min(fanin_count, CHIP_FANIN_INLINE_CAP); for (int32_t i = 0; i < inline_count; i++) { fn(inline_edges[i].slot_state(), inline_edges[i].flags()); } @@ -606,7 +606,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( int32_t start_idx = spill_start % spill_pool.capacity; int32_t first_count = std::min(spill_count, spill_pool.capacity - start_idx); - PTO2FaninSpillEntry *first = spill_pool.base + start_idx; + FaninSpillEntry *first = spill_pool.base + start_idx; for (int32_t i = 0; i < first_count; i++) { fn(first[i].slot_state(), first[i].flags()); } @@ -617,7 +617,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( } return; } else { - int32_t inline_count = std::min(fanin_count, PTO2_FANIN_INLINE_CAP); + int32_t inline_count = std::min(fanin_count, CHIP_FANIN_INLINE_CAP); for (int32_t i = 0; i < inline_count; i++) { if (!fn(inline_edges[i].slot_state(), inline_edges[i].flags())) { return false; @@ -631,7 +631,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( int32_t start_idx = spill_start % spill_pool.capacity; int32_t first_count = std::min(spill_count, spill_pool.capacity - start_idx); - PTO2FaninSpillEntry *first = spill_pool.base + start_idx; + FaninSpillEntry *first = spill_pool.base + start_idx; for (int32_t i = 0; i < first_count; i++) { if (!fn(first[i].slot_state(), first[i].flags())) { return false; @@ -649,7 +649,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( } template -inline PTO2FaninForEachReturn for_each_fanin_slot_state(const TaskPayload &payload, Fn &&fn) { +inline FaninForEachReturn for_each_fanin_slot_state(const TaskPayload &payload, Fn &&fn) { return for_each_fanin_storage( payload.fanin_inline_edges, payload.fanin_actual_count, payload.fanin_spill_start, *payload.fanin_spill_pool, static_cast(fn) @@ -811,7 +811,7 @@ struct PTO2DepListPool { */ struct ChipRingSet { TaskAllocator task_allocator; - PTO2FaninPool fanin_pool; + FaninPool fanin_pool; }; #endif // PTO_RING_BUFFER_H diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 4ad31ea0fe..e3485bec80 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -143,8 +143,8 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo int32_t local_id = slot.task->task_id.local(); uint64_t t0 = get_sys_cnt_aicpu(); int32_t spin_count = 0; - while ((slot.fanout_refcount.load(std::memory_order_acquire) & ~PTO2_FANOUT_SCOPE_BIT) < - (slot.fanout_count & ~PTO2_FANOUT_SCOPE_BIT)) { + while ((slot.fanout_refcount.load(std::memory_order_acquire) & ~FANOUT_SCOPE_BIT) < + (slot.fanout_count & ~FANOUT_SCOPE_BIT)) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { // A fatal latched elsewhere breaks this wait; cold path only. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 62502d458d..15831a0020 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -94,7 +94,7 @@ #define PTO2_EARLY_DISPATCH_QUEUE_SIZE 256 // Fanin storage -#define PTO2_FANIN_INLINE_CAP 64 +#define CHIP_FANIN_INLINE_CAP 64 // Dependency-degree diagnostic: log once at debug level when a task's fanin or // a producer's fanout first exceeds this degree, so dense dependency graphs can @@ -167,18 +167,18 @@ struct PTO2OutputLayout { * Stored in the dedicated fanin spill ring buffer. */ struct ChipTaskSlotState; // Forward declaration -struct PTO2FaninPool; // Forward declaration +struct FaninPool; // Forward declaration // One fanin edge: a producer slot pointer with the per-edge DepFlags packed into // bits 0..1 of the pointer. ChipTaskSlotState is alignas(64), so bits 0..5 are // always zero in a real pointer. Used for both the inline fanin array and the // spill pool, keeping sizeof == sizeof(uintptr_t) so neither footprint grows. -struct PTO2FaninSpillEntry { +struct FaninSpillEntry { static constexpr uintptr_t FLAG_MASK = 0x3; // No in-class initializer: the type stays trivially default-constructible so // the payload's fanin_inline_edges[64] and the orchestrator's per-submit // builder array are default-initialized at zero cost (only entries [0, count) - // are written via set()). Value-init (`PTO2FaninSpillEntry{}`) still zeroes it. + // are written via set()). Value-init (`FaninSpillEntry{}`) still zeroes it. uintptr_t packed; ChipTaskSlotState *slot_state() const { return reinterpret_cast(packed & ~FLAG_MASK); } @@ -191,7 +191,7 @@ struct PTO2FaninSpillEntry { void add_flags(DepFlags f) { packed |= (static_cast(f) & FLAG_MASK); } void clear() { packed = 0; } }; -static_assert(sizeof(PTO2FaninSpillEntry) == sizeof(uintptr_t)); +static_assert(sizeof(FaninSpillEntry) == sizeof(uintptr_t)); /** * Dependency list entry (singly-linked list node) @@ -280,11 +280,11 @@ struct TaskPayload { int32_t scalar_count{0}; int32_t fanin_actual_count{0}; // Actual fanin count (without the +1 redundance) int32_t fanin_spill_start{0}; // Linear start index in fanin spill pool (0 = no spill) - PTO2FaninPool *fanin_spill_pool{nullptr}; + FaninPool *fanin_spill_pool{nullptr}; // Inline fanin edges (producer slot + packed DepFlags). Spill beyond - // PTO2_FANIN_INLINE_CAP goes to fanin_spill_pool. Same packed layout as the + // CHIP_FANIN_INLINE_CAP goes to fanin_spill_pool. Same packed layout as the // spill entries, so the array footprint is unchanged. - PTO2FaninSpillEntry fanin_inline_edges[PTO2_FANIN_INLINE_CAP]; + FaninSpillEntry fanin_inline_edges[CHIP_FANIN_INLINE_CAP]; // Early-dispatch metadata (AICPU-side only). Ordered by descending // alignment so the block packs without internal padding. Cache line 8 // contains the rarely-touched fanin tail rather than the hot tensor/scalar @@ -444,18 +444,18 @@ static_assert( // fanout_count / fanout_refcount bit encoding (both uint32): // bits [30:0] = consumer references (count: # consumers; refcount: # released) -// bit [31] = the owning scope's reference (PTO2_FANOUT_SCOPE_BIT) -// fanout_count is seeded to PTO2_FANOUT_SCOPE_BIT and ++'d per consumer, so it +// bit [31] = the owning scope's reference (FANOUT_SCOPE_BIT) +// fanout_count is seeded to FANOUT_SCOPE_BIT and ++'d per consumer, so it // ends as (SCOPE_BIT | num_consumers). release adds 1 (consumer completion) or // SCOPE_BIT (scope_end). CONSUMED iff fanout_refcount == fanout_count (every // consumer released AND scope bit set). Keeping the scope ref in a distinct bit // (rather than folding scope + consumers into one count) lets a consumer reach -// fanout_refcount == (fanout_count & ~PTO2_FANOUT_SCOPE_BIT) while the scope bit +// fanout_refcount == (fanout_count & ~FANOUT_SCOPE_BIT) while the scope bit // is still unset -- i.e. "all consumers done but scope still open" stays // distinguishable from "fully consumed". Structural allocation deadlock is // detected separately by checking whether the blocking ring head is the oldest // task pinned by any open scope on that ring. -static constexpr uint32_t PTO2_FANOUT_SCOPE_BIT = 0x80000000u; +static constexpr uint32_t FANOUT_SCOPE_BIT = 0x80000000u; enum TaskLifecycleFlag : uint8_t { LIFECYCLE_FLAGS_NONE = 0, @@ -599,7 +599,7 @@ struct alignas(64) ChipTaskSlotState { */ void reset_for_reuse() { fanout_lock.store(0, std::memory_order_relaxed); - fanout_count = PTO2_FANOUT_SCOPE_BIT; // bit31 = owning-scope ref; consumers ++ into low bits + fanout_count = FANOUT_SCOPE_BIT; // bit31 = owning-scope ref; consumers ++ into low bits fanout_head = nullptr; fanin_refcount.store(0, std::memory_order_relaxed); fanout_refcount.store(0, std::memory_order_relaxed); @@ -662,9 +662,9 @@ struct alignas(64) ChipTaskSlotState { }; static_assert(sizeof(ChipTaskSlotState) == 64); -// PTO2FaninSpillEntry packs DepFlags into the low bits of a ChipTaskSlotState*. +// FaninSpillEntry packs DepFlags into the low bits of a ChipTaskSlotState*. // That is only lossless while the type is aligned past those tag bits. static_assert( - alignof(ChipTaskSlotState) > PTO2FaninSpillEntry::FLAG_MASK, - "ChipTaskSlotState alignment must exceed PTO2FaninSpillEntry::FLAG_MASK so the packed DepFlags bits are free" + alignof(ChipTaskSlotState) > FaninSpillEntry::FLAG_MASK, + "ChipTaskSlotState alignment must exceed FaninSpillEntry::FLAG_MASK so the packed DepFlags bits are free" ); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 051ee9f793..4c297dc36b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -688,10 +688,10 @@ struct SchedulerState { check_and_handle_consumed(slot_state); } - // Scope-end release: sets bit31 (PTO2_FANOUT_SCOPE_BIT) instead of bumping a + // Scope-end release: sets bit31 (FANOUT_SCOPE_BIT) instead of bumping a // consumer ref. Called exactly once per task from on_scope_end. void release_producer_scope(ChipTaskSlotState &slot_state) { - slot_state.fanout_refcount.fetch_add(PTO2_FANOUT_SCOPE_BIT, std::memory_order_acq_rel); + slot_state.fanout_refcount.fetch_add(FANOUT_SCOPE_BIT, std::memory_order_acq_rel); check_and_handle_consumed(slot_state); } @@ -703,7 +703,7 @@ struct SchedulerState { } void release_producer_scope(ChipTaskSlotState &slot_state, uint64_t &atomic_count) { - slot_state.fanout_refcount.fetch_add(PTO2_FANOUT_SCOPE_BIT, std::memory_order_acq_rel); + slot_state.fanout_refcount.fetch_add(FANOUT_SCOPE_BIT, std::memory_order_acq_rel); atomic_count += 1; // fanout_refcount.fetch_add check_and_handle_consumed(slot_state, atomic_count); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index c98a19a53b..28f2bc5f80 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -326,7 +326,7 @@ OrchestratorLayout OrchestratorState::reserve_layout( for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { const size_t fanin_pool_bytes = - PTO2_ALIGN_UP(static_cast(dep_pool_capacities[r]) * sizeof(PTO2FaninSpillEntry), PTO2_ALIGN_SIZE); + PTO2_ALIGN_UP(static_cast(dep_pool_capacities[r]) * sizeof(FaninSpillEntry), PTO2_ALIGN_SIZE); layout.off_fanin_pool[r] = arena.reserve(fanin_pool_bytes, PTO2_ALIGN_SIZE); always_assert(task_window_sizes[r] > 0 && (task_window_sizes[r] & (task_window_sizes[r] - 1)) == 0); @@ -387,9 +387,9 @@ bool OrchestratorState::init_data_from_layout( heap_offset += heap_sizes[r]; const size_t fanin_pool_bytes = PTO2_ALIGN_UP( - static_cast(layout.dep_pool_capacities[r]) * sizeof(PTO2FaninSpillEntry), PTO2_ALIGN_SIZE + static_cast(layout.dep_pool_capacities[r]) * sizeof(FaninSpillEntry), PTO2_ALIGN_SIZE ); - auto *fanin_entries = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); + auto *fanin_entries = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); memset(fanin_entries, 0, fanin_pool_bytes); orch->rings[r].fanin_pool.init(fanin_entries, layout.dep_pool_capacities[r], orch_err); @@ -479,7 +479,7 @@ void OrchestratorState::wire_arena_pointers( ) { auto *orch = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - orch->rings[r].fanin_pool.base = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); + orch->rings[r].fanin_pool.base = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); orch->fanin_seen_epoch[r] = static_cast(arena.region_ptr(layout.off_fanin_seen_epoch[r])); } orch->tensor_map.wire_arena_pointers(layout.tensor_map, arena); diff --git a/src/a5/runtime/host_build_graph/host/dep_gen_host_graph.cpp b/src/a5/runtime/host_build_graph/host/dep_gen_host_graph.cpp index 9450b2cdaa..f57e115dc0 100644 --- a/src/a5/runtime/host_build_graph/host/dep_gen_host_graph.cpp +++ b/src/a5/runtime/host_build_graph/host/dep_gen_host_graph.cpp @@ -19,7 +19,7 @@ * tensormap — a producer whose written slice overlaps what this task reads * (STEP 3 Step B); carries both slices' geometry. * - * Per-task producer dedup mirrors PTO2FaninBuilder::append_fanin_or_fail, which + * Per-task producer dedup mirrors FaninBuilder::append_fanin_or_fail, which * collapses all three sources into one fanin list: the first edge to name a * producer is kept. tensormap edges are exempt — a second producer slice for the * same task is a distinct fact about the data flow, and viewers rely on seeing diff --git a/src/a5/runtime/host_build_graph/runtime/dep_gen_host_graph.h b/src/a5/runtime/host_build_graph/runtime/dep_gen_host_graph.h index ea840b0610..ad67de86dc 100644 --- a/src/a5/runtime/host_build_graph/runtime/dep_gen_host_graph.h +++ b/src/a5/runtime/host_build_graph/runtime/dep_gen_host_graph.h @@ -43,7 +43,7 @@ * progress loop satisfies this by being single-threaded; emit returns -3 if the * invariant is ever broken. * - * Per-task producer dedup mirrors PTO2FaninBuilder, which keys on (ring, slot); + * Per-task producer dedup mirrors FaninBuilder, which keys on (ring, slot); * this keys on producer task id. The two agree only because host_build_graph is * whole-graph-resident and never reuses a task slot at build time (see * append_fanin_or_fail in orchestrator.cpp). A runtime that recycles slots diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator.h b/src/a5/runtime/host_build_graph/runtime/orchestrator.h index a256314067..7a9d17cda4 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator.h @@ -109,7 +109,7 @@ struct OrchestratorState { // case (task_window tasks each at their full cap), so a bump cannot overflow. // The bases live here, not in the ring header: nothing on the device resolves one. // - // The fanin cursor does not advance at bind time. PTO2FaninBuilder appends and + // The fanin cursor does not advance at bind time. FaninBuilder appends and // dedups producers afterwards, so the region's length is known only when the count // is published, and it advances there. int32_t *fanin_pool{nullptr}; diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 95a30969e8..90983af1b4 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1137,11 +1137,11 @@ static uint32_t next_fanin_seen_epoch(OrchestratorState *orch) { // Polling: fanin is a flat array of position-independent producer local ids on // the payload (no dep-pool spill, no producer pointers). The builder writes them // directly into the payload's fanin region as producers are appended, deduping by -// slot and hard-capping at PTO2_MAX_FANIN. self_local is this task's own local id +// slot and hard-capping at CHIP_MAX_FANIN. self_local is this task's own local id // (the consumer), used to bump each producer's last_consumer_local_id (the // reclaim gate the host wait_for_consumers polls via completed_watermark). -struct PTO2FaninBuilder { - PTO2FaninBuilder(OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : +struct FaninBuilder { + FaninBuilder(OrchestratorState *orch, TaskPayload *payload, int32_t self_local, uint32_t seen_epoch) : count(0), orch(orch), seen_epoch(seen_epoch), @@ -1179,7 +1179,7 @@ struct PTO2FaninBuilder { static bool append_fanin_or_fail( OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, - TaskId producer_task_id, PTO2FaninBuilder *fanin_builder + TaskId producer_task_id, FaninBuilder *fanin_builder ) { // Skip a stale/reused producer slot: the cached owner id no longer resolves // to this producer (defensive — whole-graph-resident hbg does not reuse slots @@ -1192,15 +1192,15 @@ static bool append_fanin_or_fail( if (fanin_builder->mark_seen(prod_ring, prod_slot)) { return true; } - if (fanin_builder->count >= PTO2_MAX_FANIN) { + if (fanin_builder->count >= CHIP_MAX_FANIN) { LOG_ERROR("========================================"); LOG_ERROR("FATAL: Fanin Capacity Exhausted!"); LOG_ERROR("========================================"); LOG_ERROR("HBG stores every producer dependency in the consumer task's fanin region."); - LOG_ERROR(" Fanin: used=%d/%d", fanin_builder->count, PTO2_MAX_FANIN); + LOG_ERROR(" Fanin: used=%d/%d", fanin_builder->count, CHIP_MAX_FANIN); LOG_ERROR(" Requested: at least %d distinct producer dependencies", fanin_builder->count + 1); LOG_ERROR("Solution:"); - LOG_ERROR(" Reduce the task fanin to at most PTO2_MAX_FANIN=%d.", PTO2_MAX_FANIN); + LOG_ERROR(" Reduce the task fanin to at most CHIP_MAX_FANIN=%d.", CHIP_MAX_FANIN); LOG_ERROR(" HBG has no dependency spill pool; PTO2_RING_DEP_POOL does not apply."); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); @@ -1279,7 +1279,7 @@ static bool prepare_task( const int32_t scalar_span = PTO2_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); debug_assert(static_cast(orch->tensor_pool_cursor) + args.tensor_count() <= window * MAX_TENSOR_ARGS); debug_assert(static_cast(orch->scalar_pool_cursor) + scalar_span <= window * MAX_SCALAR_ARGS); - debug_assert(static_cast(orch->fanin_pool_cursor) + PTO2_MAX_FANIN <= window * PTO2_MAX_FANIN); + debug_assert(static_cast(orch->fanin_pool_cursor) + CHIP_MAX_FANIN <= window * CHIP_MAX_FANIN); out->payload->bind_regions( orch->tensor_pool + orch->tensor_pool_cursor, orch->scalar_pool + orch->scalar_pool_cursor, orch->fanin_pool + orch->fanin_pool_cursor @@ -1484,7 +1484,7 @@ static TaskOutputTensors submit_task_common( ); } - PTO2FaninBuilder fanin_builder(orch, &payload, static_cast(task_id.local()), next_fanin_seen_epoch(orch)); + FaninBuilder fanin_builder(orch, &payload, static_cast(task_id.local()), next_fanin_seen_epoch(orch)); CYCLE_COUNT_LAP(g_orch_alloc_cycle); @@ -1845,7 +1845,7 @@ bool graph_submit_outer( ); } - PTO2FaninBuilder fanin_builder(orch, &payload, static_cast(task_id.local()), next_fanin_seen_epoch(orch)); + FaninBuilder fanin_builder(orch, &payload, static_cast(task_id.local()), next_fanin_seen_epoch(orch)); auto emit = [&](TaskId producer_id) -> bool { const int32_t producer_local = static_cast(producer_id.local()); const int32_t producer_slot = ring.get_slot_by_task_id(producer_local); diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp index b321cedce4..1645a4061c 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp @@ -12,6 +12,6 @@ // The polling completion scheduler has no dep-pool / fanin-spill pool: producer // dependencies are position-independent local-id integers on the payload and // readiness is derived from the ring completion_flags. The pool ring buffers -// (PTO2FaninPool / PTO2DepListPool) that once lived here are gone, so this +// (FaninPool / PTO2DepListPool) that once lived here are gone, so this // translation unit carries no definitions. TaskAllocator's methods are // defined inline in ring_buffer.h. diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 039cf02c73..e68e65c232 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -132,13 +132,13 @@ inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HE #define PTO2_EARLY_DISPATCH_QUEUE_SIZE 64 // Fanin storage -#define PTO2_FANIN_INLINE_CAP 64 +#define CHIP_FANIN_INLINE_CAP 64 // Polling-scheduler fanin cap. The polling model stores producer dependencies as // flat position-independent local-id integers in the fanin pool (no dep-pool // spill), so a task's fanin degree is hard-capped here. Must cover the worst-case // fanin of any workload (paged_attention is the densest). -#define PTO2_MAX_FANIN 128 +#define CHIP_MAX_FANIN 128 // Alignment of every per-task region inside an argument pool. Each region starts // and ends on a cache line so TaskPayload::init's round-up scalar memcpy stays @@ -294,7 +294,7 @@ inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; struct TaskPayload { // === Cache line 0 (64B) — the dispatch path's own line === - // sizeof is independent of PTO2_MAX_FANIN / MAX_TENSOR_ARGS / MAX_SCALAR_ARGS: + // sizeof is independent of CHIP_MAX_FANIN / MAX_TENSOR_ARGS / MAX_SCALAR_ARGS: // widening a cap costs pool bytes for the tasks that need them, not a control // block on every task. int32_t tensor_count{0}; @@ -310,7 +310,7 @@ struct TaskPayload { // fanin holds flat position-independent producer local task ids. Single-ring // hbg: every producer is ring 0, so no per-edge ring id is stored. Scanned by // classify_fanin_state against the ring completion_flags. Hard-capped at - // PTO2_MAX_FANIN (no dep-pool spill). Unbound on a Graph node, whose + // CHIP_MAX_FANIN (no dep-pool spill). Unbound on a Graph node, whose // dependencies live in the Definition's fanin CSR instead. simpler::hbg::SelfRelativePtr tensors; simpler::hbg::SelfRelativePtr scalars; diff --git a/src/a5/runtime/host_build_graph/runtime/shared_memory.h b/src/a5/runtime/host_build_graph/runtime/shared_memory.h index 8eddaf82c4..018e9e5056 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a5/runtime/host_build_graph/runtime/shared_memory.h @@ -331,7 +331,7 @@ struct RingImageExtents { inline RingImageExtents mirror_extents(uint64_t task_window_size) noexcept { return RingImageExtents{ task_window_size, - task_window_size * PTO2_MAX_FANIN, + task_window_size * CHIP_MAX_FANIN, task_window_size * MAX_TENSOR_ARGS, task_window_size * MAX_SCALAR_ARGS, }; @@ -371,7 +371,7 @@ inline ChipRingSegmentOffsets ring_segment_offsets(uint64_t task_window_size) no // round-up scalar memcpy relies on. ChipTensor is 2 cache lines, so a tensor region // is aligned for any count; the fanin and scalar strides need it stated. static_assert( - (PTO2_MAX_FANIN * sizeof(int32_t)) % ARG_POOL_ALIGN == 0, + (CHIP_MAX_FANIN * sizeof(int32_t)) % ARG_POOL_ALIGN == 0, "the fanin region's cap must be a whole number of cache lines" ); static_assert( diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index ad6f04b56d..97e0223086 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -54,7 +54,7 @@ Bundles the three per-ring resources into a single aggregate (`ring_buffer.h`): struct ChipRingSet { ChipHeapRing heap_ring; ChipTaskRing task_ring; - PTO2FaninPool fanin_pool; + FaninPool fanin_pool; }; ``` diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index d41763eb91..542def3e3d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -444,7 +444,7 @@ Key members: | 0 | `ChipTensorMap::sync_tensormap` — prune stale TensorMap entries | | 1 | `TaskAllocator::alloc` — allocate task slot (may block on flow control) | | 2 | Initialize task descriptor + slot state, copy parameters | -| 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `PTO2FaninBuilder` | +| 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `FaninBuilder` | | 4 | **Insert**: register OUTPUT/INOUT args in TensorMap | | 5 | **Record fanin metadata**: store producer edges (slot pointer + `DepFlags` packed in the low bits) in `payload->fanin_inline_edges[]` (+ spill pool if >64); claim each live producer by incrementing `fanout_count` under that producer's `fanout_lock`. Creator edges are `DEP_WAIT\|DEP_RETAIN`, tensormap-modifier edges `DEP_WAIT`. This step runs **before** `payload.init()`. | | 6 | **Orch-side wiring / ready publish**: the orchestrator wires live fanout edges into the per-ring dep_pool; zero-fanin and already-completed fanin tasks publish directly to ready queues. Only `DEP_WAIT` edges gate readiness — they count toward `fanin_count` and are linked onto the producer's `fanout_head` for completion notification. A `DEP_WAIT`-only edge releases its submit→wire retention pin **at wiring** (and on the already-completed fast path), so its producer can be CONSUMED without waiting for this consumer; a `DEP_RETAIN` edge keeps the pin until this consumer's `on_task_release`. A hypothetical `RETAIN`-only edge (none exist yet) would neither gate readiness nor link a fanout node — it only holds the lifetime pin. | diff --git a/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp b/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp index 0e3ce5513f..f3bd97ea67 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp @@ -536,7 +536,7 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c TensorArgType atype_buf[CORE_MAX_TENSOR_ARGS]; // Per-record producer ID -> accumulated DepFlags — must match runtime's - // PTO2FaninBuilder::append_fanin_or_fail semantics, which collapses STEP 1 + // FaninBuilder::append_fanin_or_fail semantics, which collapses STEP 1 // (explicit_deps) + STEP A (creator retention) + STEP B (tensormap lookup) // into a single per-task fanin edge and OR-accumulates its flags. Both oracle // and annot use this same semantics so the divergence check compares the @@ -761,7 +761,7 @@ dep_gen_replay_emit_deps_json(const DepGenRecord *records, size_t num_records, c // the same producer (different version), or two different // producers, both yield their own edges. The producer-id-set // comparison below uses annot_preds, which dedups by pred - // only, matching runtime PTO2FaninBuilder semantics. + // only, matching runtime FaninBuilder semantics. annot_preds[producer.raw] |= DEP_WAIT; EdgeAnnot e{}; e.pred = producer.raw; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h b/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h index 02ac7feb48..9d1fa42753 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.h @@ -54,7 +54,7 @@ * are carried per-arg / per-edge alongside ``start_offset`` + ``strides``. * - Distinct producers / arg indices / sources keep their own edges; per-record * deduplication of producer ids mirrors the runtime - * ``PTO2FaninBuilder::append_fanin_or_fail`` semantics so the set of + * ``FaninBuilder::append_fanin_or_fail`` semantics so the set of * ``(pred, succ)`` pairs is identical to what the runtime would have * recorded. * diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 0f13084ebe..1412143a86 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -228,8 +228,8 @@ static uint32_t next_fanin_seen_epoch(OrchestratorState *orch) { return next; } -struct PTO2FaninBuilder { - PTO2FaninBuilder(OrchestratorState *orch, PTO2FaninPool &spill_pool, uint32_t seen_epoch) : +struct FaninBuilder { + FaninBuilder(OrchestratorState *orch, FaninPool &spill_pool, uint32_t seen_epoch) : count(0), wait_count(0), spill_start(0), @@ -241,11 +241,11 @@ struct PTO2FaninBuilder { int32_t spill_start{0}; OrchestratorState *orch{nullptr}; uint32_t seen_epoch{0}; - PTO2FaninPool &spill_pool; - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP]; + FaninPool &spill_pool; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP]; template - PTO2FaninForEachReturn for_each(Fn &&fn) const { + FaninForEachReturn for_each(Fn &&fn) const { return for_each_fanin_storage(inline_slots, count, spill_start, spill_pool, static_cast(fn)); } @@ -263,7 +263,7 @@ struct PTO2FaninBuilder { } // Append a new edge (caller has already claimed the producer's fanout pin). - void push_edge(PTO2FaninSpillEntry &entry, ChipTaskSlotState *prod_state, DepFlags kind) { + void push_edge(FaninSpillEntry &entry, ChipTaskSlotState *prod_state, DepFlags kind) { entry.set(prod_state, kind); count++; if (dep_has_wait(kind)) { @@ -276,7 +276,7 @@ struct PTO2FaninBuilder { // edge instead of adding a second one — no extra fanout pin is claimed. void or_flags_into_existing(ChipTaskSlotState *prod_state, DepFlags kind) { for (int32_t i = 0; i < count; i++) { - PTO2FaninSpillEntry &entry = entry_at(i); + FaninSpillEntry &entry = entry_at(i); if (entry.slot_state() == prod_state) { accumulate_flags(entry, kind); return; @@ -289,18 +289,18 @@ struct PTO2FaninBuilder { } private: - // The i-th appended fanin edge: inline for i < PTO2_FANIN_INLINE_CAP, else in + // The i-th appended fanin edge: inline for i < CHIP_FANIN_INLINE_CAP, else in // the spill ring at the same linear->physical position for_each_fanin_storage // walks. Single source of the wrap arithmetic. - PTO2FaninSpillEntry &entry_at(int32_t i) { - if (i < PTO2_FANIN_INLINE_CAP) { + FaninSpillEntry &entry_at(int32_t i) { + if (i < CHIP_FANIN_INLINE_CAP) { return inline_slots[i]; } - int32_t spill_idx = i - PTO2_FANIN_INLINE_CAP; + int32_t spill_idx = i - CHIP_FANIN_INLINE_CAP; return spill_pool.base[(spill_start % spill_pool.capacity + spill_idx) % spill_pool.capacity]; } - void accumulate_flags(PTO2FaninSpillEntry &entry, DepFlags kind) { + void accumulate_flags(FaninSpillEntry &entry, DepFlags kind) { bool had_wait = dep_has_wait(entry.flags()); entry.add_flags(kind); if (!had_wait && dep_has_wait(kind)) { @@ -311,7 +311,7 @@ struct PTO2FaninBuilder { static bool append_fanin_or_fail( OrchestratorState *orch, uint8_t prod_ring, int32_t prod_slot, ChipTaskSlotState *prod_state, - TaskId producer_task_id, PTO2FaninBuilder *fanin_builder, uint8_t ring_id, DepFlags kind + TaskId producer_task_id, FaninBuilder *fanin_builder, uint8_t ring_id, DepFlags kind ) { // Decide-and-claim under the producer's fanout_lock. Two conditions make this // resolved slot a non-dependency, and both must be checked together with the @@ -353,11 +353,11 @@ static bool append_fanin_or_fail( // count must never carry into bit31 (would corrupt the scope-release // flag) — true for any sane fanout (<< 2^31). assert( - (prod_state->fanout_count & ~PTO2_FANOUT_SCOPE_BIT) < (PTO2_FANOUT_SCOPE_BIT - 1) && + (prod_state->fanout_count & ~FANOUT_SCOPE_BIT) < (FANOUT_SCOPE_BIT - 1) && "fanout consumer count overflow into scope bit" ); prod_state->fanout_count++; - fanout_now = static_cast(prod_state->fanout_count & ~PTO2_FANOUT_SCOPE_BIT); + fanout_now = static_cast(prod_state->fanout_count & ~FANOUT_SCOPE_BIT); } prod_state->unlock_fanout(); #if SIMPLER_ORCH_PROFILING @@ -383,30 +383,30 @@ static bool append_fanin_or_fail( return true; } - if (fanin_builder->count < PTO2_FANIN_INLINE_CAP) { + if (fanin_builder->count < CHIP_FANIN_INLINE_CAP) { fanin_builder->push_edge(fanin_builder->inline_slots[fanin_builder->count], prod_state, kind); return true; } - PTO2FaninPool &fanin_pool = fanin_builder->spill_pool; + FaninPool &fanin_pool = fanin_builder->spill_pool; if (!fanin_pool.ensure_space(orch->sm_header->rings[ring_id], 1)) { orch_mark_fatal(orch, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; } int32_t spill_idx = fanin_pool.top; - PTO2FaninSpillEntry *entry = fanin_pool.alloc(); + FaninSpillEntry *entry = fanin_pool.alloc(); if (entry == nullptr) { orch_mark_fatal(orch, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; } - if (fanin_builder->count == PTO2_FANIN_INLINE_CAP) { + if (fanin_builder->count == CHIP_FANIN_INLINE_CAP) { fanin_builder->spill_start = spill_idx; } fanin_builder->push_edge(*entry, prod_state, kind); return true; } -static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) { +static bool all_claimed_fanin_completed(const FaninBuilder &fanin_builder) { if (fanin_builder.count == 0) return true; // Only DEP_WAIT edges gate readiness; a retention-only edge never blocks // dispatch, so it is treated as satisfied here. @@ -416,7 +416,7 @@ static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) { }); } -static bool all_claimed_fanin_allow_early_resolve(const PTO2FaninBuilder &fanin_builder) { +static bool all_claimed_fanin_allow_early_resolve(const FaninBuilder &fanin_builder) { if (fanin_builder.count == 0) return true; return fanin_builder.for_each([](ChipTaskSlotState *producer, DepFlags flags) -> bool { if (!dep_has_wait(flags)) return true; @@ -939,7 +939,7 @@ static TaskOutputTensors submit_task_common( } #endif - PTO2FaninBuilder fanin_builder(orch, orch->rings[ring_id].fanin_pool, next_fanin_seen_epoch(orch)); + FaninBuilder fanin_builder(orch, orch->rings[ring_id].fanin_pool, next_fanin_seen_epoch(orch)); CYCLE_COUNT_LAP(g_orch_alloc_cycle); @@ -1034,7 +1034,7 @@ static TaskOutputTensors submit_task_common( // the producer's fanout_lock. Doing it there (rather than a separate pass // here) is what prevents a producer from transitioning to CONSUMED between // the dependency decision and the claim. - int32_t inline_count = std::min(fanin_builder.count, PTO2_FANIN_INLINE_CAP); + int32_t inline_count = std::min(fanin_builder.count, CHIP_FANIN_INLINE_CAP); // Every fanin edge produced here carries DEP_WAIT (creator = WAIT|RETAIN, // modifier = WAIT, explicit defaults to WAIT|RETAIN or opts into WAIT), so // wait_count == count. fanin_actual_count therefore doubles as the WAIT-edge diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index 5534a7867b..80d1656997 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -34,7 +34,7 @@ static void latch_pool_error(std::atomic *error_code_ptr, int32_t error // ============================================================================= // Fanin Spill Pool Implementation // ============================================================================= -void PTO2FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { +void FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { if (sm_last_task_alive <= reclaim_task_cursor) return; int32_t scan_end = sm_last_task_alive; @@ -44,7 +44,7 @@ void PTO2FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_a continue; } - int32_t inline_count = std::min(payload.fanin_actual_count, PTO2_FANIN_INLINE_CAP); + int32_t inline_count = std::min(payload.fanin_actual_count, CHIP_FANIN_INLINE_CAP); int32_t spill_edge_count = payload.fanin_actual_count - inline_count; if (spill_edge_count > 0) { advance_tail(payload.fanin_spill_start + spill_edge_count); @@ -53,7 +53,7 @@ void PTO2FaninPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_a reclaim_task_cursor = scan_end; } -bool PTO2FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { +bool FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { if (available() >= needed) return true; int spin_count = 0; @@ -178,8 +178,8 @@ void PTO2DepListPool::report_deadlock( uint32_t rc = h.fanout_refcount.load(std::memory_order_acquire); LOG_ERROR( " Head task %d: state=%d, consumers=%u/%u, scope_released=%d", last_alive, - static_cast(h.task_state.load(std::memory_order_acquire)), rc & ~PTO2_FANOUT_SCOPE_BIT, - fc & ~PTO2_FANOUT_SCOPE_BIT, (rc & PTO2_FANOUT_SCOPE_BIT) ? 1 : 0 + static_cast(h.task_state.load(std::memory_order_acquire)), rc & ~FANOUT_SCOPE_BIT, + fc & ~FANOUT_SCOPE_BIT, (rc & FANOUT_SCOPE_BIT) ? 1 : 0 ); } LOG_ERROR("Solution:"); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 1bdab7a9e1..0fc41945e0 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -529,8 +529,8 @@ class TaskAllocator { uint32_t rc = h.fanout_refcount.load(std::memory_order_acquire); LOG_ERROR( " Head task %d: state=%d, consumers=%u/%u, scope_released=%d", last_alive, - static_cast(h.task_state.load(std::memory_order_acquire)), rc & ~PTO2_FANOUT_SCOPE_BIT, - fc & ~PTO2_FANOUT_SCOPE_BIT, (rc & PTO2_FANOUT_SCOPE_BIT) ? 1 : 0 + static_cast(h.task_state.load(std::memory_order_acquire)), rc & ~FANOUT_SCOPE_BIT, + fc & ~FANOUT_SCOPE_BIT, (rc & FANOUT_SCOPE_BIT) ? 1 : 0 ); } LOG_ERROR("Solution:"); @@ -576,8 +576,8 @@ class TaskAllocator { * Linear counters (top, tail) grow monotonically; the physical index * is obtained via modulo: base[linear_index % capacity]. */ -struct PTO2FaninPool { - PTO2FaninSpillEntry *base; // Pool base address +struct FaninPool { + FaninSpillEntry *base; // Pool base address int32_t capacity; // Total number of entries int32_t top; // Linear next-allocation counter (starts from 1) int32_t tail; // Linear first-alive counter (entries before this are dead) @@ -589,7 +589,7 @@ struct PTO2FaninPool { std::atomic *reclaim_ack_mask = nullptr; uint8_t ring_id = 0; - void init(PTO2FaninSpillEntry *in_base, int32_t in_capacity, std::atomic *in_error_code_ptr) { + void init(FaninSpillEntry *in_base, int32_t in_capacity, std::atomic *in_error_code_ptr) { base = in_base; capacity = in_capacity; top = 1; @@ -630,7 +630,7 @@ struct PTO2FaninPool { bool ensure_space(SharedMemoryRingHeader &ring, int32_t needed); - PTO2FaninSpillEntry *alloc() { + FaninSpillEntry *alloc() { int32_t used = top - tail; if (used >= capacity) { LOG_ERROR("========================================"); @@ -669,25 +669,25 @@ struct PTO2FaninPool { }; template -using PTO2FaninCallbackResult = std::invoke_result_t; +using FaninCallbackResult = std::invoke_result_t; template -using PTO2FaninForEachReturn = std::conditional_t, void>, void, bool>; +using FaninForEachReturn = std::conditional_t, void>, void, bool>; // Visit each fanin edge as (producer slot, DepFlags). Inline and spill entries -// share the packed PTO2FaninSpillEntry layout, so both are unpacked the same way. +// share the packed FaninSpillEntry layout, so both are unpacked the same way. template -inline PTO2FaninForEachReturn for_each_fanin_storage( - InlineSlots &&inline_edges, int32_t fanin_count, int32_t spill_start, PTO2FaninPool &spill_pool, Fn &&fn +inline FaninForEachReturn for_each_fanin_storage( + InlineSlots &&inline_edges, int32_t fanin_count, int32_t spill_start, FaninPool &spill_pool, Fn &&fn ) { - using FaninCallbackResult = PTO2FaninCallbackResult; + using FaninCallbackResult = FaninCallbackResult; static_assert( std::is_same_v || std::is_same_v, "fanin callback must return void or bool" ); if constexpr (std::is_void_v) { - int32_t inline_count = std::min(fanin_count, PTO2_FANIN_INLINE_CAP); + int32_t inline_count = std::min(fanin_count, CHIP_FANIN_INLINE_CAP); for (int32_t i = 0; i < inline_count; i++) { fn(inline_edges[i].slot_state(), inline_edges[i].flags()); } @@ -699,7 +699,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( int32_t start_idx = spill_start % spill_pool.capacity; int32_t first_count = std::min(spill_count, spill_pool.capacity - start_idx); - PTO2FaninSpillEntry *first = spill_pool.base + start_idx; + FaninSpillEntry *first = spill_pool.base + start_idx; for (int32_t i = 0; i < first_count; i++) { fn(first[i].slot_state(), first[i].flags()); } @@ -710,7 +710,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( } return; } else { - int32_t inline_count = std::min(fanin_count, PTO2_FANIN_INLINE_CAP); + int32_t inline_count = std::min(fanin_count, CHIP_FANIN_INLINE_CAP); for (int32_t i = 0; i < inline_count; i++) { if (!fn(inline_edges[i].slot_state(), inline_edges[i].flags())) { return false; @@ -724,7 +724,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( int32_t start_idx = spill_start % spill_pool.capacity; int32_t first_count = std::min(spill_count, spill_pool.capacity - start_idx); - PTO2FaninSpillEntry *first = spill_pool.base + start_idx; + FaninSpillEntry *first = spill_pool.base + start_idx; for (int32_t i = 0; i < first_count; i++) { if (!fn(first[i].slot_state(), first[i].flags())) { return false; @@ -742,7 +742,7 @@ inline PTO2FaninForEachReturn for_each_fanin_storage( } template -inline PTO2FaninForEachReturn for_each_fanin_slot_state(const TaskPayload &payload, Fn &&fn) { +inline FaninForEachReturn for_each_fanin_slot_state(const TaskPayload &payload, Fn &&fn) { return for_each_fanin_storage( payload.fanin_inline_edges, payload.fanin_actual_count, payload.fanin_spill_start, *payload.fanin_spill_pool, static_cast(fn) @@ -925,7 +925,7 @@ struct PTO2DepListPool { */ struct ChipRingSet { TaskAllocator task_allocator; - PTO2FaninPool fanin_pool; + FaninPool fanin_pool; }; #endif // PTO_RING_BUFFER_H diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 4ad31ea0fe..e3485bec80 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -143,8 +143,8 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo int32_t local_id = slot.task->task_id.local(); uint64_t t0 = get_sys_cnt_aicpu(); int32_t spin_count = 0; - while ((slot.fanout_refcount.load(std::memory_order_acquire) & ~PTO2_FANOUT_SCOPE_BIT) < - (slot.fanout_count & ~PTO2_FANOUT_SCOPE_BIT)) { + while ((slot.fanout_refcount.load(std::memory_order_acquire) & ~FANOUT_SCOPE_BIT) < + (slot.fanout_count & ~FANOUT_SCOPE_BIT)) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { // A fatal latched elsewhere breaks this wait; cold path only. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 2eebda93da..6cc9de5ffe 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -94,7 +94,7 @@ #define PTO2_EARLY_DISPATCH_QUEUE_SIZE 256 // Fanin storage -#define PTO2_FANIN_INLINE_CAP 64 +#define CHIP_FANIN_INLINE_CAP 64 // Dependency-degree diagnostic: log once at debug level when a task's fanin or // a producer's fanout first exceeds this degree, so dense dependency graphs can @@ -167,18 +167,18 @@ struct PTO2OutputLayout { * Stored in the dedicated fanin spill ring buffer. */ struct ChipTaskSlotState; // Forward declaration -struct PTO2FaninPool; // Forward declaration +struct FaninPool; // Forward declaration // One fanin edge: a producer slot pointer with the per-edge DepFlags packed into // bits 0..1 of the pointer. ChipTaskSlotState is alignas(64), so bits 0..5 are // always zero in a real pointer. Used for both the inline fanin array and the // spill pool, keeping sizeof == sizeof(uintptr_t) so neither footprint grows. -struct PTO2FaninSpillEntry { +struct FaninSpillEntry { static constexpr uintptr_t FLAG_MASK = 0x3; // No in-class initializer: the type stays trivially default-constructible so // the payload's fanin_inline_edges[64] and the orchestrator's per-submit // builder array are default-initialized at zero cost (only entries [0, count) - // are written via set()). Value-init (`PTO2FaninSpillEntry{}`) still zeroes it. + // are written via set()). Value-init (`FaninSpillEntry{}`) still zeroes it. uintptr_t packed; ChipTaskSlotState *slot_state() const { return reinterpret_cast(packed & ~FLAG_MASK); } @@ -191,7 +191,7 @@ struct PTO2FaninSpillEntry { void add_flags(DepFlags f) { packed |= (static_cast(f) & FLAG_MASK); } void clear() { packed = 0; } }; -static_assert(sizeof(PTO2FaninSpillEntry) == sizeof(uintptr_t)); +static_assert(sizeof(FaninSpillEntry) == sizeof(uintptr_t)); /** * Dependency list entry (singly-linked list node) @@ -280,11 +280,11 @@ struct TaskPayload { int32_t scalar_count{0}; int32_t fanin_actual_count{0}; // Actual fanin count (without the +1 redundance) int32_t fanin_spill_start{0}; // Linear start index in fanin spill pool (0 = no spill) - PTO2FaninPool *fanin_spill_pool{nullptr}; + FaninPool *fanin_spill_pool{nullptr}; // Inline fanin edges (producer slot + packed DepFlags). Spill beyond - // PTO2_FANIN_INLINE_CAP goes to fanin_spill_pool. Same packed layout as the + // CHIP_FANIN_INLINE_CAP goes to fanin_spill_pool. Same packed layout as the // spill entries, so the array footprint is unchanged. - PTO2FaninSpillEntry fanin_inline_edges[PTO2_FANIN_INLINE_CAP]; + FaninSpillEntry fanin_inline_edges[CHIP_FANIN_INLINE_CAP]; // Early-dispatch metadata (AICPU-side only). Ordered by descending // alignment so the block packs without internal padding. Cache line 8 // contains the rarely-touched fanin tail rather than the hot tensor/scalar @@ -444,18 +444,18 @@ static_assert( // fanout_count / fanout_refcount bit encoding (both uint32): // bits [30:0] = consumer references (count: # consumers; refcount: # released) -// bit [31] = the owning scope's reference (PTO2_FANOUT_SCOPE_BIT) -// fanout_count is seeded to PTO2_FANOUT_SCOPE_BIT and ++'d per consumer, so it +// bit [31] = the owning scope's reference (FANOUT_SCOPE_BIT) +// fanout_count is seeded to FANOUT_SCOPE_BIT and ++'d per consumer, so it // ends as (SCOPE_BIT | num_consumers). release adds 1 (consumer completion) or // SCOPE_BIT (scope_end). CONSUMED iff fanout_refcount == fanout_count (every // consumer released AND scope bit set). Keeping the scope ref in a distinct bit // (rather than folding scope + consumers into one count) lets a consumer reach -// fanout_refcount == (fanout_count & ~PTO2_FANOUT_SCOPE_BIT) while the scope bit +// fanout_refcount == (fanout_count & ~FANOUT_SCOPE_BIT) while the scope bit // is still unset -- i.e. "all consumers done but scope still open" stays // distinguishable from "fully consumed". Structural allocation deadlock is // detected separately by checking whether the blocking ring head is the oldest // task pinned by any open scope on that ring. -static constexpr uint32_t PTO2_FANOUT_SCOPE_BIT = 0x80000000u; +static constexpr uint32_t FANOUT_SCOPE_BIT = 0x80000000u; enum TaskLifecycleFlag : uint8_t { LIFECYCLE_FLAGS_NONE = 0, @@ -599,7 +599,7 @@ struct alignas(64) ChipTaskSlotState { */ void reset_for_reuse() { fanout_lock.store(0, std::memory_order_relaxed); - fanout_count = PTO2_FANOUT_SCOPE_BIT; // bit31 = owning-scope ref; consumers ++ into low bits + fanout_count = FANOUT_SCOPE_BIT; // bit31 = owning-scope ref; consumers ++ into low bits fanout_head = nullptr; fanin_refcount.store(0, std::memory_order_relaxed); fanout_refcount.store(0, std::memory_order_relaxed); @@ -662,9 +662,9 @@ struct alignas(64) ChipTaskSlotState { }; static_assert(sizeof(ChipTaskSlotState) == 64); -// PTO2FaninSpillEntry packs DepFlags into the low bits of a ChipTaskSlotState*. +// FaninSpillEntry packs DepFlags into the low bits of a ChipTaskSlotState*. // That is only lossless while the type is aligned past those tag bits. static_assert( - alignof(ChipTaskSlotState) > PTO2FaninSpillEntry::FLAG_MASK, - "ChipTaskSlotState alignment must exceed PTO2FaninSpillEntry::FLAG_MASK so the packed DepFlags bits are free" + alignof(ChipTaskSlotState) > FaninSpillEntry::FLAG_MASK, + "ChipTaskSlotState alignment must exceed FaninSpillEntry::FLAG_MASK so the packed DepFlags bits are free" ); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index c902864f72..30d497d4db 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -733,13 +733,13 @@ struct SchedulerState { check_and_handle_consumed(slot_state); } - // Scope-end release: sets bit31 (PTO2_FANOUT_SCOPE_BIT) instead of bumping a + // Scope-end release: sets bit31 (FANOUT_SCOPE_BIT) instead of bumping a // consumer ref. Called exactly once per task from on_scope_end. Keeping it a // distinct add lets a consumer release leave the scope bit unset, so "all // consumers done but scope still open" stays distinguishable from "fully // consumed". void release_producer_scope(ChipTaskSlotState &slot_state) { - slot_state.fanout_refcount.fetch_add(PTO2_FANOUT_SCOPE_BIT, std::memory_order_acq_rel); + slot_state.fanout_refcount.fetch_add(FANOUT_SCOPE_BIT, std::memory_order_acq_rel); check_and_handle_consumed(slot_state); } @@ -751,7 +751,7 @@ struct SchedulerState { } void release_producer_scope(ChipTaskSlotState &slot_state, uint64_t &atomic_count) { - slot_state.fanout_refcount.fetch_add(PTO2_FANOUT_SCOPE_BIT, std::memory_order_acq_rel); + slot_state.fanout_refcount.fetch_add(FANOUT_SCOPE_BIT, std::memory_order_acq_rel); atomic_count += 1; // fanout_refcount.fetch_add check_and_handle_consumed(slot_state, atomic_count); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 5c3687d237..b60e1f6331 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -343,7 +343,7 @@ OrchestratorLayout OrchestratorState::reserve_layout( for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { const size_t fanin_pool_bytes = - PTO2_ALIGN_UP(static_cast(dep_pool_capacities[r]) * sizeof(PTO2FaninSpillEntry), PTO2_ALIGN_SIZE); + PTO2_ALIGN_UP(static_cast(dep_pool_capacities[r]) * sizeof(FaninSpillEntry), PTO2_ALIGN_SIZE); layout.off_fanin_pool[r] = arena.reserve(fanin_pool_bytes, PTO2_ALIGN_SIZE); always_assert(task_window_sizes[r] > 0 && (task_window_sizes[r] & (task_window_sizes[r] - 1)) == 0); @@ -404,9 +404,9 @@ bool OrchestratorState::init_data_from_layout( heap_offset += heap_sizes[r]; const size_t fanin_pool_bytes = PTO2_ALIGN_UP( - static_cast(layout.dep_pool_capacities[r]) * sizeof(PTO2FaninSpillEntry), PTO2_ALIGN_SIZE + static_cast(layout.dep_pool_capacities[r]) * sizeof(FaninSpillEntry), PTO2_ALIGN_SIZE ); - auto *fanin_entries = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); + auto *fanin_entries = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); memset(fanin_entries, 0, fanin_pool_bytes); orch->rings[r].fanin_pool.init(fanin_entries, layout.dep_pool_capacities[r], orch_err); @@ -501,7 +501,7 @@ void OrchestratorState::wire_arena_pointers( ) { auto *orch = this; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - orch->rings[r].fanin_pool.base = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); + orch->rings[r].fanin_pool.base = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); orch->fanin_seen_epoch[r] = static_cast(arena.region_ptr(layout.off_fanin_seen_epoch[r])); } orch->tensor_map.wire_arena_pointers(layout.tensor_map, arena); diff --git a/src/common/runtime_status/error_names.h b/src/common/runtime_status/error_names.h index 5f542db01e..40382e54de 100644 --- a/src/common/runtime_status/error_names.h +++ b/src/common/runtime_status/error_names.h @@ -144,7 +144,7 @@ static inline const char *error_hint(int32_t code) { "runtime also check for nested submission or a stalled consumer"; case SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED: return "follow the preceding resource diagnostic: HBG has no spill pool, so reduce fanin to " - "PTO2_MAX_FANIN or less; for a reclaiming runtime raise PTO2_RING_DEP_POOL or cut fanin"; + "CHIP_MAX_FANIN or less; for a reclaiming runtime raise PTO2_RING_DEP_POOL or cut fanin"; case SIMPLER_ERROR_INVALID_ARGS: return "an orchestration bug, not a capacity problem -- resizing the rings will not help; " "recheck the arguments of the orchestration API calls listed above"; diff --git a/tests/st/runtime_fatal_codes/kernels/orchestration/dep_pool_overflow_orch.cpp b/tests/st/runtime_fatal_codes/kernels/orchestration/dep_pool_overflow_orch.cpp index c4e2589e92..5eeff07adf 100644 --- a/tests/st/runtime_fatal_codes/kernels/orchestration/dep_pool_overflow_orch.cpp +++ b/tests/st/runtime_fatal_codes/kernels/orchestration/dep_pool_overflow_orch.cpp @@ -13,13 +13,13 @@ * Negative ST orchestration: SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED (code 4). * * One consumer with more fanin edges than the dep pool can hold. The first - * PTO2_FANIN_INLINE_CAP (64) edges live in the consumer's inline fanin slots; + * CHIP_FANIN_INLINE_CAP (64) edges live in the consumer's inline fanin slots; * only edge 65+ spills into the per-ring fanin spill pool. With the pool pinned * to 4 entries (CallConfig.runtime_env.ring_dep_pool) and 64 spill edges that * all belong to the one in-construction consumer (so none can be reclaimed), the * spill allocator exhausts and the orchestrator latches FANIN_CAPACITY_EXCEEDED. * - * PRODUCER_COUNT must exceed PTO2_FANIN_INLINE_CAP, otherwise every edge fits + * PRODUCER_COUNT must exceed CHIP_FANIN_INLINE_CAP, otherwise every edge fits * inline and the spill pool is never touched. With Orch-side dependency wiring, * already-completed producer fanins bypass the dep pool entirely, so the old * inline-boundary scheduler-timeout fixture is no longer a stable public-API @@ -31,7 +31,7 @@ #include "orchestration_api.h" // NOLINT(build/include_subdir) -static constexpr int32_t PRODUCER_COUNT = 128; // > PTO2_FANIN_INLINE_CAP (64) +static constexpr int32_t PRODUCER_COUNT = 128; // > CHIP_FANIN_INLINE_CAP (64) extern "C" { diff --git a/tests/ut/cpp/a2a3/test_fanin_pool.cpp b/tests/ut/cpp/a2a3/test_fanin_pool.cpp index 586539b36e..da42e597df 100644 --- a/tests/ut/cpp/a2a3/test_fanin_pool.cpp +++ b/tests/ut/cpp/a2a3/test_fanin_pool.cpp @@ -9,11 +9,11 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2FaninPool and for_each_fanin_storage/slot_state + * Unit tests for FaninPool and for_each_fanin_storage/slot_state * from ring_buffer.h / ring_buffer.cpp * * Tests: - * 1. PTO2FaninPool — ring buffer allocation, overflow, tail advance, + * 1. FaninPool — ring buffer allocation, overflow, tail advance, * high-water tracking * 2. for_each_fanin_storage — inline-only, spill without wrap, * spill with wrap, callback early return @@ -36,12 +36,12 @@ class FaninPoolTest : public ::testing::Test { protected: static constexpr int32_t POOL_CAP = 32; - std::vector entries; + std::vector entries; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2FaninPool pool{}; + FaninPool pool{}; void SetUp() override { - entries.assign(POOL_CAP, PTO2FaninSpillEntry{}); + entries.assign(POOL_CAP, FaninSpillEntry{}); error_code.store(SIMPLER_ERROR_NONE); pool.init(entries.data(), POOL_CAP, &error_code); } @@ -158,14 +158,14 @@ class ForEachFaninTest : public ::testing::Test { protected: static constexpr int32_t POOL_CAP = 32; - std::vector spill_entries; + std::vector spill_entries; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2FaninPool spill_pool{}; + FaninPool spill_pool{}; alignas(64) ChipTaskSlotState slots[64]; void SetUp() override { - spill_entries.assign(POOL_CAP, PTO2FaninSpillEntry{}); + spill_entries.assign(POOL_CAP, FaninSpillEntry{}); error_code.store(SIMPLER_ERROR_NONE); spill_pool.init(spill_entries.data(), POOL_CAP, &error_code); memset(slots, 0, sizeof(slots)); @@ -173,7 +173,7 @@ class ForEachFaninTest : public ::testing::Test { }; TEST_F(ForEachFaninTest, InlineOnlyVoid) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; for (int i = 0; i < 5; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -190,7 +190,7 @@ TEST_F(ForEachFaninTest, InlineOnlyVoid) { } TEST_F(ForEachFaninTest, InlineOnlyBoolEarlyReturn) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; for (int i = 0; i < 5; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -206,7 +206,7 @@ TEST_F(ForEachFaninTest, InlineOnlyBoolEarlyReturn) { } TEST_F(ForEachFaninTest, InlineOnlyBoolAllTrue) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; for (int i = 0; i < 3; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -219,7 +219,7 @@ TEST_F(ForEachFaninTest, InlineOnlyBoolAllTrue) { } TEST_F(ForEachFaninTest, ZeroFanin) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; int count = 0; for_each_fanin_storage(inline_slots, 0, 0, spill_pool, [&](ChipTaskSlotState *, DepFlags) { count++; @@ -233,8 +233,8 @@ TEST_F(ForEachFaninTest, ZeroFanin) { TEST_F(ForEachFaninTest, SpillNoWrap) { // 18 fanins = 16 inline + 2 spill - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; - for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; + for (int i = 0; i < CHIP_FANIN_INLINE_CAP; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -268,8 +268,8 @@ TEST_F(ForEachFaninTest, SpillWithWrap) { spill_pool.top = POOL_CAP - 2; spill_pool.tail = POOL_CAP - 2; - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; - for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; + for (int i = 0; i < CHIP_FANIN_INLINE_CAP; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -302,8 +302,8 @@ TEST_F(ForEachFaninTest, SpillWithWrap) { // ============================================================================= TEST_F(ForEachFaninTest, SpillBoolEarlyReturnInSpillRegion) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; - for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; + for (int i = 0; i < CHIP_FANIN_INLINE_CAP; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -325,16 +325,16 @@ TEST_F(ForEachFaninTest, SpillBoolEarlyReturnInSpillRegion) { } // ============================================================================= -// PTO2FaninSpillEntry: DepFlags packing round-trips across inline and spill +// FaninSpillEntry: DepFlags packing round-trips across inline and spill // ============================================================================= TEST_F(ForEachFaninTest, DepFlagsRoundTripInlineAndSpill) { // Fill all 64 inline edges; slots 0..2 carry distinct flag combinations. - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; inline_slots[0].set(&slots[0], DEP_WAIT); inline_slots[1].set(&slots[1], DEP_RETAIN); inline_slots[2].set(&slots[2], DEP_WAIT | DEP_RETAIN); - for (int i = 3; i < PTO2_FANIN_INLINE_CAP; i++) { + for (int i = 3; i < CHIP_FANIN_INLINE_CAP; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -345,7 +345,7 @@ TEST_F(ForEachFaninTest, DepFlagsRoundTripInlineAndSpill) { auto *s1 = spill_pool.alloc(); s1->set(&slots[1], DEP_WAIT | DEP_RETAIN); - const int32_t total = PTO2_FANIN_INLINE_CAP + 2; // 64 inline + 2 spill + const int32_t total = CHIP_FANIN_INLINE_CAP + 2; // 64 inline + 2 spill std::vector flags; for_each_fanin_storage(inline_slots, total, spill_start, spill_pool, [&](ChipTaskSlotState *, DepFlags f) { flags.push_back(f); @@ -357,6 +357,6 @@ TEST_F(ForEachFaninTest, DepFlagsRoundTripInlineAndSpill) { EXPECT_EQ(flags[1], DEP_RETAIN); EXPECT_EQ(flags[2], DEP_WAIT | DEP_RETAIN); // Spill flags survive. - EXPECT_EQ(flags[PTO2_FANIN_INLINE_CAP + 0], DEP_RETAIN); - EXPECT_EQ(flags[PTO2_FANIN_INLINE_CAP + 1], DEP_WAIT | DEP_RETAIN); + EXPECT_EQ(flags[CHIP_FANIN_INLINE_CAP + 0], DEP_RETAIN); + EXPECT_EQ(flags[CHIP_FANIN_INLINE_CAP + 1], DEP_WAIT | DEP_RETAIN); } diff --git a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp index 89c6c02f4c..9afc901a64 100644 --- a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp @@ -152,7 +152,7 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { EXPECT_LE(cflag, uint8_t{1}); // Payload counts are real, not the poison bit pattern. EXPECT_GE(pl.fanin_count, 0); - EXPECT_LE(pl.fanin_count, PTO2_MAX_FANIN); + EXPECT_LE(pl.fanin_count, CHIP_MAX_FANIN); EXPECT_GE(pl.tensor_count, 0); EXPECT_GE(pl.scalar_count, 0); // predicate.op is a dispatch-time field, read only for tasks the device diff --git a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp index 3847b48146..b4cb662b86 100644 --- a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp @@ -90,10 +90,10 @@ TEST_F(OrchestratorFaninTest, DuplicateExplicitProducerAddsOneFanin) { EXPECT_EQ(consumer_slot.payload->fanin_inline_edges[0].slot_state(), &producer_slot); // A plain set_dependencies() dep is conservative RETAIN: DEP_WAIT|DEP_RETAIN. EXPECT_EQ(consumer_slot.payload->fanin_inline_edges[0].flags(), DEP_WAIT | DEP_RETAIN); - // fanout_count is bit-packed: bit31 (PTO2_FANOUT_SCOPE_BIT) is the owning-scope + // fanout_count is bit-packed: bit31 (FANOUT_SCOPE_BIT) is the owning-scope // ref, low bits the consumer count. The duplicate explicit dep is deduped to a // single consumer, so this is scope + 1. - EXPECT_EQ(producer_slot.fanout_count, PTO2_FANOUT_SCOPE_BIT + 1); + EXPECT_EQ(producer_slot.fanout_count, FANOUT_SCOPE_BIT + 1); } // An explicit ordering-only dep (the primitive add_dep_wait() lowers to) yields a @@ -142,7 +142,7 @@ TEST_F(OrchestratorFaninTest, DuplicateProducerOrAccumulatesFlags) { ASSERT_NE(consumer_slot.payload, nullptr); ASSERT_EQ(consumer_slot.payload->fanin_actual_count, 1); EXPECT_EQ(consumer_slot.payload->fanin_inline_edges[0].flags(), DEP_WAIT | DEP_RETAIN); - EXPECT_EQ(producer_slot.fanout_count, PTO2_FANOUT_SCOPE_BIT + 1); + EXPECT_EQ(producer_slot.fanout_count, FANOUT_SCOPE_BIT + 1); } // The duplicate lands in the spill region (>64 fanin), exercising @@ -151,7 +151,7 @@ TEST_F(OrchestratorFaninTest, DuplicateProducerOrAccumulatesFlags) { TEST_F(OrchestratorFaninTest, DuplicateProducerInSpillRegionDedups) { orch.begin_scope(); - constexpr int kProducers = PTO2_FANIN_INLINE_CAP + 1; // 65: the last one spills + constexpr int kProducers = CHIP_FANIN_INLINE_CAP + 1; // 65: the last one spills std::vector producers; producers.reserve(kProducers); for (int i = 0; i < kProducers; i++) { @@ -184,13 +184,13 @@ TEST_F(OrchestratorFaninTest, DuplicateProducerInSpillRegionDedups) { TaskId dup = producers.back().task_id(); auto &dup_slot = sm_handle->header->rings[dup.ring()].get_slot_state_by_task_id(dup.local()); - EXPECT_EQ(dup_slot.fanout_count, PTO2_FANOUT_SCOPE_BIT + 1); // one pin, not two + EXPECT_EQ(dup_slot.fanout_count, FANOUT_SCOPE_BIT + 1); // one pin, not two // The first spilled edge is the duplicated producer; its flags OR-folded to // WAIT|RETAIN across the two discovery kinds. ASSERT_NE(payload->fanin_spill_pool, nullptr); - PTO2FaninPool &spill_pool = *payload->fanin_spill_pool; - PTO2FaninSpillEntry &spill_edge = spill_pool.base[payload->fanin_spill_start % spill_pool.capacity]; + FaninPool &spill_pool = *payload->fanin_spill_pool; + FaninSpillEntry &spill_edge = spill_pool.base[payload->fanin_spill_start % spill_pool.capacity]; EXPECT_EQ(spill_edge.slot_state(), &dup_slot); EXPECT_EQ(spill_edge.flags(), DEP_WAIT | DEP_RETAIN); } diff --git a/tests/ut/cpp/a2a3/test_scheduler_state.cpp b/tests/ut/cpp/a2a3/test_scheduler_state.cpp index b79b1670b3..292529f637 100644 --- a/tests/ut/cpp/a2a3/test_scheduler_state.cpp +++ b/tests/ut/cpp/a2a3/test_scheduler_state.cpp @@ -343,8 +343,8 @@ TEST_F(SchedulerStateTest, ScopeEndBatchRelease) { for (int i = 0; i < N; i++) { // on_scope_end releases the owning-scope ref via release_producer_scope, - // which adds PTO2_FANOUT_SCOPE_BIT (bit31) to fanout_refcount. - EXPECT_EQ(slots[i].fanout_refcount.load(), PTO2_FANOUT_SCOPE_BIT); + // which adds FANOUT_SCOPE_BIT (bit31) to fanout_refcount. + EXPECT_EQ(slots[i].fanout_refcount.load(), FANOUT_SCOPE_BIT); } } diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index 009118fb6d..43c2f6d4fc 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -1154,8 +1154,8 @@ TEST_F(WiringTest, OnTaskReleaseReleasesProducers) { payload.fanin_inline_edges[0].set(&producers[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producers[1], DEP_WAIT | DEP_RETAIN); // Need a valid fanin_spill_pool even though we don't spill - PTO2FaninPool dummy_pool{}; - PTO2FaninSpillEntry dummy_entries[4]; + FaninPool dummy_pool{}; + FaninSpillEntry dummy_entries[4]; std::atomic dummy_error{SIMPLER_ERROR_NONE}; dummy_pool.init(dummy_entries, 4, &dummy_error); payload.fanin_spill_pool = &dummy_pool; @@ -1196,8 +1196,8 @@ TEST_F(WiringTest, OrderingOnlyReleasedAtWiringRetentionHeldUntilRelease) { payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&wait_producer, DEP_WAIT); payload.fanin_inline_edges[1].set(&retain_producer, DEP_WAIT | DEP_RETAIN); - PTO2FaninPool dummy_pool{}; - PTO2FaninSpillEntry dummy_entries[4]; + FaninPool dummy_pool{}; + FaninSpillEntry dummy_entries[4]; std::atomic dummy_error{SIMPLER_ERROR_NONE}; dummy_pool.init(dummy_entries, 4, &dummy_error); payload.fanin_spill_pool = &dummy_pool; @@ -1235,18 +1235,18 @@ TEST_F(WiringTest, ReleaseHonorsRetainFlagInSpillRegion) { init_slot(spill_retain, CHIP_TASK_COMPLETED, 1, 1); init_slot(task_slot, CHIP_TASK_COMPLETED, 0, 1); - for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { + for (int i = 0; i < CHIP_FANIN_INLINE_CAP; i++) { payload.fanin_inline_edges[i].set(&filler, DEP_WAIT); } - PTO2FaninPool spill_pool{}; - PTO2FaninSpillEntry spill_entries[4]; + FaninPool spill_pool{}; + FaninSpillEntry spill_entries[4]; std::atomic err{SIMPLER_ERROR_NONE}; spill_pool.init(spill_entries, 4, &err); auto *e = spill_pool.alloc(); int32_t spill_start = spill_pool.top - 1; e->set(&spill_retain, DEP_WAIT | DEP_RETAIN); - payload.fanin_actual_count = PTO2_FANIN_INLINE_CAP + 1; + payload.fanin_actual_count = CHIP_FANIN_INLINE_CAP + 1; payload.fanin_spill_start = spill_start; payload.fanin_spill_pool = &spill_pool; task_slot.payload = &payload; @@ -1316,9 +1316,9 @@ TEST_F(WiringTest, AdvanceRingPointersResetsSlots) { rss.advance_ring_pointers(); - // After reset_for_reuse: fanout_count=PTO2_FANOUT_SCOPE_BIT (bit31 owning-scope + // After reset_for_reuse: fanout_count=FANOUT_SCOPE_BIT (bit31 owning-scope // ref, 0 consumers), fanin_refcount=0, etc. - EXPECT_EQ(slot.fanout_count, PTO2_FANOUT_SCOPE_BIT); + EXPECT_EQ(slot.fanout_count, FANOUT_SCOPE_BIT); EXPECT_EQ(slot.fanin_refcount.load(), 0); EXPECT_EQ(slot.fanout_refcount.load(), 0); EXPECT_EQ(slot.completed_subtasks.load(), 0); diff --git a/tests/ut/cpp/a5/test_fanin_pool.cpp b/tests/ut/cpp/a5/test_fanin_pool.cpp index 586539b36e..da42e597df 100644 --- a/tests/ut/cpp/a5/test_fanin_pool.cpp +++ b/tests/ut/cpp/a5/test_fanin_pool.cpp @@ -9,11 +9,11 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2FaninPool and for_each_fanin_storage/slot_state + * Unit tests for FaninPool and for_each_fanin_storage/slot_state * from ring_buffer.h / ring_buffer.cpp * * Tests: - * 1. PTO2FaninPool — ring buffer allocation, overflow, tail advance, + * 1. FaninPool — ring buffer allocation, overflow, tail advance, * high-water tracking * 2. for_each_fanin_storage — inline-only, spill without wrap, * spill with wrap, callback early return @@ -36,12 +36,12 @@ class FaninPoolTest : public ::testing::Test { protected: static constexpr int32_t POOL_CAP = 32; - std::vector entries; + std::vector entries; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2FaninPool pool{}; + FaninPool pool{}; void SetUp() override { - entries.assign(POOL_CAP, PTO2FaninSpillEntry{}); + entries.assign(POOL_CAP, FaninSpillEntry{}); error_code.store(SIMPLER_ERROR_NONE); pool.init(entries.data(), POOL_CAP, &error_code); } @@ -158,14 +158,14 @@ class ForEachFaninTest : public ::testing::Test { protected: static constexpr int32_t POOL_CAP = 32; - std::vector spill_entries; + std::vector spill_entries; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2FaninPool spill_pool{}; + FaninPool spill_pool{}; alignas(64) ChipTaskSlotState slots[64]; void SetUp() override { - spill_entries.assign(POOL_CAP, PTO2FaninSpillEntry{}); + spill_entries.assign(POOL_CAP, FaninSpillEntry{}); error_code.store(SIMPLER_ERROR_NONE); spill_pool.init(spill_entries.data(), POOL_CAP, &error_code); memset(slots, 0, sizeof(slots)); @@ -173,7 +173,7 @@ class ForEachFaninTest : public ::testing::Test { }; TEST_F(ForEachFaninTest, InlineOnlyVoid) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; for (int i = 0; i < 5; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -190,7 +190,7 @@ TEST_F(ForEachFaninTest, InlineOnlyVoid) { } TEST_F(ForEachFaninTest, InlineOnlyBoolEarlyReturn) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; for (int i = 0; i < 5; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -206,7 +206,7 @@ TEST_F(ForEachFaninTest, InlineOnlyBoolEarlyReturn) { } TEST_F(ForEachFaninTest, InlineOnlyBoolAllTrue) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; for (int i = 0; i < 3; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -219,7 +219,7 @@ TEST_F(ForEachFaninTest, InlineOnlyBoolAllTrue) { } TEST_F(ForEachFaninTest, ZeroFanin) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; int count = 0; for_each_fanin_storage(inline_slots, 0, 0, spill_pool, [&](ChipTaskSlotState *, DepFlags) { count++; @@ -233,8 +233,8 @@ TEST_F(ForEachFaninTest, ZeroFanin) { TEST_F(ForEachFaninTest, SpillNoWrap) { // 18 fanins = 16 inline + 2 spill - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; - for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; + for (int i = 0; i < CHIP_FANIN_INLINE_CAP; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -268,8 +268,8 @@ TEST_F(ForEachFaninTest, SpillWithWrap) { spill_pool.top = POOL_CAP - 2; spill_pool.tail = POOL_CAP - 2; - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; - for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; + for (int i = 0; i < CHIP_FANIN_INLINE_CAP; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -302,8 +302,8 @@ TEST_F(ForEachFaninTest, SpillWithWrap) { // ============================================================================= TEST_F(ForEachFaninTest, SpillBoolEarlyReturnInSpillRegion) { - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; - for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; + for (int i = 0; i < CHIP_FANIN_INLINE_CAP; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -325,16 +325,16 @@ TEST_F(ForEachFaninTest, SpillBoolEarlyReturnInSpillRegion) { } // ============================================================================= -// PTO2FaninSpillEntry: DepFlags packing round-trips across inline and spill +// FaninSpillEntry: DepFlags packing round-trips across inline and spill // ============================================================================= TEST_F(ForEachFaninTest, DepFlagsRoundTripInlineAndSpill) { // Fill all 64 inline edges; slots 0..2 carry distinct flag combinations. - PTO2FaninSpillEntry inline_slots[PTO2_FANIN_INLINE_CAP] = {}; + FaninSpillEntry inline_slots[CHIP_FANIN_INLINE_CAP] = {}; inline_slots[0].set(&slots[0], DEP_WAIT); inline_slots[1].set(&slots[1], DEP_RETAIN); inline_slots[2].set(&slots[2], DEP_WAIT | DEP_RETAIN); - for (int i = 3; i < PTO2_FANIN_INLINE_CAP; i++) { + for (int i = 3; i < CHIP_FANIN_INLINE_CAP; i++) { inline_slots[i].set(&slots[i], DEP_WAIT); } @@ -345,7 +345,7 @@ TEST_F(ForEachFaninTest, DepFlagsRoundTripInlineAndSpill) { auto *s1 = spill_pool.alloc(); s1->set(&slots[1], DEP_WAIT | DEP_RETAIN); - const int32_t total = PTO2_FANIN_INLINE_CAP + 2; // 64 inline + 2 spill + const int32_t total = CHIP_FANIN_INLINE_CAP + 2; // 64 inline + 2 spill std::vector flags; for_each_fanin_storage(inline_slots, total, spill_start, spill_pool, [&](ChipTaskSlotState *, DepFlags f) { flags.push_back(f); @@ -357,6 +357,6 @@ TEST_F(ForEachFaninTest, DepFlagsRoundTripInlineAndSpill) { EXPECT_EQ(flags[1], DEP_RETAIN); EXPECT_EQ(flags[2], DEP_WAIT | DEP_RETAIN); // Spill flags survive. - EXPECT_EQ(flags[PTO2_FANIN_INLINE_CAP + 0], DEP_RETAIN); - EXPECT_EQ(flags[PTO2_FANIN_INLINE_CAP + 1], DEP_WAIT | DEP_RETAIN); + EXPECT_EQ(flags[CHIP_FANIN_INLINE_CAP + 0], DEP_RETAIN); + EXPECT_EQ(flags[CHIP_FANIN_INLINE_CAP + 1], DEP_WAIT | DEP_RETAIN); } diff --git a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp index 89c6c02f4c..9afc901a64 100644 --- a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp @@ -152,7 +152,7 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { EXPECT_LE(cflag, uint8_t{1}); // Payload counts are real, not the poison bit pattern. EXPECT_GE(pl.fanin_count, 0); - EXPECT_LE(pl.fanin_count, PTO2_MAX_FANIN); + EXPECT_LE(pl.fanin_count, CHIP_MAX_FANIN); EXPECT_GE(pl.tensor_count, 0); EXPECT_GE(pl.scalar_count, 0); // predicate.op is a dispatch-time field, read only for tasks the device diff --git a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp index bd91cb8398..479b3d81a9 100644 --- a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp @@ -102,10 +102,10 @@ TEST_F(OrchestratorFaninTest, DuplicateExplicitProducerAddsOneFanin) { EXPECT_EQ(consumer_slot.payload->fanin_inline_edges[0].slot_state(), &producer_slot); // A plain set_dependencies() dep is conservative RETAIN: DEP_WAIT|DEP_RETAIN. EXPECT_EQ(consumer_slot.payload->fanin_inline_edges[0].flags(), DEP_WAIT | DEP_RETAIN); - // fanout_count is bit-packed: bit31 (PTO2_FANOUT_SCOPE_BIT) is the owning-scope + // fanout_count is bit-packed: bit31 (FANOUT_SCOPE_BIT) is the owning-scope // ref, low bits the consumer count. The duplicate explicit dep is deduped to a // single consumer, so this is scope + 1. - EXPECT_EQ(producer_slot.fanout_count, PTO2_FANOUT_SCOPE_BIT + 1); + EXPECT_EQ(producer_slot.fanout_count, FANOUT_SCOPE_BIT + 1); } // An explicit ordering-only dep (the primitive add_dep_wait() lowers to) yields a @@ -154,7 +154,7 @@ TEST_F(OrchestratorFaninTest, DuplicateProducerOrAccumulatesFlags) { ASSERT_NE(consumer_slot.payload, nullptr); ASSERT_EQ(consumer_slot.payload->fanin_actual_count, 1); EXPECT_EQ(consumer_slot.payload->fanin_inline_edges[0].flags(), DEP_WAIT | DEP_RETAIN); - EXPECT_EQ(producer_slot.fanout_count, PTO2_FANOUT_SCOPE_BIT + 1); + EXPECT_EQ(producer_slot.fanout_count, FANOUT_SCOPE_BIT + 1); } // The duplicate lands in the spill region (>64 fanin), exercising @@ -163,7 +163,7 @@ TEST_F(OrchestratorFaninTest, DuplicateProducerOrAccumulatesFlags) { TEST_F(OrchestratorFaninTest, DuplicateProducerInSpillRegionDedups) { orch.begin_scope(); - constexpr int kProducers = PTO2_FANIN_INLINE_CAP + 1; // 65: the last one spills + constexpr int kProducers = CHIP_FANIN_INLINE_CAP + 1; // 65: the last one spills std::vector producers; producers.reserve(kProducers); for (int i = 0; i < kProducers; i++) { @@ -196,13 +196,13 @@ TEST_F(OrchestratorFaninTest, DuplicateProducerInSpillRegionDedups) { TaskId dup = producers.back().task_id(); auto &dup_slot = sm_handle->header->rings[dup.ring()].get_slot_state_by_task_id(dup.local()); - EXPECT_EQ(dup_slot.fanout_count, PTO2_FANOUT_SCOPE_BIT + 1); // one pin, not two + EXPECT_EQ(dup_slot.fanout_count, FANOUT_SCOPE_BIT + 1); // one pin, not two // The first spilled edge is the duplicated producer; its flags OR-folded to // WAIT|RETAIN across the two discovery kinds. ASSERT_NE(payload->fanin_spill_pool, nullptr); - PTO2FaninPool &spill_pool = *payload->fanin_spill_pool; - PTO2FaninSpillEntry &spill_edge = spill_pool.base[payload->fanin_spill_start % spill_pool.capacity]; + FaninPool &spill_pool = *payload->fanin_spill_pool; + FaninSpillEntry &spill_edge = spill_pool.base[payload->fanin_spill_start % spill_pool.capacity]; EXPECT_EQ(spill_edge.slot_state(), &dup_slot); EXPECT_EQ(spill_edge.flags(), DEP_WAIT | DEP_RETAIN); } diff --git a/tests/ut/cpp/a5/test_scheduler_state.cpp b/tests/ut/cpp/a5/test_scheduler_state.cpp index 84dd21601b..90730b524b 100644 --- a/tests/ut/cpp/a5/test_scheduler_state.cpp +++ b/tests/ut/cpp/a5/test_scheduler_state.cpp @@ -396,8 +396,8 @@ TEST_F(SchedulerStateTest, ScopeEndBatchRelease) { for (int i = 0; i < N; i++) { // on_scope_end releases the owning-scope ref via release_producer_scope, - // which adds PTO2_FANOUT_SCOPE_BIT (bit31) to fanout_refcount. - EXPECT_EQ(slots[i].fanout_refcount.load(), PTO2_FANOUT_SCOPE_BIT); + // which adds FANOUT_SCOPE_BIT (bit31) to fanout_refcount. + EXPECT_EQ(slots[i].fanout_refcount.load(), FANOUT_SCOPE_BIT); } } diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index dc3e82b914..eca549ceed 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -473,8 +473,8 @@ TEST_F(WiringTest, OnTaskReleaseReleasesProducers) { payload.fanin_inline_edges[0].set(&producers[0], DEP_WAIT | DEP_RETAIN); payload.fanin_inline_edges[1].set(&producers[1], DEP_WAIT | DEP_RETAIN); // Need a valid fanin_spill_pool even though we don't spill - PTO2FaninPool dummy_pool{}; - PTO2FaninSpillEntry dummy_entries[4]; + FaninPool dummy_pool{}; + FaninSpillEntry dummy_entries[4]; std::atomic dummy_error{SIMPLER_ERROR_NONE}; dummy_pool.init(dummy_entries, 4, &dummy_error); payload.fanin_spill_pool = &dummy_pool; @@ -515,8 +515,8 @@ TEST_F(WiringTest, OrderingOnlyReleasedAtWiringRetentionHeldUntilRelease) { payload.fanin_actual_count = 2; payload.fanin_inline_edges[0].set(&wait_producer, DEP_WAIT); payload.fanin_inline_edges[1].set(&retain_producer, DEP_WAIT | DEP_RETAIN); - PTO2FaninPool dummy_pool{}; - PTO2FaninSpillEntry dummy_entries[4]; + FaninPool dummy_pool{}; + FaninSpillEntry dummy_entries[4]; std::atomic dummy_error{SIMPLER_ERROR_NONE}; dummy_pool.init(dummy_entries, 4, &dummy_error); payload.fanin_spill_pool = &dummy_pool; @@ -554,18 +554,18 @@ TEST_F(WiringTest, ReleaseHonorsRetainFlagInSpillRegion) { init_slot(spill_retain, CHIP_TASK_COMPLETED, 1, 1); init_slot(task_slot, CHIP_TASK_COMPLETED, 0, 1); - for (int i = 0; i < PTO2_FANIN_INLINE_CAP; i++) { + for (int i = 0; i < CHIP_FANIN_INLINE_CAP; i++) { payload.fanin_inline_edges[i].set(&filler, DEP_WAIT); } - PTO2FaninPool spill_pool{}; - PTO2FaninSpillEntry spill_entries[4]; + FaninPool spill_pool{}; + FaninSpillEntry spill_entries[4]; std::atomic err{SIMPLER_ERROR_NONE}; spill_pool.init(spill_entries, 4, &err); auto *e = spill_pool.alloc(); int32_t spill_start = spill_pool.top - 1; e->set(&spill_retain, DEP_WAIT | DEP_RETAIN); - payload.fanin_actual_count = PTO2_FANIN_INLINE_CAP + 1; + payload.fanin_actual_count = CHIP_FANIN_INLINE_CAP + 1; payload.fanin_spill_start = spill_start; payload.fanin_spill_pool = &spill_pool; task_slot.payload = &payload; @@ -857,8 +857,8 @@ TEST_F(WiringTest, FaninPoolReclaimsOnceWithheldProgressIsPublished) { ASSERT_EQ(rss.last_task_alive, 1); ASSERT_EQ(ring->fc.last_task_alive.load(), 0); - PTO2FaninSpillEntry entries[4]{}; - PTO2FaninPool pool{}; + FaninSpillEntry entries[4]{}; + FaninPool pool{}; auto *orch_err = sm_layout::orch_error_code_addr(sm_handle->sm_base); pool.init(entries, 4, orch_err); pool.set_reclaim_publication_request(&sched.publication_request_mask, &sched.publication_ack_mask, 0); @@ -867,7 +867,7 @@ TEST_F(WiringTest, FaninPoolReclaimsOnceWithheldProgressIsPublished) { } auto &payload = ring->get_payload_by_task_id(0); - payload.fanin_actual_count = PTO2_FANIN_INLINE_CAP + 1; + payload.fanin_actual_count = CHIP_FANIN_INLINE_CAP + 1; payload.fanin_spill_start = 1; payload.fanin_spill_pool = &pool; @@ -928,9 +928,9 @@ TEST_F(WiringTest, AdvanceRingPointersResetsSlots) { rss.advance_ring_pointers(); - // After reset_for_reuse: fanout_count=PTO2_FANOUT_SCOPE_BIT (bit31 owning-scope + // After reset_for_reuse: fanout_count=FANOUT_SCOPE_BIT (bit31 owning-scope // ref, 0 consumers), fanin_refcount=0, etc. - EXPECT_EQ(slot.fanout_count, PTO2_FANOUT_SCOPE_BIT); + EXPECT_EQ(slot.fanout_count, FANOUT_SCOPE_BIT); EXPECT_EQ(slot.fanin_refcount.load(), 0); EXPECT_EQ(slot.fanout_refcount.load(), 0); EXPECT_EQ(slot.completed_subtasks.load(), 0); diff --git a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp index 51c7c491ec..696597fd49 100644 --- a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp @@ -392,7 +392,7 @@ TEST_F(HbgGraphSubmitFailureTest, FaninFailureLatchesFatalWithoutPartialUpload) CoreTaskArgs producer_args; producer_args.add_output(boundary); - for (int32_t i = 0; i < PTO2_MAX_FANIN + 1; ++i) { + for (int32_t i = 0; i < CHIP_MAX_FANIN + 1; ++i) { ASSERT_TRUE(orch.submit_dummy_task(producer_args).task_id().is_valid()); } diff --git a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp index ea526e461f..06a3f8953e 100644 --- a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp +++ b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp @@ -28,7 +28,7 @@ void make_head_match_old_structural_predicate( descriptor.task_id = TaskId::make(ring_id, local_task_id); head.task = &descriptor; head.task_state.store(CHIP_TASK_COMPLETED, std::memory_order_release); - head.fanout_count = PTO2_FANOUT_SCOPE_BIT; + head.fanout_count = FANOUT_SCOPE_BIT; head.fanout_refcount.store(0, std::memory_order_release); } From 8b434bced0bda6157c95173b26c841f524748aa8 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:45:10 -0700 Subject: [PATCH 12/22] Refactor: retire the PTO2 prefix on the dependency-list pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PTO2DepListEntry` -> `DepListEntry` and `PTO2DepListPool` -> `DepListPool`. 124 occurrences across 35 files. The four constants that size and instrument the pool keep a prefix — `CHIP_DEP_LIST_POOL_SIZE`, `CHIP_DEP_POOL_CLEANUP_INTERVAL`, `CHIP_DEP_DEGREE_DEBUG_THRESHOLD`, `CHIP_DEP_DEGREE_WARN_THRESHOLD` — because they are object-like macros in `runtime/runtime_types.h` and a macro rewrites its identifier in every translation unit that includes the header rather than declaring something the compiler can diagnose. Co-Authored-By: Claude Opus 5 (1M context) --- .../runtime/orchestrator_core/ring_buffer.cpp | 2 +- .../host_build_graph/runtime/runtime_types.h | 2 +- .../docs/MULTI_RING.md | 8 +++---- .../docs/RUNTIME_LOGIC.md | 2 +- .../host/runtime_maker.cpp | 2 +- .../runtime/orchestrator.cpp | 8 +++---- .../runtime/orchestrator.h | 2 +- .../runtime/ring_buffer.cpp | 14 +++++------- .../runtime/ring_buffer.h | 16 +++++++------- .../runtime/runtime_core.h | 2 +- .../runtime/runtime_types.h | 12 +++++----- .../runtime/scheduler/scheduler.h | 8 +++---- .../runtime/shared/runtime_init.cpp | 8 +++---- .../runtime/orchestrator_core/ring_buffer.cpp | 2 +- .../host_build_graph/runtime/runtime_types.h | 2 +- .../docs/MULTI_RING.md | 8 +++---- .../docs/RUNTIME_LOGIC.md | 2 +- .../host/runtime_maker.cpp | 2 +- .../runtime/orchestrator.cpp | 8 +++---- .../runtime/orchestrator.h | 2 +- .../runtime/ring_buffer.cpp | 14 +++++------- .../runtime/ring_buffer.h | 16 +++++++------- .../runtime/runtime_core.h | 2 +- .../runtime/runtime_types.h | 12 +++++----- .../runtime/scheduler/scheduler.h | 8 +++---- .../runtime/shared/runtime_init.cpp | 8 +++---- .../orchestration/chain_barrier_orch.cpp | 2 +- .../kernels/orchestration/dummy_task_orch.cpp | 2 +- .../orchestration/chain_barrier_orch.cpp | 2 +- .../kernels/orchestration/dummy_task_orch.cpp | 2 +- tests/ut/cpp/a2a3/test_dep_list_pool.cpp | 22 +++++++++---------- tests/ut/cpp/a2a3/test_wiring.cpp | 10 ++++----- tests/ut/cpp/a5/test_dep_list_pool.cpp | 22 +++++++++---------- tests/ut/cpp/a5/test_wiring.cpp | 6 ++--- .../common/test_scope_deadlock_detection.cpp | 8 +++---- 35 files changed, 122 insertions(+), 126 deletions(-) diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp index 1645a4061c..26b277c8a3 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp @@ -12,6 +12,6 @@ // The polling completion scheduler has no dep-pool / fanin-spill pool: producer // dependencies are position-independent local-id integers on the payload and // readiness is derived from the ring completion_flags. The pool ring buffers -// (FaninPool / PTO2DepListPool) that once lived here are gone, so this +// (FaninPool / DepListPool) that once lived here are gone, so this // translation unit carries no definitions. TaskAllocator's methods are // defined inline in ring_buffer.h. diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index 344f8a9776..bde7a6e49a 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -148,7 +148,7 @@ inline constexpr int32_t ARG_POOL_ALIGN = 64; // Dependency-degree diagnostic: warn once when a task's fanin or a producer's // fanout first exceeds this degree, so dense dependency graphs surface without // flooding the AICPU hot-path device log. -#define PTO2_DEP_DEGREE_WARN_THRESHOLD 16 +#define CHIP_DEP_DEGREE_WARN_THRESHOLD 16 // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle // count (PTO2_TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index f79dc0c687..7c54f0753c 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -64,7 +64,7 @@ struct ChipRingSet { // Before: single ring ChipHeapRing heap_ring; ChipTaskRing task_ring; -PTO2DepListPool dep_pool; +DepListPool dep_pool; // After: per-ring array (dep_pool moved to scheduler, see §4.5) ChipRingSet rings[CHIP_MAX_RING_DEPTH]; @@ -136,7 +136,7 @@ struct RingSchedState { std::atomic advance_lock; // multi-thread CAS // Cache Line 1+: Orch-side wiring dep_pool, cache-isolated - alignas(64) PTO2DepListPool dep_pool; + alignas(64) DepListPool dep_pool; }; RingSchedState ring_sched_states[CHIP_MAX_RING_DEPTH]; @@ -165,7 +165,7 @@ bool entry_valid(const ChipTensorMapEntry& e) { | Structure | Reason | | --------- | ------ | -| `PTO2DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | +| `DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | | `TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | | `ChipReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | | `PTO2DispatchPayload` | Built per-dispatch, no ring state needed | @@ -234,7 +234,7 @@ AICore uses `last_reg_val` to detect new dispatches — identical values cause s | -------- | ------- | ---------------- | | `PTO2_TASK_WINDOW_SIZE` | 16384 | 65536 | | `PTO2_HEAP_SIZE` | 256 MB | 1 GB | -| `PTO2_DEP_LIST_POOL_SIZE` | 16384 | 65536 | +| `CHIP_DEP_LIST_POOL_SIZE` | 16384 | 65536 | ### 7.2 Runtime Overrides diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 7ab3360317..8f9f7b6997 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -234,7 +234,7 @@ allocation retires. ### 4.3 Dependency List Pool -A simple bump allocator for `PTO2DepListEntry` nodes used in fanin/fanout linked lists. +A simple bump allocator for `DepListEntry` nodes used in fanin/fanout linked lists. - **Entry 0**: NULL sentinel (`task_id=-1, next_offset=0`) - **Allocation**: `pool->top++`, wraps around when full diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index f260f66392..3dc8660818 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -242,7 +242,7 @@ static bool resolve_ring_config( for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { eff_task_window_sizes[r] = PTO2_TASK_WINDOW_SIZE; eff_heap_sizes[r] = PTO2_HEAP_SIZE; - dep_pool_values[r] = PTO2_DEP_LIST_POOL_SIZE; + dep_pool_values[r] = CHIP_DEP_LIST_POOL_SIZE; } apply_env_ring_values("PTO2_RING_TASK_WINDOW", 4, static_cast(INT32_MAX), true, eff_task_window_sizes); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 196c00283b..8ed2dc174a 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -374,10 +374,10 @@ static bool append_fanin_or_fail( // Dense-fanout diagnostic, emitted outside fanout_lock (no logging under // the spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly // once, so each dense producer emits one debug message when enabled. - if (fanout_now == PTO2_DEP_DEGREE_DEBUG_THRESHOLD + 1) { + if (fanout_now == CHIP_DEP_DEGREE_DEBUG_THRESHOLD + 1) { LOG_DEBUG( "dense dependency: task ring=%u id=%u fanout>%d [orch submit]", - static_cast(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD + static_cast(producer_task_id.ring()), producer_task_id.local(), CHIP_DEP_DEGREE_DEBUG_THRESHOLD ); } // Stale/consumed producer: no edge at all. @@ -1048,10 +1048,10 @@ static TaskOutputTensors submit_task_common( // fanin_builder.count is finalized here and submit runs once per task, so // each dense consumer emits one debug message when enabled. This checks > // THRESHOLD because the count lands at its final total here. - if (fanin_builder.count > PTO2_DEP_DEGREE_DEBUG_THRESHOLD) { + if (fanin_builder.count > CHIP_DEP_DEGREE_DEBUG_THRESHOLD) { LOG_DEBUG( "dense dependency: task ring=%u id=%u fanin>%d [orch submit]", static_cast(task_id.ring()), - task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD + task_id.local(), CHIP_DEP_DEGREE_DEBUG_THRESHOLD ); } payload.fanin_spill_start = fanin_builder.spill_start; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index cb91daa77c..7782ce32db 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -141,7 +141,7 @@ struct OrchestratorState { // init_data_from_layout. static OrchestratorLayout reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], - int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE + int32_t dep_pool_capacity = CHIP_DEP_LIST_POOL_SIZE ); static OrchestratorLayout reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index 19cbdf473d..16ae4b83b7 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -109,7 +109,7 @@ bool FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { LOG_ERROR( " Increase fanin spill pool capacity (current: %d, recommended: %d)", capacity, high_water * 2 ); - LOG_ERROR(" Compile-time: PTO2_DEP_LIST_POOL_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", high_water * 2); LOG_ERROR("========================================"); latch_pool_error(error_code_ptr, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); @@ -124,8 +124,8 @@ bool FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { // ============================================================================= // Dependency List Pool Implementation // ============================================================================= -void PTO2DepListPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { - if (sm_last_task_alive >= last_reclaimed + PTO2_DEP_POOL_CLEANUP_INTERVAL && sm_last_task_alive > 0) { +void DepListPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { + if (sm_last_task_alive >= last_reclaimed + CHIP_DEP_POOL_CLEANUP_INTERVAL && sm_last_task_alive > 0) { int32_t mark = ring.get_slot_state_by_task_id(sm_last_task_alive - 1).dep_pool_mark; if (mark > 0) { advance_tail(mark); @@ -134,9 +134,7 @@ void PTO2DepListPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task } } -void PTO2DepListPool::report_deadlock( - SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated -) { +void DepListPool::report_deadlock(SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated) { int32_t current = ring.fc.current_task_index.load(std::memory_order_acquire); LOG_ERROR("========================================"); LOG_ERROR("FATAL: Dependency Pool Deadlock Detected!"); @@ -171,12 +169,12 @@ void PTO2DepListPool::report_deadlock( } LOG_ERROR("Solution:"); LOG_ERROR(" Increase dep pool capacity (current: %d, recommended: %d)", capacity, high_water * 2); - LOG_ERROR(" Compile-time: PTO2_DEP_LIST_POOL_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", high_water * 2); LOG_ERROR("========================================"); } -bool PTO2DepListPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task) { +bool DepListPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task) { if (available() >= needed) return true; int spin_count = 0; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 284eb21a29..e6f57b4aba 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -549,7 +549,7 @@ struct FaninPool { LOG_ERROR(" - High water: %d", high_water); LOG_ERROR("Solution:"); LOG_ERROR(" Increase fanin spill pool capacity (current: %d, recommended: %d).", capacity, capacity * 2); - LOG_ERROR(" Compile-time: PTO2_DEP_LIST_POOL_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", capacity * 2); LOG_ERROR("========================================"); if (error_code_ptr) { @@ -670,8 +670,8 @@ inline FaninForEachReturn for_each_fanin_slot_state(const TaskPayload &paylo * Linear counters (top, tail) grow monotonically; the physical index * is obtained via modulo: base[linear_index % capacity]. */ -struct PTO2DepListPool { - PTO2DepListEntry *base; // Pool base address +struct DepListPool { + DepListEntry *base; // Pool base address int32_t capacity; // Total number of entries int32_t top; // Linear next-allocation counter (starts from 1) int32_t tail; // Linear first-alive counter (entries before this are dead) @@ -687,7 +687,7 @@ struct PTO2DepListPool { * @param base Pool base address from shared memory * @param capacity Total number of entries */ - void init(PTO2DepListEntry *in_base, int32_t in_capacity, std::atomic *in_error_code_ptr) { + void init(DepListEntry *in_base, int32_t in_capacity, std::atomic *in_error_code_ptr) { base = in_base; capacity = in_capacity; top = 1; // Start from 1, 0 means NULL/empty @@ -742,7 +742,7 @@ struct PTO2DepListPool { * * @return Pointer to allocated entry, or nullptr on fatal error */ - PTO2DepListEntry *alloc() { + DepListEntry *alloc() { int32_t used = top - tail; if (used >= capacity) { LOG_ERROR("========================================"); @@ -754,7 +754,7 @@ struct PTO2DepListPool { LOG_ERROR(" - High water: %d", high_water); LOG_ERROR("Solution:"); LOG_ERROR(" Increase dep pool capacity (current: %d, recommended: %d).", capacity, capacity * 2); - LOG_ERROR(" Compile-time: PTO2_DEP_LIST_POOL_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", capacity * 2); LOG_ERROR("========================================"); if (error_code_ptr) { @@ -788,8 +788,8 @@ struct PTO2DepListPool { * @param task_slot Task slot to prepend * @return New head offset */ - PTO2DepListEntry *prepend(PTO2DepListEntry *cur, ChipTaskSlotState *slot_state) { - PTO2DepListEntry *new_entry = alloc(); + DepListEntry *prepend(DepListEntry *cur, ChipTaskSlotState *slot_state) { + DepListEntry *new_entry = alloc(); if (!new_entry) return nullptr; new_entry->slot_state = slot_state; new_entry->next = cur; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index 94ec967711..7d2c7990aa 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -195,7 +195,7 @@ struct RuntimeContext { * Phase 2/3. */ RuntimeArenaLayout runtime_reserve_layout( - DeviceArena &arena, uint64_t task_window_size, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE + DeviceArena &arena, uint64_t task_window_size, int32_t dep_pool_capacity = CHIP_DEP_LIST_POOL_SIZE ); RuntimeArenaLayout runtime_reserve_layout( DeviceArena &arena, const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 15831a0020..fc78c4ee6e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -70,7 +70,7 @@ // Memory pools (per-ring defaults; total = value × CHIP_MAX_RING_DEPTH) #define PTO2_HEAP_SIZE (256 * 1024 * 1024) // 256MB per ring (1GB total) -#define PTO2_DEP_LIST_POOL_SIZE 16384 // Per-ring dependency list pool entries +#define CHIP_DEP_LIST_POOL_SIZE 16384 // Per-ring dependency list pool entries #define CHIP_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool #define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) @@ -99,11 +99,11 @@ // Dependency-degree diagnostic: log once at debug level when a task's fanin or // a producer's fanout first exceeds this degree, so dense dependency graphs can // be inspected without adding noise to normal runtime logs. -#define PTO2_DEP_DEGREE_DEBUG_THRESHOLD 16 +#define CHIP_DEP_DEGREE_DEBUG_THRESHOLD 16 // TensorMap cleanup interval #define CHIP_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks -#define PTO2_DEP_POOL_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks +#define CHIP_DEP_POOL_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle // count (PTO2_TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp @@ -197,9 +197,9 @@ static_assert(sizeof(FaninSpillEntry) == sizeof(uintptr_t)); * Dependency list entry (singly-linked list node) * Stored in DepListPool ring buffer. */ -struct PTO2DepListEntry { +struct DepListEntry { ChipTaskSlotState *slot_state; // Consumer slot state (direct pointer) - PTO2DepListEntry *next; // next entry + DepListEntry *next; // next entry }; // ============================================================================= @@ -475,7 +475,7 @@ struct alignas(64) ChipTaskSlotState { std::atomic fanout_lock; // Per-task spinlock (0=unlocked, 1=locked) uint32_t fanout_count; // SCOPE_BIT (owning scope) | number of consumers - PTO2DepListEntry *fanout_head; // Pointer to first fanout entry (nullptr = empty) + DepListEntry *fanout_head; // Pointer to first fanout entry (nullptr = empty) // Task state (completion, consumed check, ready check) std::atomic task_state; // PENDING/COMPLETED/CONSUMED diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 4c297dc36b..0921cc844a 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -453,7 +453,7 @@ struct SchedulerState { std::atomic advance_lock; // multi-thread CAS // --- Cache Line 1+: Orch-side wiring dep_pool --- - alignas(64) PTO2DepListPool dep_pool; + alignas(64) DepListPool dep_pool; #if SIMPLER_DFX // Published only for scope_stats; orchestrator must not read dep_pool's non-atomic counters directly. alignas(64) std::atomic dep_pool_snapshot_tail; @@ -993,7 +993,7 @@ struct SchedulerState { } // Nodes reachable from the captured head are immutable; wiring serializes // later prepends under fanout_lock and seeds them separately. - PTO2DepListEntry *edge = p.fanout_head; + DepListEntry *edge = p.fanout_head; p.unlock_fanout(); for (; edge != nullptr; edge = edge->next) { ChipTaskSlotState *c = edge->slot_state; @@ -1265,7 +1265,7 @@ struct SchedulerState { slot_state.lock_fanout(); #endif slot_state.mark_completed(); - PTO2DepListEntry *current = slot_state.fanout_head; // Protected by fanout_lock + DepListEntry *current = slot_state.fanout_head; // Protected by fanout_lock slot_state.unlock_fanout(); #if SIMPLER_SCHED_PROFILING @@ -1380,7 +1380,7 @@ struct SchedulerState { // per-ring dep_pool entries) on the supplied arena. // Capacities are baked into the returned layout; init_data_from_layout uses // the same values. - static SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE); + static SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = CHIP_DEP_LIST_POOL_SIZE); static SchedulerLayout reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]); // Phase 3a: write everything *except* arena-internal pointer fields. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 28f2bc5f80..5e3c569b4f 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -150,7 +150,7 @@ SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capaci // Force a cache-line base so Orch-side dep_pool writes do not invalidate // adjacent multi-threaded regions like ready_queue.slots. layout.off_dep_pool_entries[r] = - arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(PTO2DepListEntry), PTO2_ALIGN_SIZE); + arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(DepListEntry), PTO2_ALIGN_SIZE); } return layout; } @@ -206,8 +206,8 @@ bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, Device auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - auto *dep_entries = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); - memset(dep_entries, 0, static_cast(layout.dep_pool_capacities[r]) * sizeof(PTO2DepListEntry)); + auto *dep_entries = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); + memset(dep_entries, 0, static_cast(layout.dep_pool_capacities[r]) * sizeof(DepListEntry)); sched->ring_sched_states[r].dep_pool.init(dep_entries, layout.dep_pool_capacities[r], orch_err); } @@ -261,7 +261,7 @@ void SchedulerState::wire_arena_pointers(const SchedulerLayout &layout, DeviceAr ready_queue_wire_arena_pointers(&sched->early_sync_start_queue, arena, layout.off_early_sync_start_queue_slots); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { sched->ring_sched_states[r].dep_pool.base = - static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); + static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); } } diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp index 1645a4061c..26b277c8a3 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/ring_buffer.cpp @@ -12,6 +12,6 @@ // The polling completion scheduler has no dep-pool / fanin-spill pool: producer // dependencies are position-independent local-id integers on the payload and // readiness is derived from the ring completion_flags. The pool ring buffers -// (FaninPool / PTO2DepListPool) that once lived here are gone, so this +// (FaninPool / DepListPool) that once lived here are gone, so this // translation unit carries no definitions. TaskAllocator's methods are // defined inline in ring_buffer.h. diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index e68e65c232..137dec2672 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -149,7 +149,7 @@ inline constexpr int32_t ARG_POOL_ALIGN = 64; // Dependency-degree diagnostic: warn once when a task's fanin or a producer's // fanout first exceeds this degree, so dense dependency graphs surface without // flooding the AICPU hot-path device log. -#define PTO2_DEP_DEGREE_WARN_THRESHOLD 16 +#define CHIP_DEP_DEGREE_WARN_THRESHOLD 16 // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle // count (PTO2_TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 97e0223086..d4642f0a5d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -64,7 +64,7 @@ struct ChipRingSet { // Before: single ring ChipHeapRing heap_ring; ChipTaskRing task_ring; -PTO2DepListPool dep_pool; +DepListPool dep_pool; // After: per-ring array (dep_pool moved to scheduler, see §4.5) ChipRingSet rings[CHIP_MAX_RING_DEPTH]; @@ -136,7 +136,7 @@ struct RingSchedState { std::atomic advance_lock; // multi-thread CAS // Cache Line 1+: Orch-side wiring dep_pool, cache-isolated - alignas(64) PTO2DepListPool dep_pool; + alignas(64) DepListPool dep_pool; }; RingSchedState ring_sched_states[CHIP_MAX_RING_DEPTH]; @@ -165,7 +165,7 @@ bool entry_valid(const ChipTensorMapEntry& e) { | Structure | Reason | | --------- | ------ | -| `PTO2DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | +| `DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | | `TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | | `ChipReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | | `PTO2DispatchPayload` | Built per-dispatch, no ring state needed | @@ -236,7 +236,7 @@ AICore uses `last_reg_val` to detect new dispatches — identical values cause s | -------- | ------- | ---------------- | | `PTO2_TASK_WINDOW_SIZE` | 16384 | 65536 | | `PTO2_HEAP_SIZE` | 256 MB | 1 GB | -| `PTO2_DEP_LIST_POOL_SIZE` | 16384 | 65536 | +| `CHIP_DEP_LIST_POOL_SIZE` | 16384 | 65536 | ### 7.2 Runtime Overrides diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 542def3e3d..0007559d09 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -235,7 +235,7 @@ allocation retires. ### 4.3 Dependency List Pool -A simple bump allocator for `PTO2DepListEntry` nodes used in fanin/fanout linked lists. +A simple bump allocator for `DepListEntry` nodes used in fanin/fanout linked lists. - **Entry 0**: NULL sentinel (`task_id=-1, next_offset=0`) - **Allocation**: `pool->top++`, wraps around when full diff --git a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index 9e321bdb10..1b165480f6 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -242,7 +242,7 @@ static bool resolve_ring_config( for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { eff_task_window_sizes[r] = PTO2_TASK_WINDOW_SIZE; eff_heap_sizes[r] = PTO2_HEAP_SIZE; - dep_pool_values[r] = PTO2_DEP_LIST_POOL_SIZE; + dep_pool_values[r] = CHIP_DEP_LIST_POOL_SIZE; } apply_env_ring_values("PTO2_RING_TASK_WINDOW", 4, static_cast(INT32_MAX), true, eff_task_window_sizes); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 1412143a86..38bce0d8f2 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -367,10 +367,10 @@ static bool append_fanin_or_fail( // Dense-fanout diagnostic, emitted outside fanout_lock (no logging under // the spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly // once, so each dense producer emits one debug message when enabled. - if (fanout_now == PTO2_DEP_DEGREE_DEBUG_THRESHOLD + 1) { + if (fanout_now == CHIP_DEP_DEGREE_DEBUG_THRESHOLD + 1) { LOG_DEBUG( "dense dependency: task ring=%u id=%u fanout>%d [orch submit]", - static_cast(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD + static_cast(producer_task_id.ring()), producer_task_id.local(), CHIP_DEP_DEGREE_DEBUG_THRESHOLD ); } // Stale/consumed producer: no edge at all. @@ -1050,10 +1050,10 @@ static TaskOutputTensors submit_task_common( // fanin_builder.count is finalized here and submit runs once per task, so // each dense consumer emits one debug message when enabled. This checks > // THRESHOLD because the count lands at its final total here. - if (fanin_builder.count > PTO2_DEP_DEGREE_DEBUG_THRESHOLD) { + if (fanin_builder.count > CHIP_DEP_DEGREE_DEBUG_THRESHOLD) { LOG_DEBUG( "dense dependency: task ring=%u id=%u fanin>%d [orch submit]", static_cast(task_id.ring()), - task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD + task_id.local(), CHIP_DEP_DEGREE_DEBUG_THRESHOLD ); } payload.fanin_spill_start = fanin_builder.spill_start; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index cb91daa77c..7782ce32db 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -141,7 +141,7 @@ struct OrchestratorState { // init_data_from_layout. static OrchestratorLayout reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], - int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE + int32_t dep_pool_capacity = CHIP_DEP_LIST_POOL_SIZE ); static OrchestratorLayout reserve_layout( DeviceArena &arena, const int32_t task_window_sizes[CHIP_MAX_RING_DEPTH], diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index 80d1656997..1e206b6dc4 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -121,7 +121,7 @@ bool FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { LOG_ERROR( " Increase fanin spill pool capacity (current: %d, recommended: %d)", capacity, high_water * 2 ); - LOG_ERROR(" Compile-time: PTO2_DEP_LIST_POOL_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", high_water * 2); LOG_ERROR("========================================"); latch_pool_error(error_code_ptr, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); @@ -137,8 +137,8 @@ bool FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { // ============================================================================= // Dependency List Pool Implementation // ============================================================================= -void PTO2DepListPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { - if (sm_last_task_alive >= last_reclaimed + PTO2_DEP_POOL_CLEANUP_INTERVAL && sm_last_task_alive > 0) { +void DepListPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task_alive) { + if (sm_last_task_alive >= last_reclaimed + CHIP_DEP_POOL_CLEANUP_INTERVAL && sm_last_task_alive > 0) { int32_t mark = ring.get_slot_state_by_task_id(sm_last_task_alive - 1).dep_pool_mark; if (mark > 0) { advance_tail(mark); @@ -147,9 +147,7 @@ void PTO2DepListPool::reclaim(SharedMemoryRingHeader &ring, int32_t sm_last_task } } -void PTO2DepListPool::report_deadlock( - SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated -) { +void DepListPool::report_deadlock(SharedMemoryRingHeader &ring, int32_t needed, int32_t last_alive, bool scope_gated) { int32_t current = ring.fc.current_task_index.load(std::memory_order_acquire); LOG_ERROR("========================================"); LOG_ERROR("FATAL: Dependency Pool Deadlock Detected!"); @@ -184,12 +182,12 @@ void PTO2DepListPool::report_deadlock( } LOG_ERROR("Solution:"); LOG_ERROR(" Increase dep pool capacity (current: %d, recommended: %d)", capacity, high_water * 2); - LOG_ERROR(" Compile-time: PTO2_DEP_LIST_POOL_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", high_water * 2); LOG_ERROR("========================================"); } -bool PTO2DepListPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task) { +bool DepListPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed, ChipTaskSlotState *oldest_open_task) { if (available() >= needed) return true; int spin_count = 0; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 0fc41945e0..298f39a958 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -642,7 +642,7 @@ struct FaninPool { LOG_ERROR(" - High water: %d", high_water); LOG_ERROR("Solution:"); LOG_ERROR(" Increase fanin spill pool capacity (current: %d, recommended: %d).", capacity, capacity * 2); - LOG_ERROR(" Compile-time: PTO2_DEP_LIST_POOL_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", capacity * 2); LOG_ERROR("========================================"); if (error_code_ptr) { @@ -763,8 +763,8 @@ inline FaninForEachReturn for_each_fanin_slot_state(const TaskPayload &paylo * Linear counters (top, tail) grow monotonically; the physical index * is obtained via modulo: base[linear_index % capacity]. */ -struct PTO2DepListPool { - PTO2DepListEntry *base; // Pool base address +struct DepListPool { + DepListEntry *base; // Pool base address int32_t capacity; // Total number of entries int32_t top; // Linear next-allocation counter (starts from 1) int32_t tail; // Linear first-alive counter (entries before this are dead) @@ -783,7 +783,7 @@ struct PTO2DepListPool { * @param base Pool base address from shared memory * @param capacity Total number of entries */ - void init(PTO2DepListEntry *in_base, int32_t in_capacity, std::atomic *in_error_code_ptr) { + void init(DepListEntry *in_base, int32_t in_capacity, std::atomic *in_error_code_ptr) { base = in_base; capacity = in_capacity; top = 1; // Start from 1, 0 means NULL/empty @@ -856,7 +856,7 @@ struct PTO2DepListPool { * * @return Pointer to allocated entry, or nullptr on fatal error */ - PTO2DepListEntry *alloc() { + DepListEntry *alloc() { int32_t used = top - tail; if (used >= capacity) { LOG_ERROR("========================================"); @@ -868,7 +868,7 @@ struct PTO2DepListPool { LOG_ERROR(" - High water: %d", high_water); LOG_ERROR("Solution:"); LOG_ERROR(" Increase dep pool capacity (current: %d, recommended: %d).", capacity, capacity * 2); - LOG_ERROR(" Compile-time: PTO2_DEP_LIST_POOL_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", capacity * 2); LOG_ERROR("========================================"); if (error_code_ptr) { @@ -902,8 +902,8 @@ struct PTO2DepListPool { * @param task_slot Task slot to prepend * @return New head offset */ - PTO2DepListEntry *prepend(PTO2DepListEntry *cur, ChipTaskSlotState *slot_state) { - PTO2DepListEntry *new_entry = alloc(); + DepListEntry *prepend(DepListEntry *cur, ChipTaskSlotState *slot_state) { + DepListEntry *new_entry = alloc(); if (!new_entry) return nullptr; new_entry->slot_state = slot_state; new_entry->next = cur; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index 94ec967711..7d2c7990aa 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -195,7 +195,7 @@ struct RuntimeContext { * Phase 2/3. */ RuntimeArenaLayout runtime_reserve_layout( - DeviceArena &arena, uint64_t task_window_size, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE + DeviceArena &arena, uint64_t task_window_size, int32_t dep_pool_capacity = CHIP_DEP_LIST_POOL_SIZE ); RuntimeArenaLayout runtime_reserve_layout( DeviceArena &arena, const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 6cc9de5ffe..43c81f085e 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -70,7 +70,7 @@ // Memory pools (per-ring defaults; total = value × CHIP_MAX_RING_DEPTH) #define PTO2_HEAP_SIZE (256 * 1024 * 1024) // 256MB per ring (1GB total) -#define PTO2_DEP_LIST_POOL_SIZE 16384 // Per-ring dependency list pool entries +#define CHIP_DEP_LIST_POOL_SIZE 16384 // Per-ring dependency list pool entries #define CHIP_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool #define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) @@ -99,11 +99,11 @@ // Dependency-degree diagnostic: log once at debug level when a task's fanin or // a producer's fanout first exceeds this degree, so dense dependency graphs can // be inspected without adding noise to normal runtime logs. -#define PTO2_DEP_DEGREE_DEBUG_THRESHOLD 16 +#define CHIP_DEP_DEGREE_DEBUG_THRESHOLD 16 // TensorMap cleanup interval #define CHIP_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks -#define PTO2_DEP_POOL_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks +#define CHIP_DEP_POOL_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle // count (PTO2_TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp @@ -197,9 +197,9 @@ static_assert(sizeof(FaninSpillEntry) == sizeof(uintptr_t)); * Dependency list entry (singly-linked list node) * Stored in DepListPool ring buffer. */ -struct PTO2DepListEntry { +struct DepListEntry { ChipTaskSlotState *slot_state; // Consumer slot state (direct pointer) - PTO2DepListEntry *next; // next entry + DepListEntry *next; // next entry }; // ============================================================================= @@ -475,7 +475,7 @@ struct alignas(64) ChipTaskSlotState { std::atomic fanout_lock; // Per-task spinlock (0=unlocked, 1=locked) uint32_t fanout_count; // SCOPE_BIT (owning scope) | number of consumers - PTO2DepListEntry *fanout_head; // Pointer to first fanout entry (nullptr = empty) + DepListEntry *fanout_head; // Pointer to first fanout entry (nullptr = empty) // Task state (completion, consumed check, ready check) std::atomic task_state; // PENDING/COMPLETED/CONSUMED diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 30d497d4db..50792d3f1f 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -460,7 +460,7 @@ struct SchedulerState { std::atomic advance_lock; // multi-thread CAS // --- Cache Line 1+: Orch-side wiring dep_pool --- - alignas(64) PTO2DepListPool dep_pool; + alignas(64) DepListPool dep_pool; #if SIMPLER_DFX // Published only for scope_stats; orchestrator must not read dep_pool's non-atomic counters directly. alignas(64) std::atomic dep_pool_snapshot_tail; @@ -959,7 +959,7 @@ struct SchedulerState { p.unlock_fanout(); return; } - PTO2DepListEntry *edge = p.fanout_head; + DepListEntry *edge = p.fanout_head; p.unlock_fanout(); for (; edge != nullptr; edge = edge->next) { ChipTaskSlotState *c = edge->slot_state; @@ -1187,7 +1187,7 @@ struct SchedulerState { slot_state.lock_fanout(); #endif slot_state.mark_completed(); - PTO2DepListEntry *current = slot_state.fanout_head; // Protected by fanout_lock + DepListEntry *current = slot_state.fanout_head; // Protected by fanout_lock slot_state.unlock_fanout(); #if SIMPLER_SCHED_PROFILING @@ -1287,7 +1287,7 @@ struct SchedulerState { // per-ring dep_pool entries) on the supplied arena. // Capacities are baked into the returned layout; init_data_from_layout uses // the same values. - static SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE); + static SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = CHIP_DEP_LIST_POOL_SIZE); static SchedulerLayout reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[CHIP_MAX_RING_DEPTH]); // Phase 3a: write everything *except* arena-internal pointer fields. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index b60e1f6331..a909a17a6b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -157,7 +157,7 @@ SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capaci // Force a cache-line base so Orch-side dep_pool writes do not invalidate // adjacent multi-threaded regions like ready_queue.slots. layout.off_dep_pool_entries[r] = - arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(PTO2DepListEntry), PTO2_ALIGN_SIZE); + arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(DepListEntry), PTO2_ALIGN_SIZE); } return layout; } @@ -215,8 +215,8 @@ bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, Device auto *orch_err = sm_layout::orch_error_code_addr(sm_dev_base); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - auto *dep_entries = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); - memset(dep_entries, 0, static_cast(layout.dep_pool_capacities[r]) * sizeof(PTO2DepListEntry)); + auto *dep_entries = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); + memset(dep_entries, 0, static_cast(layout.dep_pool_capacities[r]) * sizeof(DepListEntry)); sched->ring_sched_states[r].dep_pool.init(dep_entries, layout.dep_pool_capacities[r], orch_err); sched->ring_sched_states[r].dep_pool.set_reclaim_publication_request( &sched->publication_request_mask, &sched->publication_ack_mask, static_cast(r) @@ -275,7 +275,7 @@ void SchedulerState::wire_arena_pointers(const SchedulerLayout &layout, DeviceAr ready_queue_wire_arena_pointers(&sched->early_sync_start_queue, arena, layout.off_early_sync_start_queue_slots); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { auto &dep_pool = sched->ring_sched_states[r].dep_pool; - dep_pool.base = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); + dep_pool.base = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); dep_pool.set_reclaim_publication_request( &sched->publication_request_mask, &sched->publication_ack_mask, static_cast(r) ); diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/kernels/orchestration/chain_barrier_orch.cpp b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/kernels/orchestration/chain_barrier_orch.cpp index fecdb7a78b..2db7b7b57a 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/kernels/orchestration/chain_barrier_orch.cpp +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/kernels/orchestration/chain_barrier_orch.cpp @@ -37,7 +37,7 @@ #define FUNC_COPY_FIRST 1 // Stack room for producer_ids[]. 500 covers everything we expect to test; -// PTO2_DEP_LIST_POOL_SIZE (16384) is the real ceiling on a per-ring basis. +// CHIP_DEP_LIST_POOL_SIZE (16384) is the real ceiling on a per-ring basis. static constexpr int32_t MAX_PRODUCERS = 500; extern "C" { diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp b/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp index 4a9e44b52f..48a9ce9153 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp @@ -49,7 +49,7 @@ #define FUNC_COPY_FIRST 1 static constexpr int32_t LONG_CHAIN_DUMMIES = 4; -// case=4 exceeds PTO2_DEP_DEGREE_DEBUG_THRESHOLD (16), exercising both the +// case=4 exceeds CHIP_DEP_DEGREE_DEBUG_THRESHOLD (16), exercising both the // producer-fanout and final-consumer-fanin debug diagnostics. static constexpr int32_t DENSE_DEP_COUNT = 18; diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/kernels/orchestration/chain_barrier_orch.cpp b/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/kernels/orchestration/chain_barrier_orch.cpp index fecdb7a78b..2db7b7b57a 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/kernels/orchestration/chain_barrier_orch.cpp +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/kernels/orchestration/chain_barrier_orch.cpp @@ -37,7 +37,7 @@ #define FUNC_COPY_FIRST 1 // Stack room for producer_ids[]. 500 covers everything we expect to test; -// PTO2_DEP_LIST_POOL_SIZE (16384) is the real ceiling on a per-ring basis. +// CHIP_DEP_LIST_POOL_SIZE (16384) is the real ceiling on a per-ring basis. static constexpr int32_t MAX_PRODUCERS = 500; extern "C" { diff --git a/tests/st/a5/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp b/tests/st/a5/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp index 4a9e44b52f..48a9ce9153 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp +++ b/tests/st/a5/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp @@ -49,7 +49,7 @@ #define FUNC_COPY_FIRST 1 static constexpr int32_t LONG_CHAIN_DUMMIES = 4; -// case=4 exceeds PTO2_DEP_DEGREE_DEBUG_THRESHOLD (16), exercising both the +// case=4 exceeds CHIP_DEP_DEGREE_DEBUG_THRESHOLD (16), exercising both the // producer-fanout and final-consumer-fanin debug diagnostics. static constexpr int32_t DENSE_DEP_COUNT = 18; diff --git a/tests/ut/cpp/a2a3/test_dep_list_pool.cpp b/tests/ut/cpp/a2a3/test_dep_list_pool.cpp index 160f4be764..f3b91bd6c8 100644 --- a/tests/ut/cpp/a2a3/test_dep_list_pool.cpp +++ b/tests/ut/cpp/a2a3/test_dep_list_pool.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2DepListPool from ring_buffer.h + * Unit tests for DepListPool from ring_buffer.h * * Tests dependency list pool allocation, prepend chaining, overflow detection, * tail advancement, and high-water mark tracking. @@ -39,9 +39,9 @@ class DepListPoolTest : public ::testing::Test { protected: static constexpr int32_t POOL_CAP = 8; - PTO2DepListEntry entries[POOL_CAP]{}; + DepListEntry entries[POOL_CAP]{}; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2DepListPool pool{}; + DepListPool pool{}; void SetUp() override { std::memset(entries, 0, sizeof(entries)); @@ -60,7 +60,7 @@ TEST_F(DepListPoolTest, InitialState) { } TEST_F(DepListPoolTest, SingleAlloc) { - PTO2DepListEntry *entry = pool.alloc(); + DepListEntry *entry = pool.alloc(); ASSERT_NE(entry, nullptr); EXPECT_EQ(pool.used(), 1); EXPECT_EQ(pool.available(), POOL_CAP - 1); @@ -68,13 +68,13 @@ TEST_F(DepListPoolTest, SingleAlloc) { TEST_F(DepListPoolTest, OverflowDetection) { for (int i = 0; i < POOL_CAP; i++) { - PTO2DepListEntry *e = pool.alloc(); + DepListEntry *e = pool.alloc(); ASSERT_NE(e, nullptr) << "Unexpected failure at alloc " << i; } EXPECT_EQ(pool.used(), POOL_CAP); EXPECT_EQ(pool.available(), 0); - PTO2DepListEntry *overflow = pool.alloc(); + DepListEntry *overflow = pool.alloc(); EXPECT_EQ(overflow, nullptr); EXPECT_EQ(error_code.load(), SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); } @@ -96,7 +96,7 @@ TEST_F(DepListPoolTest, EnsureSpaceDeadlockReturnsFalseAndLatchesError) { // Prepend builds LIFO linked list: verify each slot_state pointer. TEST_F(DepListPoolTest, PrependChainCorrectness) { ChipTaskSlotState slots[5]{}; - PTO2DepListEntry *head = nullptr; + DepListEntry *head = nullptr; for (int i = 0; i < 5; i++) { head = pool.prepend(head, &slots[i]); @@ -104,7 +104,7 @@ TEST_F(DepListPoolTest, PrependChainCorrectness) { } // LIFO order: head -> slots[4] -> slots[3] -> ... -> slots[0] -> nullptr. - PTO2DepListEntry *cur = head; + DepListEntry *cur = head; for (int i = 4; i >= 0; i--) { ASSERT_NE(cur, nullptr); EXPECT_EQ(cur->slot_state, &slots[i]) << "Entry " << (4 - i) << " should point to slots[" << i << "]"; @@ -158,18 +158,18 @@ TEST_F(DepListPoolTest, HighWaterAccuracy) { // Prepend chain integrity under pool exhaustion: chain must be walkable. TEST_F(DepListPoolTest, PrependUnderExhaustion) { ChipTaskSlotState slots[POOL_CAP]{}; - PTO2DepListEntry *head = nullptr; + DepListEntry *head = nullptr; int count = 0; while (count < POOL_CAP + 5) { - PTO2DepListEntry *new_head = pool.prepend(head, &slots[count % POOL_CAP]); + DepListEntry *new_head = pool.prepend(head, &slots[count % POOL_CAP]); if (!new_head) break; head = new_head; count++; } int walk = 0; - PTO2DepListEntry *cur = head; + DepListEntry *cur = head; while (cur) { walk++; cur = cur->next; diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index 43c2f6d4fc..31926a301e 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -691,7 +691,7 @@ TEST_F(WiringTest, EarlyDispatchFanoutWaitsForDoorbellPass) { producer.payload->published_block_count.store(1, std::memory_order_relaxed); consumer.payload->fanin_actual_count = 1; - PTO2DepListEntry dep{}; + DepListEntry dep{}; dep.slot_state = &consumer; producer.fanout_head = &dep; @@ -813,7 +813,7 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); downstream.payload->fanin_actual_count = 1; - PTO2DepListEntry dep{}; + DepListEntry dep{}; dep.slot_state = &downstream; sync_consumer.fanout_head = &dep; @@ -850,7 +850,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); downstream.payload->fanin_actual_count = 1; - PTO2DepListEntry dep{}; + DepListEntry dep{}; dep.slot_state = &downstream; sync_consumer.fanout_head = &dep; @@ -1040,7 +1040,7 @@ TEST_F(WiringTest, UnflaggedProducerDoesNotPropagate) { consumer.payload = &cons_payload; cons_payload.fanin_actual_count = 1; - PTO2DepListEntry dep{}; + DepListEntry dep{}; dep.slot_state = &consumer; dep.next = nullptr; producer.fanout_head = &dep; @@ -1110,7 +1110,7 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { consumer2.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); // Build fanout chain: producer -> consumer2 -> consumer1 - PTO2DepListEntry dep_entries[2]; + DepListEntry dep_entries[2]; dep_entries[0].slot_state = &consumer1; dep_entries[0].next = nullptr; dep_entries[1].slot_state = &consumer2; diff --git a/tests/ut/cpp/a5/test_dep_list_pool.cpp b/tests/ut/cpp/a5/test_dep_list_pool.cpp index 160f4be764..f3b91bd6c8 100644 --- a/tests/ut/cpp/a5/test_dep_list_pool.cpp +++ b/tests/ut/cpp/a5/test_dep_list_pool.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2DepListPool from ring_buffer.h + * Unit tests for DepListPool from ring_buffer.h * * Tests dependency list pool allocation, prepend chaining, overflow detection, * tail advancement, and high-water mark tracking. @@ -39,9 +39,9 @@ class DepListPoolTest : public ::testing::Test { protected: static constexpr int32_t POOL_CAP = 8; - PTO2DepListEntry entries[POOL_CAP]{}; + DepListEntry entries[POOL_CAP]{}; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2DepListPool pool{}; + DepListPool pool{}; void SetUp() override { std::memset(entries, 0, sizeof(entries)); @@ -60,7 +60,7 @@ TEST_F(DepListPoolTest, InitialState) { } TEST_F(DepListPoolTest, SingleAlloc) { - PTO2DepListEntry *entry = pool.alloc(); + DepListEntry *entry = pool.alloc(); ASSERT_NE(entry, nullptr); EXPECT_EQ(pool.used(), 1); EXPECT_EQ(pool.available(), POOL_CAP - 1); @@ -68,13 +68,13 @@ TEST_F(DepListPoolTest, SingleAlloc) { TEST_F(DepListPoolTest, OverflowDetection) { for (int i = 0; i < POOL_CAP; i++) { - PTO2DepListEntry *e = pool.alloc(); + DepListEntry *e = pool.alloc(); ASSERT_NE(e, nullptr) << "Unexpected failure at alloc " << i; } EXPECT_EQ(pool.used(), POOL_CAP); EXPECT_EQ(pool.available(), 0); - PTO2DepListEntry *overflow = pool.alloc(); + DepListEntry *overflow = pool.alloc(); EXPECT_EQ(overflow, nullptr); EXPECT_EQ(error_code.load(), SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); } @@ -96,7 +96,7 @@ TEST_F(DepListPoolTest, EnsureSpaceDeadlockReturnsFalseAndLatchesError) { // Prepend builds LIFO linked list: verify each slot_state pointer. TEST_F(DepListPoolTest, PrependChainCorrectness) { ChipTaskSlotState slots[5]{}; - PTO2DepListEntry *head = nullptr; + DepListEntry *head = nullptr; for (int i = 0; i < 5; i++) { head = pool.prepend(head, &slots[i]); @@ -104,7 +104,7 @@ TEST_F(DepListPoolTest, PrependChainCorrectness) { } // LIFO order: head -> slots[4] -> slots[3] -> ... -> slots[0] -> nullptr. - PTO2DepListEntry *cur = head; + DepListEntry *cur = head; for (int i = 4; i >= 0; i--) { ASSERT_NE(cur, nullptr); EXPECT_EQ(cur->slot_state, &slots[i]) << "Entry " << (4 - i) << " should point to slots[" << i << "]"; @@ -158,18 +158,18 @@ TEST_F(DepListPoolTest, HighWaterAccuracy) { // Prepend chain integrity under pool exhaustion: chain must be walkable. TEST_F(DepListPoolTest, PrependUnderExhaustion) { ChipTaskSlotState slots[POOL_CAP]{}; - PTO2DepListEntry *head = nullptr; + DepListEntry *head = nullptr; int count = 0; while (count < POOL_CAP + 5) { - PTO2DepListEntry *new_head = pool.prepend(head, &slots[count % POOL_CAP]); + DepListEntry *new_head = pool.prepend(head, &slots[count % POOL_CAP]); if (!new_head) break; head = new_head; count++; } int walk = 0; - PTO2DepListEntry *cur = head; + DepListEntry *cur = head; while (cur) { walk++; cur = cur->next; diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index eca549ceed..c87ff6e9e2 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -324,7 +324,7 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); downstream.payload->fanin_actual_count = 1; - PTO2DepListEntry dep{}; + DepListEntry dep{}; dep.slot_state = &downstream; sync_consumer.fanout_head = &dep; @@ -359,7 +359,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); downstream.payload->fanin_actual_count = 1; - PTO2DepListEntry dep{}; + DepListEntry dep{}; dep.slot_state = &downstream; sync_consumer.fanout_head = &dep; @@ -424,7 +424,7 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { consumer2.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); // Build fanout chain: producer -> consumer2 -> consumer1 - PTO2DepListEntry dep_entries[2]; + DepListEntry dep_entries[2]; dep_entries[0].slot_state = &consumer1; dep_entries[0].next = nullptr; dep_entries[1].slot_state = &consumer2; diff --git a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp index 06a3f8953e..90b675b2af 100644 --- a/tests/ut/cpp/common/test_scope_deadlock_detection.cpp +++ b/tests/ut/cpp/common/test_scope_deadlock_detection.cpp @@ -35,9 +35,9 @@ void make_head_match_old_structural_predicate( } // namespace TEST(ScopeDeadlockDetectionTest, DepPoolUsesTimeoutForDifferentScopeHead) { - PTO2DepListEntry entries[POOL_CAPACITY]{}; + DepListEntry entries[POOL_CAPACITY]{}; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2DepListPool pool; + DepListPool pool; pool.init(entries, POOL_CAPACITY, &error_code); for (int32_t i = 0; i < POOL_CAPACITY; ++i) { ASSERT_NE(pool.alloc(), nullptr); @@ -67,9 +67,9 @@ TEST(ScopeDeadlockDetectionTest, DepPoolUsesTimeoutForDifferentScopeHead) { } TEST(ScopeDeadlockDetectionTest, DepPoolRejectsCurrentScopeHeadStructurally) { - PTO2DepListEntry entries[POOL_CAPACITY]{}; + DepListEntry entries[POOL_CAPACITY]{}; std::atomic error_code{SIMPLER_ERROR_NONE}; - PTO2DepListPool pool; + DepListPool pool; pool.init(entries, POOL_CAPACITY, &error_code); for (int32_t i = 0; i < POOL_CAPACITY; ++i) { ASSERT_NE(pool.alloc(), nullptr); From aac8af1ddde1da2f030e45f6a1b121dd6ebe0e85 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:46:14 -0700 Subject: [PATCH 13/22] Refactor: retire the PTO2 prefix on the early-dispatch state machines The two state machines lose the prefix together: `EarlyDispatchState` with its `EARLY_DISPATCH_*` values and the `EarlyDispatchLaunchState` sub-state, `EarlySyncDrainState` with its `EARLY_SYNC_DRAIN_*` values, and `EARLY_DISPATCH_CORE_MASK_WORDS`. 356 occurrences across 25 files. An enum and its enumerators cannot be split across commits without leaving the type and its values on two conventions, and the drain state is the handshake the dispatch state waits on, so the two read as one mechanism. `CHIP_EARLY_DISPATCH_QUEUE_SIZE` keeps a prefix: unlike the enumerators, it is an object-like macro in `runtime/runtime_types.h`, so a bare name would capture the identifier textually across every including translation unit. Co-Authored-By: Claude Opus 5 (1M context) --- .../orchestrator_core/orchestrator.cpp | 6 +- .../host_build_graph/runtime/runtime_types.h | 48 +++++------ .../runtime/scheduler/scheduler.h | 41 +++++----- .../scheduler/scheduler_completion.cpp | 13 ++- .../runtime/scheduler/scheduler_dispatch.cpp | 21 +++-- .../runtime/shared/runtime_init.cpp | 8 +- .../runtime/runtime_types.h | 48 +++++------ .../runtime/scheduler/scheduler.h | 71 ++++++++-------- .../scheduler/scheduler_completion.cpp | 13 ++- .../runtime/scheduler/scheduler_dispatch.cpp | 21 +++-- .../runtime/shared/runtime_init.cpp | 8 +- .../orchestrator_core/orchestrator.cpp | 6 +- .../host_build_graph/runtime/runtime_types.h | 48 +++++------ .../runtime/scheduler/scheduler.h | 41 +++++----- .../scheduler/scheduler_completion.cpp | 13 ++- .../runtime/scheduler/scheduler_dispatch.cpp | 21 +++-- .../runtime/shared/runtime_init.cpp | 8 +- .../runtime/runtime_types.h | 48 +++++------ .../runtime/scheduler/scheduler.h | 71 ++++++++-------- .../scheduler/scheduler_completion.cpp | 13 ++- .../runtime/scheduler/scheduler_dispatch.cpp | 18 ++--- .../runtime/shared/runtime_init.cpp | 8 +- .../host_build_graph/graph_execution.cpp | 8 +- tests/ut/cpp/a2a3/test_wiring.cpp | 81 +++++++++---------- tests/ut/cpp/a5/test_wiring.cpp | 31 ++++--- 25 files changed, 348 insertions(+), 365 deletions(-) diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 90983af1b4..ea9fefd4e1 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1739,15 +1739,15 @@ void graph_reset_outer_payload(TaskPayload &payload) { payload.scalar_count = 0; payload.fanin_count = 0; payload.predicate = DispatchPredicate{}; - payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_NONE, std::memory_order_relaxed); + payload.early_dispatch_state.store(EARLY_DISPATCH_NONE, std::memory_order_relaxed); for (auto &word : payload.staged_core_mask) word.store(0, std::memory_order_relaxed); payload.dispatch_fanin.store(0, std::memory_order_relaxed); payload.dispatch_propagated.store(0, std::memory_order_relaxed); payload.published_block_count.store(0, std::memory_order_relaxed); - payload.early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); + payload.early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); payload.running_slot_count.store(0, std::memory_order_relaxed); - payload.early_sync_drain_state.store(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); + payload.early_sync_drain_state.store(EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); } bool graph_submit_outer( diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index bde7a6e49a..774b33bdcc 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -128,7 +128,7 @@ inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HE #define CHIP_READY_QUEUE_SIZE 8192 // Cross-thread early-dispatch work queue (power of two) -#define PTO2_EARLY_DISPATCH_QUEUE_SIZE 64 +#define CHIP_EARLY_DISPATCH_QUEUE_SIZE 64 // Fanin storage #define CHIP_FANIN_INLINE_CAP 64 @@ -261,25 +261,25 @@ static_assert(offsetof(TaskDescriptor, packed_buffer_base) == 24, "packed_buffer * fanins spill into a per-ring ring buffer slice. */ // Early-dispatch claim states for TaskPayload::early_dispatch_state. -enum PTO2EarlyDispatchState : uint8_t { - PTO2_EARLY_DISPATCH_NONE = 0, // not pre-staged - PTO2_EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress - PTO2_EARLY_DISPATCH_STAGED = 2, // reserved - PTO2_EARLY_DISPATCH_DISPATCHED = 3 // producers released; staged blocks may still be gated +enum EarlyDispatchState : uint8_t { + EARLY_DISPATCH_NONE = 0, // not pre-staged + EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress + EARLY_DISPATCH_STAGED = 2, // reserved + EARLY_DISPATCH_DISPATCHED = 3 // producers released; staged blocks may still be gated }; -enum PTO2EarlyDispatchLaunchState : uint8_t { - PTO2_EARLY_DISPATCH_LAUNCH_NONE = 0, - PTO2_EARLY_DISPATCH_LAUNCH_RINGING = 1, - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE = 2, +enum EarlyDispatchLaunchState : uint8_t { + EARLY_DISPATCH_LAUNCH_NONE = 0, + EARLY_DISPATCH_LAUNCH_RINGING = 1, + EARLY_DISPATCH_LAUNCH_COMPLETE = 2, }; -enum PTO2EarlySyncDrainState : uint8_t { - PTO2_EARLY_SYNC_DRAIN_NONE = 0, - PTO2_EARLY_SYNC_DRAIN_OWNER = 1 << 0, - PTO2_EARLY_SYNC_DRAIN_ARMED = 1 << 1, - PTO2_EARLY_SYNC_DRAIN_READY = 1 << 2, - PTO2_EARLY_SYNC_DRAIN_COMPLETE = 1 << 3, +enum EarlySyncDrainState : uint8_t { + EARLY_SYNC_DRAIN_NONE = 0, + EARLY_SYNC_DRAIN_OWNER = 1 << 0, + EARLY_SYNC_DRAIN_ARMED = 1 << 1, + EARLY_SYNC_DRAIN_READY = 1 << 2, + EARLY_SYNC_DRAIN_COMPLETE = 1 << 3, }; // A pre-staged consumer occupies one core per gated subtask block. WHICH cores @@ -289,7 +289,7 @@ enum PTO2EarlySyncDrainState : uint8_t { // chip's core count (RUNTIME_MAX_WORKER = 72; no two-level pre-dispatch means // gated cores in flight <= core count), NOT by block_num — so a wide SPMD // consumer can pre-stage all its idle cores. 2 words = 128 bits >= 72. -inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; +inline constexpr int EARLY_DISPATCH_CORE_MASK_WORDS = 2; struct TaskPayload { // === Cache line 0 (64B) — the dispatch path's own line === @@ -327,7 +327,7 @@ struct TaskPayload { // splits them between release and late-stager owners; a sync_start cohort keeps // the completed mask stable for its single launch owner, whether staging is local // or uses the global drain fallback. - alignas(64) std::atomic staged_core_mask[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS]{}; + alignas(64) std::atomic staged_core_mask[EARLY_DISPATCH_CORE_MASK_WORDS]{}; // Early-dispatch CANDIDATE detection (event-driven, dual of fanin_refcount): // seeded at wiring with producers already complete, then a flagged producer // bumps each consumer after all of its logical blocks are published. @@ -352,7 +352,7 @@ struct TaskPayload { std::atomic dispatch_propagated{0}; // PRODUCER side: once-guard for fanout propagation // The launch owner publishes COMPLETE only after all owned doorbells are // visible, keeping fanout private until every gated block has launched. - std::atomic early_dispatch_launch_state{PTO2_EARLY_DISPATCH_LAUNCH_NONE}; + std::atomic early_dispatch_launch_state{EARLY_DISPATCH_LAUNCH_NONE}; // sync_start early-dispatch rendezvous: count of this task's gated CORES currently // occupying a RUNNING slot (staged directly to an idle core, or promoted from a // gated pending slot). Counted per-core (not per-block) so it is shape-agnostic: a @@ -364,7 +364,7 @@ struct TaskPayload { // A successful OWNER persists through ARMED and COMPLETE until payload // reinitialization. READY records that producer release observed OWNER; // only cancellation clears OWNER during the current task lifetime. - std::atomic early_sync_drain_state{PTO2_EARLY_SYNC_DRAIN_NONE}; + std::atomic early_sync_drain_state{EARLY_SYNC_DRAIN_NONE}; // === Cache line 2 — dispatch predicate + dump metadata (AICPU-only) === // AICore never reads either — args are materialized from line 0's counts and // region deltas. Resolved at submit; evaluated by the scheduler at dispatch. @@ -486,15 +486,15 @@ struct TaskPayload { // consumer's own hint). So they MUST be zeroed here unconditionally. // Publication, propagation, and launch fields share this same // per-submit lifetime and are reset here too. - early_dispatch_state.store(PTO2_EARLY_DISPATCH_NONE, std::memory_order_relaxed); - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + early_dispatch_state.store(EARLY_DISPATCH_NONE, std::memory_order_relaxed); + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) staged_core_mask[w].store(0, std::memory_order_relaxed); dispatch_fanin.store(0, std::memory_order_relaxed); dispatch_propagated.store(0, std::memory_order_relaxed); published_block_count.store(0, std::memory_order_relaxed); - early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); + early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); running_slot_count.store(0, std::memory_order_relaxed); - early_sync_drain_state.store(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); + early_sync_drain_state.store(EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); } }; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h index 26569d83a7..6f4e50fdb5 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -687,7 +687,7 @@ struct SchedulerState { uint64_t addr{0}; uint32_t token{0}; }; - EarlyDispatchDoorbell early_dispatch_doorbell_table[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; + EarlyDispatchDoorbell early_dispatch_doorbell_table[EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; // Cross-thread early-dispatch work queues, one ChipReadyQueue MPMC instance per // resource shape (AIC/AIV/MIX) — arena-backed, reserved/wired in pto_runtime2_init @@ -747,7 +747,7 @@ struct SchedulerState { } static inline bool should_gate_early_dispatch(bool force_gate, uint8_t early_dispatch_state) { - return force_gate || early_dispatch_state == PTO2_EARLY_DISPATCH_STAGING; + return force_gate || early_dispatch_state == EARLY_DISPATCH_STAGING; } static inline bool @@ -758,9 +758,9 @@ struct SchedulerState { } static inline bool try_claim_early_dispatch_launch(TaskPayload &payload) { - uint8_t expected = PTO2_EARLY_DISPATCH_LAUNCH_NONE; + uint8_t expected = EARLY_DISPATCH_LAUNCH_NONE; return payload.early_dispatch_launch_state.compare_exchange_strong( - expected, PTO2_EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst ); } @@ -773,7 +773,7 @@ struct SchedulerState { // the NONE->RINGING launch latch and invokes this exactly once after local or // global staging completes, while the corresponding per-core table entries are live. inline void ring_all_staged_doorbells(ChipTaskSlotState &slot_state) { - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { uint64_t bits = slot_state.payload->staged_core_mask[w].load(std::memory_order_seq_cst); while (bits != 0) { int core_id = w * 64 + __builtin_ctzll(bits); @@ -786,43 +786,42 @@ struct SchedulerState { } static inline bool try_claim_early_sync_drain(TaskPayload &payload) { - uint8_t expected = PTO2_EARLY_SYNC_DRAIN_NONE; + uint8_t expected = EARLY_SYNC_DRAIN_NONE; return payload.early_sync_drain_state.compare_exchange_strong( - expected, PTO2_EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst ); } static inline bool owns_early_sync_drain(const TaskPayload &payload) { - return (payload.early_sync_drain_state.load(std::memory_order_acquire) & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; + return (payload.early_sync_drain_state.load(std::memory_order_acquire) & EARLY_SYNC_DRAIN_OWNER) != 0; } static inline void mark_early_sync_drain_armed(TaskPayload &payload) { - payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); + payload.early_sync_drain_state.fetch_or(EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); } static inline bool publish_ready_to_early_sync_drain(TaskPayload &payload) { - uint8_t previous = - payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); - return (previous & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; + uint8_t previous = payload.early_sync_drain_state.fetch_or(EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); + return (previous & EARLY_SYNC_DRAIN_OWNER) != 0; } inline void cancel_early_sync_drain(ChipTaskSlotState &slot_state) { uint8_t previous = - slot_state.payload->early_sync_drain_state.exchange(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_seq_cst); - if ((previous & PTO2_EARLY_SYNC_DRAIN_OWNER) == 0) return; - if ((previous & PTO2_EARLY_SYNC_DRAIN_READY) != 0) { + slot_state.payload->early_sync_drain_state.exchange(EARLY_SYNC_DRAIN_NONE, std::memory_order_seq_cst); + if ((previous & EARLY_SYNC_DRAIN_OWNER) == 0) return; + if ((previous & EARLY_SYNC_DRAIN_READY) != 0) { push_ready_routed(&slot_state); return; } - if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) == PTO2_EARLY_DISPATCH_STAGING) { + if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) == EARLY_DISPATCH_STAGING) { early_sync_start_queue.push_tagged(&slot_state, static_cast(slot_state.task->task_id.raw)); } } static inline void finish_early_sync_drain(TaskPayload &payload) { uint8_t state = payload.early_sync_drain_state.load(std::memory_order_seq_cst); - while ((state & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0 && (state & PTO2_EARLY_SYNC_DRAIN_COMPLETE) == 0) { - uint8_t desired = state | PTO2_EARLY_SYNC_DRAIN_COMPLETE; + while ((state & EARLY_SYNC_DRAIN_OWNER) != 0 && (state & EARLY_SYNC_DRAIN_COMPLETE) == 0) { + uint8_t desired = state | EARLY_SYNC_DRAIN_COMPLETE; if (payload.early_sync_drain_state.compare_exchange_weak( state, desired, std::memory_order_seq_cst, std::memory_order_seq_cst )) { @@ -846,18 +845,18 @@ struct SchedulerState { // mask when producer release races staging. int32_t running_cores = slot_state.payload->running_slot_count.load(std::memory_order_seq_cst); int32_t staged_cores = 0; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) staged_cores += __builtin_popcountll(slot_state.payload->staged_core_mask[w].load(std::memory_order_seq_cst)); if (staged_cores == 0) return false; if (running_cores != staged_cores) return false; - if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_DISPATCHED) + if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) != EARLY_DISPATCH_DISPATCHED) return false; if (!try_claim_early_dispatch_launch(*slot_state.payload)) return false; ring_all_staged_doorbells(slot_state); wmb(); slot_state.payload->early_dispatch_launch_state.store( - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_release + EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_release ); return true; } diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index 5cfdac0007..eee285fa90 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -274,7 +274,7 @@ void SchedulerContext::check_running_cores_for_completion( { ChipTaskSlotState *rs = core.running_slot_state; if (rs != nullptr && rs->payload != nullptr && - rs->payload->early_dispatch_state.load(std::memory_order_relaxed) == PTO2_EARLY_DISPATCH_STAGING) { + rs->payload->early_dispatch_state.load(std::memory_order_relaxed) == EARLY_DISPATCH_STAGING) { continue; } } @@ -314,12 +314,11 @@ void SchedulerContext::check_running_cores_for_completion( uint8_t pending_ss = (core.pending_slot_state != nullptr && core.pending_slot_state->payload != nullptr) ? core.pending_slot_state->payload->early_dispatch_state.load(std::memory_order_relaxed) : - static_cast(PTO2_EARLY_DISPATCH_NONE); + static_cast(EARLY_DISPATCH_NONE); bool pending_gated = (core.pending_slot_state != nullptr && core.pending_slot_state->payload != nullptr && - (pending_ss == PTO2_EARLY_DISPATCH_STAGING || - (pending_ss == PTO2_EARLY_DISPATCH_DISPATCHED && - core.pending_slot_state->task_attrs.requires_sync_start()))); + (pending_ss == EARLY_DISPATCH_STAGING || + (pending_ss == EARLY_DISPATCH_DISPATCHED && core.pending_slot_state->task_attrs.requires_sync_start()))); SlotTransition t = decide_slot_transition( reg_task_id, reg_state, core.running_reg_task_id, core.pending_reg_task_id, pending_gated ); @@ -552,7 +551,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( // per-write atomic contends across all drain threads on the same 2 words and was // ~half the publish cost. The doorbell-table writes stay per-core (unique cid, no // contention). - uint64_t my_mask[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; + uint64_t my_mask[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; for (int i = 0; i < handle_count; i++) { publish_subtask_to_core(handles[i], dispatch_ts, thread_idx); if (gated) { @@ -563,7 +562,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( } } if (gated) { - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_mask[w] != 0) { slot_state->payload->staged_core_mask[w].fetch_or(my_mask[w], std::memory_order_seq_cst); } diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 7d2b580660..d0d0db44f5 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -52,11 +52,10 @@ static_assert(sizeof(ChipTensor) == TASKPAYLOAD_TENSOR_STRIDE); namespace {} -// The early-dispatch core bitmask (PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64 bits) must cover +// The early-dispatch core bitmask (EARLY_DISPATCH_CORE_MASK_WORDS * 64 bits) must cover // every global core_id, and the per-core doorbell table is sized to match. static_assert( - RUNTIME_MAX_WORKER <= PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64, - "staged_core_mask too small for RUNTIME_MAX_WORKER cores" + RUNTIME_MAX_WORKER <= EARLY_DISPATCH_CORE_MASK_WORDS * 64, "staged_core_mask too small for RUNTIME_MAX_WORKER cores" ); const char *SchedulerContext::shape_name(ResourceShape shape) { @@ -627,7 +626,7 @@ int32_t SchedulerContext::stage_consumer_blocks( // Stamp the real pre-stage time (NOT 0) so the swimlane shows these blocks // dispatched during the producer's run, not at trace start. uint64_t early_dispatch_ts = get_sys_cnt_aicpu(); - uint64_t my_cores[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; // cores gated by this staging pass + uint64_t my_cores[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; // cores gated by this staging pass int32_t staged = 0; int32_t block = start; // Mirror the normal flush_publish (scheduler_dispatch.cpp wmb()+publish loop): @@ -664,20 +663,20 @@ int32_t SchedulerContext::stage_consumer_blocks( } // Publish all this thread's gated cores into the shared mask in one OR per word // (vs one per subtask) so release sees them; seq_cst keeps the self-ring order. - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) if (my_cores[w] != 0) c->payload->staged_core_mask[w].fetch_or(my_cores[w], std::memory_order_seq_cst); // Full publication and release are independent events. The seq_cst // state/launch/count operations form a two-sided handshake. A released // block must ring before contributing to the publication count. - bool released = staged > 0 && - c->payload->early_dispatch_state.load(std::memory_order_seq_cst) == PTO2_EARLY_DISPATCH_DISPATCHED; + bool released = + staged > 0 && c->payload->early_dispatch_state.load(std::memory_order_seq_cst) == EARLY_DISPATCH_DISPATCHED; // Claim only bits the release path did not take. Local handles remain valid // even if the shared per-core table is reused before this thread resumes. if (released) { - uint64_t owned[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + uint64_t owned[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_cores[w] != 0) { owned[w] = SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); @@ -737,7 +736,7 @@ SchedulerContext::early_dispatch_shape(int32_t thread_idx, ResourceShape shape, for (int bi = 0; bi < got; bi++) { ChipTaskSlotState *c = batch[bi]; if (static_cast(c->task->task_id.raw) != task_id_snapshots[bi]) continue; - if (c->payload->early_dispatch_state.load(std::memory_order_acquire) != PTO2_EARLY_DISPATCH_STAGING) + if (c->payload->early_dispatch_state.load(std::memory_order_acquire) != EARLY_DISPATCH_STAGING) continue; // released // The single free-core bucket for this phase. For MIX, an active-mask-aware @@ -826,7 +825,7 @@ int32_t SchedulerContext::try_early_dispatch( bool current_sync_task = static_cast(c->task->task_id.raw) == sync_task_id_snapshot && c->task_attrs.requires_sync_start(); if (current_sync_task && SchedulerState::try_claim_early_sync_drain(*c->payload)) { - if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_STAGING) { + if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != EARLY_DISPATCH_STAGING) { sched_->cancel_early_sync_drain(*c); } else if (drain_state_.sync_start_pending.load(std::memory_order_acquire) == 0 && tracker.count_available_blocks( diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp index cc46987eb1..0d6ce3571a 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -86,9 +86,9 @@ SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena) { // ones. The big nine are the arena's last reservations so that the bytes bind // uploads stay one contiguous range no matter how much of them is in use. for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_EARLY_DISPATCH_QUEUE_SIZE); } - layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, CHIP_EARLY_DISPATCH_QUEUE_SIZE); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } @@ -125,9 +125,9 @@ bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, Device ready_queue_init_data_from_layout(&sched->graph_ready_queue, layout.ready_queue_capacity); ready_queue_init_data_from_layout(&sched->graph_prepare_queue, layout.ready_queue_capacity); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - ready_queue_init_data_from_layout(&sched->early_dispatch_queues[i], PTO2_EARLY_DISPATCH_QUEUE_SIZE); + ready_queue_init_data_from_layout(&sched->early_dispatch_queues[i], CHIP_EARLY_DISPATCH_QUEUE_SIZE); } - ready_queue_init_data_from_layout(&sched->early_sync_start_queue, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + ready_queue_init_data_from_layout(&sched->early_sync_start_queue, CHIP_EARLY_DISPATCH_QUEUE_SIZE); // Polling: no dep_pool arena region to initialize. (void)arena; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index fc78c4ee6e..4f853215e0 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -91,7 +91,7 @@ // capacity tracks the publication burst rather than the simultaneously // stageable cohort: a 128-token EP8 routed-expert layer bursts 128 gate/up MM // candidates onto one shape, and this holds 2x that. -#define PTO2_EARLY_DISPATCH_QUEUE_SIZE 256 +#define CHIP_EARLY_DISPATCH_QUEUE_SIZE 256 // Fanin storage #define CHIP_FANIN_INLINE_CAP 64 @@ -237,25 +237,25 @@ static_assert(offsetof(TaskDescriptor, packed_buffer_base) == 24, "packed_buffer // ============================================================================= // Early-dispatch claim states for TaskPayload::early_dispatch_state. -enum PTO2EarlyDispatchState : uint8_t { - PTO2_EARLY_DISPATCH_NONE = 0, // not pre-staged - PTO2_EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress - PTO2_EARLY_DISPATCH_STAGED = 2, // reserved - PTO2_EARLY_DISPATCH_DISPATCHED = 3 // producers released; staged blocks may still be gated +enum EarlyDispatchState : uint8_t { + EARLY_DISPATCH_NONE = 0, // not pre-staged + EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress + EARLY_DISPATCH_STAGED = 2, // reserved + EARLY_DISPATCH_DISPATCHED = 3 // producers released; staged blocks may still be gated }; -enum PTO2EarlyDispatchLaunchState : uint8_t { - PTO2_EARLY_DISPATCH_LAUNCH_NONE = 0, - PTO2_EARLY_DISPATCH_LAUNCH_RINGING = 1, - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE = 2, +enum EarlyDispatchLaunchState : uint8_t { + EARLY_DISPATCH_LAUNCH_NONE = 0, + EARLY_DISPATCH_LAUNCH_RINGING = 1, + EARLY_DISPATCH_LAUNCH_COMPLETE = 2, }; -enum PTO2EarlySyncDrainState : uint8_t { - PTO2_EARLY_SYNC_DRAIN_NONE = 0, - PTO2_EARLY_SYNC_DRAIN_OWNER = 1 << 0, - PTO2_EARLY_SYNC_DRAIN_ARMED = 1 << 1, - PTO2_EARLY_SYNC_DRAIN_READY = 1 << 2, - PTO2_EARLY_SYNC_DRAIN_COMPLETE = 1 << 3, +enum EarlySyncDrainState : uint8_t { + EARLY_SYNC_DRAIN_NONE = 0, + EARLY_SYNC_DRAIN_OWNER = 1 << 0, + EARLY_SYNC_DRAIN_ARMED = 1 << 1, + EARLY_SYNC_DRAIN_READY = 1 << 2, + EARLY_SYNC_DRAIN_COMPLETE = 1 << 3, }; // A pre-staged consumer occupies one core per gated subtask block. WHICH cores @@ -265,7 +265,7 @@ enum PTO2EarlySyncDrainState : uint8_t { // chip's core count (RUNTIME_MAX_WORKER = 72; no two-level pre-dispatch means // gated cores in flight <= core count), NOT by block_num — so a wide SPMD // consumer can pre-stage all its idle cores. 2 words = 128 bits >= 72. -inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; +inline constexpr int EARLY_DISPATCH_CORE_MASK_WORDS = 2; /** * Task payload data (cold path - only accessed during orchestration and dispatch) @@ -295,7 +295,7 @@ struct TaskPayload { // consumer destructively splits those bits between release and late-stager // owners; a sync_start cohort keeps the completed mask stable for one launch // owner across both local staging and the global-drain fallback. - std::atomic staged_core_mask[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS]{}; + std::atomic staged_core_mask[EARLY_DISPATCH_CORE_MASK_WORDS]{}; // Candidate detection is the event-driven dual of fanin_refcount. Wiring // seeds producers already complete, and flagged producers increment the // count only after all logical blocks are launch-visible. Equality with @@ -312,7 +312,7 @@ struct TaskPayload { std::atomic early_dispatch_state{0}; // COMPLETE is published only after every owned doorbell is visible, keeping // fanout private until all gated blocks have launched. - std::atomic early_dispatch_launch_state{PTO2_EARLY_DISPATCH_LAUNCH_NONE}; + std::atomic early_dispatch_launch_state{EARLY_DISPATCH_LAUNCH_NONE}; // Number of this sync_start task's gated cores occupying RUNNING slots. // Counting cores is shape-agnostic for MIX, whose cluster cores promote // independently. Doorbells launch only after this count matches the staged @@ -321,7 +321,7 @@ struct TaskPayload { // OWNER persists through ARMED and COMPLETE for one task lifetime. READY // records that producer release observed the owner; only cancellation clears // ownership before payload reinitialization. - std::atomic early_sync_drain_state{PTO2_EARLY_SYNC_DRAIN_NONE}; + std::atomic early_sync_drain_state{EARLY_SYNC_DRAIN_NONE}; // === Cache line 9 (byte 576) — dispatch predicate (AICPU-only) === // Offset is a fixed 576, independent of MAX_TENSOR_ARGS / MAX_SCALAR_ARGS. // AICore never reads it — args are materialized from the tensor_count / tensors @@ -401,14 +401,14 @@ struct TaskPayload { // are consumer-side state and must be initialized on every submit even // when the consumer itself does not allow early resolve, because one of // its flagged producers may still update them. - early_dispatch_state.store(PTO2_EARLY_DISPATCH_NONE, std::memory_order_relaxed); - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + early_dispatch_state.store(EARLY_DISPATCH_NONE, std::memory_order_relaxed); + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) staged_core_mask[w].store(0, std::memory_order_relaxed); dispatch_fanin.store(0, std::memory_order_relaxed); published_block_count.store(0, std::memory_order_relaxed); - early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); + early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); running_slot_count.store(0, std::memory_order_relaxed); - early_sync_drain_state.store(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); + early_sync_drain_state.store(EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); } }; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 0921cc844a..42fae719fa 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -732,7 +732,7 @@ struct SchedulerState { uint64_t addr{0}; uint32_t token{0}; }; - EarlyDispatchDoorbell early_dispatch_doorbell_table[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; + EarlyDispatchDoorbell early_dispatch_doorbell_table[EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; // Cross-thread early-dispatch work queues, one ChipReadyQueue MPMC instance per // resource shape (AIC/AIV/MIX) — arena-backed, reserved/wired in pto_runtime2_init @@ -797,7 +797,7 @@ struct SchedulerState { } static inline bool should_gate_early_dispatch(bool force_gate, uint8_t early_dispatch_state) { - return force_gate || early_dispatch_state == PTO2_EARLY_DISPATCH_STAGING; + return force_gate || early_dispatch_state == EARLY_DISPATCH_STAGING; } static inline bool @@ -808,9 +808,9 @@ struct SchedulerState { } static inline bool try_claim_early_dispatch_launch(TaskPayload &payload) { - uint8_t expected = PTO2_EARLY_DISPATCH_LAUNCH_NONE; + uint8_t expected = EARLY_DISPATCH_LAUNCH_NONE; return payload.early_dispatch_launch_state.compare_exchange_strong( - expected, PTO2_EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst ); } @@ -823,7 +823,7 @@ struct SchedulerState { // the NONE->RINGING launch latch and invokes this exactly once after local or // global staging completes, while the corresponding per-core table entries are live. inline void ring_all_staged_doorbells(ChipTaskSlotState &slot_state) { - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { uint64_t bits = slot_state.payload->staged_core_mask[w].load(std::memory_order_seq_cst); while (bits != 0) { int core_id = w * 64 + __builtin_ctzll(bits); @@ -836,43 +836,42 @@ struct SchedulerState { } static inline bool try_claim_early_sync_drain(TaskPayload &payload) { - uint8_t expected = PTO2_EARLY_SYNC_DRAIN_NONE; + uint8_t expected = EARLY_SYNC_DRAIN_NONE; return payload.early_sync_drain_state.compare_exchange_strong( - expected, PTO2_EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst ); } static inline bool owns_early_sync_drain(const TaskPayload &payload) { - return (payload.early_sync_drain_state.load(std::memory_order_acquire) & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; + return (payload.early_sync_drain_state.load(std::memory_order_acquire) & EARLY_SYNC_DRAIN_OWNER) != 0; } static inline void mark_early_sync_drain_armed(TaskPayload &payload) { - payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); + payload.early_sync_drain_state.fetch_or(EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); } static inline bool publish_ready_to_early_sync_drain(TaskPayload &payload) { - uint8_t previous = - payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); - return (previous & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; + uint8_t previous = payload.early_sync_drain_state.fetch_or(EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); + return (previous & EARLY_SYNC_DRAIN_OWNER) != 0; } inline void cancel_early_sync_drain(ChipTaskSlotState &slot_state) { uint8_t previous = - slot_state.payload->early_sync_drain_state.exchange(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_seq_cst); - if ((previous & PTO2_EARLY_SYNC_DRAIN_OWNER) == 0) return; - if ((previous & PTO2_EARLY_SYNC_DRAIN_READY) != 0) { + slot_state.payload->early_sync_drain_state.exchange(EARLY_SYNC_DRAIN_NONE, std::memory_order_seq_cst); + if ((previous & EARLY_SYNC_DRAIN_OWNER) == 0) return; + if ((previous & EARLY_SYNC_DRAIN_READY) != 0) { push_ready_routed(&slot_state); return; } - if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) == PTO2_EARLY_DISPATCH_STAGING) { + if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) == EARLY_DISPATCH_STAGING) { early_sync_start_queue.push_tagged(&slot_state, static_cast(slot_state.task->task_id.raw)); } } static inline void finish_early_sync_drain(TaskPayload &payload) { uint8_t state = payload.early_sync_drain_state.load(std::memory_order_seq_cst); - while ((state & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0 && (state & PTO2_EARLY_SYNC_DRAIN_COMPLETE) == 0) { - uint8_t desired = state | PTO2_EARLY_SYNC_DRAIN_COMPLETE; + while ((state & EARLY_SYNC_DRAIN_OWNER) != 0 && (state & EARLY_SYNC_DRAIN_COMPLETE) == 0) { + uint8_t desired = state | EARLY_SYNC_DRAIN_COMPLETE; if (payload.early_sync_drain_state.compare_exchange_weak( state, desired, std::memory_order_seq_cst, std::memory_order_seq_cst )) { @@ -896,18 +895,18 @@ struct SchedulerState { // mask when producer release races staging. int32_t running_cores = slot_state.payload->running_slot_count.load(std::memory_order_seq_cst); int32_t staged_cores = 0; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) staged_cores += __builtin_popcountll(slot_state.payload->staged_core_mask[w].load(std::memory_order_seq_cst)); if (staged_cores == 0) return false; if (running_cores != staged_cores) return false; - if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_DISPATCHED) + if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) != EARLY_DISPATCH_DISPATCHED) return false; if (!try_claim_early_dispatch_launch(*slot_state.payload)) return false; ring_all_staged_doorbells(slot_state); wmb(); slot_state.payload->early_dispatch_launch_state.store( - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_release + EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_release ); return true; } @@ -929,9 +928,9 @@ struct SchedulerState { return; } - uint8_t expected = PTO2_EARLY_DISPATCH_NONE; + uint8_t expected = EARLY_DISPATCH_NONE; if (!consumer.payload->early_dispatch_state.compare_exchange_strong( - expected, PTO2_EARLY_DISPATCH_STAGING, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_DISPATCH_STAGING, std::memory_order_seq_cst, std::memory_order_seq_cst )) { return; } @@ -947,9 +946,9 @@ struct SchedulerState { // drop the speculative claim and let readiness take the ordinary // queue path. A concurrent producer release may already have moved // STAGING to DISPATCHED; that path then owns the transition. - expected = PTO2_EARLY_DISPATCH_STAGING; + expected = EARLY_DISPATCH_STAGING; consumer.payload->early_dispatch_state.compare_exchange_strong( - expected, PTO2_EARLY_DISPATCH_NONE, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_DISPATCH_NONE, std::memory_order_seq_cst, std::memory_order_seq_cst ); } } @@ -972,15 +971,15 @@ struct SchedulerState { void propagate_dispatch_fanin(ChipTaskSlotState &p) { if (!p.task_attrs.allow_early_resolve()) return; // only codegen-flagged (direct) producers propagate if (p.payload->published_block_count.load(std::memory_order_seq_cst) < p.logical_block_num) return; - if (p.payload->early_dispatch_state.load(std::memory_order_acquire) == PTO2_EARLY_DISPATCH_STAGING) return; + if (p.payload->early_dispatch_state.load(std::memory_order_acquire) == EARLY_DISPATCH_STAGING) return; uint8_t launch_state = p.payload->early_dispatch_launch_state.load(std::memory_order_acquire); - if (launch_state == PTO2_EARLY_DISPATCH_LAUNCH_RINGING) return; + if (launch_state == EARLY_DISPATCH_LAUNCH_RINGING) return; if (p.task_attrs.requires_sync_start()) { bool was_pre_staged = false; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { was_pre_staged |= p.payload->staged_core_mask[w].load(std::memory_order_acquire) != 0; } - if (was_pre_staged && launch_state != PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE) return; + if (was_pre_staged && launch_state != EARLY_DISPATCH_LAUNCH_COMPLETE) return; } // The once-claim and fanout snapshot share fanout_lock with wiring, so a // set marker tells wiring that its new edge is outside this snapshot. @@ -1029,21 +1028,21 @@ struct SchedulerState { // A predicated task is evaluated at the ready point, never early-dispatched. if (slot_state.task_attrs.has_predicate()) return false; // Never staged => CAS NONE->DISPATCHED wins => dispatch normally. - uint8_t expect = PTO2_EARLY_DISPATCH_NONE; + uint8_t expect = EARLY_DISPATCH_NONE; if (slot_state.payload->early_dispatch_state.compare_exchange_strong( - expect, PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst, std::memory_order_seq_cst + expect, EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst, std::memory_order_seq_cst )) { return false; } // route_ready_once admits one caller, but keep the helper defensive: a // duplicate that observes or loses an in-progress launch must never // route the same partial task a second time. - if (expect != PTO2_EARLY_DISPATCH_STAGING) return true; + if (expect != EARLY_DISPATCH_STAGING) return true; bool sync_start = slot_state.task_attrs.requires_sync_start(); if (!sync_start && !try_claim_early_dispatch_launch(*slot_state.payload)) return true; - expect = PTO2_EARLY_DISPATCH_STAGING; + expect = EARLY_DISPATCH_STAGING; slot_state.payload->early_dispatch_state.compare_exchange_strong( - expect, PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst, std::memory_order_seq_cst + expect, EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst, std::memory_order_seq_cst ); // Producer released: ring the gated cores. A non-sync_start consumer launches // each block the instant its doorbell fires. A sync_start consumer instead holds @@ -1058,13 +1057,13 @@ struct SchedulerState { // Destructively claim every published bit. A stager racing this pass // can claim only bits that land after the exchange, so each gated core // has exactly one doorbell writer. - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { uint64_t owned = claim_all_staged_doorbell_bits(slot_state.payload->staged_core_mask[w]); ring_staged_doorbell_bits(w, owned); } wmb(); slot_state.payload->early_dispatch_launch_state.store( - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_seq_cst + EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_seq_cst ); } // This pre-staged consumer was just released by its doorbell — it starts diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index 847e642a09..5bbe448327 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -322,7 +322,7 @@ void SchedulerContext::check_running_cores_for_completion( { ChipTaskSlotState *rs = core.running_slot_state; if (rs != nullptr && rs->payload != nullptr && - rs->payload->early_dispatch_state.load(std::memory_order_relaxed) == PTO2_EARLY_DISPATCH_STAGING) { + rs->payload->early_dispatch_state.load(std::memory_order_relaxed) == EARLY_DISPATCH_STAGING) { continue; } } @@ -366,12 +366,11 @@ void SchedulerContext::check_running_cores_for_completion( uint8_t pending_ss = (core.pending_slot_state != nullptr && core.pending_slot_state->payload != nullptr) ? core.pending_slot_state->payload->early_dispatch_state.load(std::memory_order_relaxed) : - static_cast(PTO2_EARLY_DISPATCH_NONE); + static_cast(EARLY_DISPATCH_NONE); bool pending_gated = (core.pending_slot_state != nullptr && core.pending_slot_state->payload != nullptr && - (pending_ss == PTO2_EARLY_DISPATCH_STAGING || - (pending_ss == PTO2_EARLY_DISPATCH_DISPATCHED && - core.pending_slot_state->task_attrs.requires_sync_start()))); + (pending_ss == EARLY_DISPATCH_STAGING || + (pending_ss == EARLY_DISPATCH_DISPATCHED && core.pending_slot_state->task_attrs.requires_sync_start()))); SlotTransition t = decide_slot_transition( reg_task_id, reg_state, core.running_reg_task_id, core.pending_reg_task_id, pending_gated ); @@ -613,7 +612,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( // per-write atomic contends across all drain threads on the same 2 words and was // ~half the publish cost. The doorbell-table writes stay per-core (unique cid, no // contention). - uint64_t my_mask[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; + uint64_t my_mask[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; for (int i = 0; i < handle_count; i++) { publish_subtask_to_core(handles[i], dispatch_ts, thread_idx); if (gated) { @@ -624,7 +623,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( } } if (gated) { - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_mask[w] != 0) { slot_state->payload->staged_core_mask[w].fetch_or(my_mask[w], std::memory_order_seq_cst); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 7672f5cb0b..48f79d934d 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -54,11 +54,10 @@ namespace { inline constexpr int32_t PTO2_DEFERRED_RELEASE_CAP = 256; } -// The early-dispatch core bitmask (PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64 bits) must cover +// The early-dispatch core bitmask (EARLY_DISPATCH_CORE_MASK_WORDS * 64 bits) must cover // every global core_id, and the per-core doorbell table is sized to match. static_assert( - RUNTIME_MAX_WORKER <= PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64, - "staged_core_mask too small for RUNTIME_MAX_WORKER cores" + RUNTIME_MAX_WORKER <= EARLY_DISPATCH_CORE_MASK_WORDS * 64, "staged_core_mask too small for RUNTIME_MAX_WORKER cores" ); const char *SchedulerContext::shape_name(ResourceShape shape) { @@ -628,7 +627,7 @@ int32_t SchedulerContext::stage_consumer_blocks( // Stamp the real pre-stage time (NOT 0) so the swimlane shows these blocks // dispatched during the producer's run, not at trace start. uint64_t early_dispatch_ts = get_sys_cnt_aicpu(); - uint64_t my_cores[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; // cores gated by this staging pass + uint64_t my_cores[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; // cores gated by this staging pass int32_t staged = 0; int32_t block = start; // Mirror the normal flush_publish (scheduler_dispatch.cpp wmb()+publish loop): @@ -665,20 +664,20 @@ int32_t SchedulerContext::stage_consumer_blocks( } // Publish all this thread's gated cores into the shared mask in one OR per word // (vs one per subtask) so release sees them; seq_cst keeps the self-ring order. - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) if (my_cores[w] != 0) c->payload->staged_core_mask[w].fetch_or(my_cores[w], std::memory_order_seq_cst); // Full publication and release are independent events. The seq_cst // state/launch/count operations form a two-sided handshake. A released // block must ring before contributing to the publication count. - bool released = staged > 0 && - c->payload->early_dispatch_state.load(std::memory_order_seq_cst) == PTO2_EARLY_DISPATCH_DISPATCHED; + bool released = + staged > 0 && c->payload->early_dispatch_state.load(std::memory_order_seq_cst) == EARLY_DISPATCH_DISPATCHED; // Claim only bits the release path did not take. Local handles remain valid // even if the shared per-core table is reused before this thread resumes. if (released) { - uint64_t owned[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + uint64_t owned[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_cores[w] != 0) { owned[w] = SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); @@ -738,7 +737,7 @@ SchedulerContext::early_dispatch_shape(int32_t thread_idx, ResourceShape shape, for (int bi = 0; bi < got; bi++) { ChipTaskSlotState *c = batch[bi]; if (static_cast(c->task->task_id.raw) != task_id_snapshots[bi]) continue; - if (c->payload->early_dispatch_state.load(std::memory_order_acquire) != PTO2_EARLY_DISPATCH_STAGING) + if (c->payload->early_dispatch_state.load(std::memory_order_acquire) != EARLY_DISPATCH_STAGING) continue; // released // The single free-core bucket for this phase. For MIX, an active-mask-aware @@ -833,7 +832,7 @@ int32_t SchedulerContext::try_early_dispatch( bool current_sync_task = static_cast(c->task->task_id.raw) == sync_task_id_snapshot && c->task_attrs.requires_sync_start(); if (current_sync_task && SchedulerState::try_claim_early_sync_drain(*c->payload)) { - if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_STAGING) { + if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != EARLY_DISPATCH_STAGING) { sched_->cancel_early_sync_drain(*c); } else if (drain_state_.sync_start_pending.load(std::memory_order_acquire) == 0 && tracker.count_available_blocks( diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 5e3c569b4f..f35f9588ad 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -143,9 +143,9 @@ SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capaci } layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_EARLY_DISPATCH_QUEUE_SIZE); } - layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, CHIP_EARLY_DISPATCH_QUEUE_SIZE); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { // Force a cache-line base so Orch-side dep_pool writes do not invalidate // adjacent multi-threaded regions like ready_queue.slots. @@ -192,14 +192,14 @@ bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, Device for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { if (!ready_queue_init_data_from_layout( &sched->early_dispatch_queues[i], arena, layout.off_early_dispatch_queue_slots[i], - PTO2_EARLY_DISPATCH_QUEUE_SIZE + CHIP_EARLY_DISPATCH_QUEUE_SIZE )) { return false; } } if (!ready_queue_init_data_from_layout( &sched->early_sync_start_queue, arena, layout.off_early_sync_start_queue_slots, - PTO2_EARLY_DISPATCH_QUEUE_SIZE + CHIP_EARLY_DISPATCH_QUEUE_SIZE )) { return false; } diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 90983af1b4..ea9fefd4e1 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1739,15 +1739,15 @@ void graph_reset_outer_payload(TaskPayload &payload) { payload.scalar_count = 0; payload.fanin_count = 0; payload.predicate = DispatchPredicate{}; - payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_NONE, std::memory_order_relaxed); + payload.early_dispatch_state.store(EARLY_DISPATCH_NONE, std::memory_order_relaxed); for (auto &word : payload.staged_core_mask) word.store(0, std::memory_order_relaxed); payload.dispatch_fanin.store(0, std::memory_order_relaxed); payload.dispatch_propagated.store(0, std::memory_order_relaxed); payload.published_block_count.store(0, std::memory_order_relaxed); - payload.early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); + payload.early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); payload.running_slot_count.store(0, std::memory_order_relaxed); - payload.early_sync_drain_state.store(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); + payload.early_sync_drain_state.store(EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); } bool graph_submit_outer( diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 137dec2672..57aadc907d 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -129,7 +129,7 @@ inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HE #define CHIP_READY_QUEUE_SIZE 8192 // Cross-thread early-dispatch work queue (power of two) -#define PTO2_EARLY_DISPATCH_QUEUE_SIZE 64 +#define CHIP_EARLY_DISPATCH_QUEUE_SIZE 64 // Fanin storage #define CHIP_FANIN_INLINE_CAP 64 @@ -262,25 +262,25 @@ static_assert(offsetof(TaskDescriptor, packed_buffer_base) == 24, "packed_buffer * fanins spill into a per-ring ring buffer slice. */ // Early-dispatch claim states for TaskPayload::early_dispatch_state. -enum PTO2EarlyDispatchState : uint8_t { - PTO2_EARLY_DISPATCH_NONE = 0, // not pre-staged - PTO2_EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress - PTO2_EARLY_DISPATCH_STAGED = 2, // reserved - PTO2_EARLY_DISPATCH_DISPATCHED = 3 // producers released; staged blocks may still be gated +enum EarlyDispatchState : uint8_t { + EARLY_DISPATCH_NONE = 0, // not pre-staged + EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress + EARLY_DISPATCH_STAGED = 2, // reserved + EARLY_DISPATCH_DISPATCHED = 3 // producers released; staged blocks may still be gated }; -enum PTO2EarlyDispatchLaunchState : uint8_t { - PTO2_EARLY_DISPATCH_LAUNCH_NONE = 0, - PTO2_EARLY_DISPATCH_LAUNCH_RINGING = 1, - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE = 2, +enum EarlyDispatchLaunchState : uint8_t { + EARLY_DISPATCH_LAUNCH_NONE = 0, + EARLY_DISPATCH_LAUNCH_RINGING = 1, + EARLY_DISPATCH_LAUNCH_COMPLETE = 2, }; -enum PTO2EarlySyncDrainState : uint8_t { - PTO2_EARLY_SYNC_DRAIN_NONE = 0, - PTO2_EARLY_SYNC_DRAIN_OWNER = 1 << 0, - PTO2_EARLY_SYNC_DRAIN_ARMED = 1 << 1, - PTO2_EARLY_SYNC_DRAIN_READY = 1 << 2, - PTO2_EARLY_SYNC_DRAIN_COMPLETE = 1 << 3, +enum EarlySyncDrainState : uint8_t { + EARLY_SYNC_DRAIN_NONE = 0, + EARLY_SYNC_DRAIN_OWNER = 1 << 0, + EARLY_SYNC_DRAIN_ARMED = 1 << 1, + EARLY_SYNC_DRAIN_READY = 1 << 2, + EARLY_SYNC_DRAIN_COMPLETE = 1 << 3, }; // A pre-staged consumer occupies one core per gated subtask block. WHICH cores @@ -290,7 +290,7 @@ enum PTO2EarlySyncDrainState : uint8_t { // chip's core count (RUNTIME_MAX_WORKER = 72; no two-level pre-dispatch means // gated cores in flight <= core count), NOT by block_num — so a wide SPMD // consumer can pre-stage all its idle cores. 2 words = 128 bits >= 72. -inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; +inline constexpr int EARLY_DISPATCH_CORE_MASK_WORDS = 2; struct TaskPayload { // === Cache line 0 (64B) — the dispatch path's own line === @@ -328,7 +328,7 @@ struct TaskPayload { // splits them between release and late-stager owners; a sync_start cohort keeps // the completed mask stable for its single launch owner, whether staging is local // or uses the global drain fallback. - alignas(64) std::atomic staged_core_mask[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS]{}; + alignas(64) std::atomic staged_core_mask[EARLY_DISPATCH_CORE_MASK_WORDS]{}; // Early-dispatch CANDIDATE detection (event-driven, dual of fanin_refcount): // seeded at wiring with producers already complete, then a flagged producer // bumps each consumer after all of its logical blocks are published. @@ -353,7 +353,7 @@ struct TaskPayload { std::atomic dispatch_propagated{0}; // PRODUCER side: once-guard for fanout propagation // The launch owner publishes COMPLETE only after all owned doorbells are // visible, keeping fanout private until every gated block has launched. - std::atomic early_dispatch_launch_state{PTO2_EARLY_DISPATCH_LAUNCH_NONE}; + std::atomic early_dispatch_launch_state{EARLY_DISPATCH_LAUNCH_NONE}; // sync_start early-dispatch rendezvous: count of this task's gated CORES currently // occupying a RUNNING slot (staged directly to an idle core, or promoted from a // gated pending slot). Counted per-core (not per-block) so it is shape-agnostic: a @@ -365,7 +365,7 @@ struct TaskPayload { // A successful OWNER persists through ARMED and COMPLETE until payload // reinitialization. READY records that producer release observed OWNER; // only cancellation clears OWNER during the current task lifetime. - std::atomic early_sync_drain_state{PTO2_EARLY_SYNC_DRAIN_NONE}; + std::atomic early_sync_drain_state{EARLY_SYNC_DRAIN_NONE}; // === Cache line 2 — dispatch predicate + dump metadata (AICPU-only) === // AICore never reads either — args are materialized from line 0's counts and // region deltas. Resolved at submit; evaluated by the scheduler at dispatch. @@ -487,15 +487,15 @@ struct TaskPayload { // consumer's own hint). So they MUST be zeroed here unconditionally. // Publication, propagation, and launch fields share this same // per-submit lifetime and are reset here too. - early_dispatch_state.store(PTO2_EARLY_DISPATCH_NONE, std::memory_order_relaxed); - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + early_dispatch_state.store(EARLY_DISPATCH_NONE, std::memory_order_relaxed); + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) staged_core_mask[w].store(0, std::memory_order_relaxed); dispatch_fanin.store(0, std::memory_order_relaxed); dispatch_propagated.store(0, std::memory_order_relaxed); published_block_count.store(0, std::memory_order_relaxed); - early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); + early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); running_slot_count.store(0, std::memory_order_relaxed); - early_sync_drain_state.store(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); + early_sync_drain_state.store(EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); } }; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h index 26569d83a7..6f4e50fdb5 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler.h @@ -687,7 +687,7 @@ struct SchedulerState { uint64_t addr{0}; uint32_t token{0}; }; - EarlyDispatchDoorbell early_dispatch_doorbell_table[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; + EarlyDispatchDoorbell early_dispatch_doorbell_table[EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; // Cross-thread early-dispatch work queues, one ChipReadyQueue MPMC instance per // resource shape (AIC/AIV/MIX) — arena-backed, reserved/wired in pto_runtime2_init @@ -747,7 +747,7 @@ struct SchedulerState { } static inline bool should_gate_early_dispatch(bool force_gate, uint8_t early_dispatch_state) { - return force_gate || early_dispatch_state == PTO2_EARLY_DISPATCH_STAGING; + return force_gate || early_dispatch_state == EARLY_DISPATCH_STAGING; } static inline bool @@ -758,9 +758,9 @@ struct SchedulerState { } static inline bool try_claim_early_dispatch_launch(TaskPayload &payload) { - uint8_t expected = PTO2_EARLY_DISPATCH_LAUNCH_NONE; + uint8_t expected = EARLY_DISPATCH_LAUNCH_NONE; return payload.early_dispatch_launch_state.compare_exchange_strong( - expected, PTO2_EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst ); } @@ -773,7 +773,7 @@ struct SchedulerState { // the NONE->RINGING launch latch and invokes this exactly once after local or // global staging completes, while the corresponding per-core table entries are live. inline void ring_all_staged_doorbells(ChipTaskSlotState &slot_state) { - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { uint64_t bits = slot_state.payload->staged_core_mask[w].load(std::memory_order_seq_cst); while (bits != 0) { int core_id = w * 64 + __builtin_ctzll(bits); @@ -786,43 +786,42 @@ struct SchedulerState { } static inline bool try_claim_early_sync_drain(TaskPayload &payload) { - uint8_t expected = PTO2_EARLY_SYNC_DRAIN_NONE; + uint8_t expected = EARLY_SYNC_DRAIN_NONE; return payload.early_sync_drain_state.compare_exchange_strong( - expected, PTO2_EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst ); } static inline bool owns_early_sync_drain(const TaskPayload &payload) { - return (payload.early_sync_drain_state.load(std::memory_order_acquire) & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; + return (payload.early_sync_drain_state.load(std::memory_order_acquire) & EARLY_SYNC_DRAIN_OWNER) != 0; } static inline void mark_early_sync_drain_armed(TaskPayload &payload) { - payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); + payload.early_sync_drain_state.fetch_or(EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); } static inline bool publish_ready_to_early_sync_drain(TaskPayload &payload) { - uint8_t previous = - payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); - return (previous & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; + uint8_t previous = payload.early_sync_drain_state.fetch_or(EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); + return (previous & EARLY_SYNC_DRAIN_OWNER) != 0; } inline void cancel_early_sync_drain(ChipTaskSlotState &slot_state) { uint8_t previous = - slot_state.payload->early_sync_drain_state.exchange(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_seq_cst); - if ((previous & PTO2_EARLY_SYNC_DRAIN_OWNER) == 0) return; - if ((previous & PTO2_EARLY_SYNC_DRAIN_READY) != 0) { + slot_state.payload->early_sync_drain_state.exchange(EARLY_SYNC_DRAIN_NONE, std::memory_order_seq_cst); + if ((previous & EARLY_SYNC_DRAIN_OWNER) == 0) return; + if ((previous & EARLY_SYNC_DRAIN_READY) != 0) { push_ready_routed(&slot_state); return; } - if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) == PTO2_EARLY_DISPATCH_STAGING) { + if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) == EARLY_DISPATCH_STAGING) { early_sync_start_queue.push_tagged(&slot_state, static_cast(slot_state.task->task_id.raw)); } } static inline void finish_early_sync_drain(TaskPayload &payload) { uint8_t state = payload.early_sync_drain_state.load(std::memory_order_seq_cst); - while ((state & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0 && (state & PTO2_EARLY_SYNC_DRAIN_COMPLETE) == 0) { - uint8_t desired = state | PTO2_EARLY_SYNC_DRAIN_COMPLETE; + while ((state & EARLY_SYNC_DRAIN_OWNER) != 0 && (state & EARLY_SYNC_DRAIN_COMPLETE) == 0) { + uint8_t desired = state | EARLY_SYNC_DRAIN_COMPLETE; if (payload.early_sync_drain_state.compare_exchange_weak( state, desired, std::memory_order_seq_cst, std::memory_order_seq_cst )) { @@ -846,18 +845,18 @@ struct SchedulerState { // mask when producer release races staging. int32_t running_cores = slot_state.payload->running_slot_count.load(std::memory_order_seq_cst); int32_t staged_cores = 0; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) staged_cores += __builtin_popcountll(slot_state.payload->staged_core_mask[w].load(std::memory_order_seq_cst)); if (staged_cores == 0) return false; if (running_cores != staged_cores) return false; - if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_DISPATCHED) + if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) != EARLY_DISPATCH_DISPATCHED) return false; if (!try_claim_early_dispatch_launch(*slot_state.payload)) return false; ring_all_staged_doorbells(slot_state); wmb(); slot_state.payload->early_dispatch_launch_state.store( - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_release + EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_release ); return true; } diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index a1ddd50333..7612353f24 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -279,7 +279,7 @@ void SchedulerContext::check_running_cores_for_completion( { ChipTaskSlotState *rs = core.running_slot_state; if (rs != nullptr && rs->payload != nullptr && - rs->payload->early_dispatch_state.load(std::memory_order_relaxed) == PTO2_EARLY_DISPATCH_STAGING) { + rs->payload->early_dispatch_state.load(std::memory_order_relaxed) == EARLY_DISPATCH_STAGING) { continue; } } @@ -319,12 +319,11 @@ void SchedulerContext::check_running_cores_for_completion( uint8_t pending_ss = (core.pending_slot_state != nullptr && core.pending_slot_state->payload != nullptr) ? core.pending_slot_state->payload->early_dispatch_state.load(std::memory_order_relaxed) : - static_cast(PTO2_EARLY_DISPATCH_NONE); + static_cast(EARLY_DISPATCH_NONE); bool pending_gated = (core.pending_slot_state != nullptr && core.pending_slot_state->payload != nullptr && - (pending_ss == PTO2_EARLY_DISPATCH_STAGING || - (pending_ss == PTO2_EARLY_DISPATCH_DISPATCHED && - core.pending_slot_state->task_attrs.requires_sync_start()))); + (pending_ss == EARLY_DISPATCH_STAGING || + (pending_ss == EARLY_DISPATCH_DISPATCHED && core.pending_slot_state->task_attrs.requires_sync_start()))); SlotTransition t = decide_slot_transition( reg_task_id, reg_state, core.running_reg_task_id, core.pending_reg_task_id, pending_gated ); @@ -557,7 +556,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( // per-write atomic contends across all drain threads on the same 2 words and was // ~half the publish cost. The doorbell-table writes stay per-core (unique cid, no // contention). - uint64_t my_mask[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; + uint64_t my_mask[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; for (int i = 0; i < handle_count; i++) { publish_subtask_to_core(handles[i], dispatch_ts, thread_idx); if (gated) { @@ -568,7 +567,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( } } if (gated) { - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_mask[w] != 0) { slot_state->payload->staged_core_mask[w].fetch_or(my_mask[w], std::memory_order_seq_cst); } diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 7d2b580660..d0d0db44f5 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -52,11 +52,10 @@ static_assert(sizeof(ChipTensor) == TASKPAYLOAD_TENSOR_STRIDE); namespace {} -// The early-dispatch core bitmask (PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64 bits) must cover +// The early-dispatch core bitmask (EARLY_DISPATCH_CORE_MASK_WORDS * 64 bits) must cover // every global core_id, and the per-core doorbell table is sized to match. static_assert( - RUNTIME_MAX_WORKER <= PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64, - "staged_core_mask too small for RUNTIME_MAX_WORKER cores" + RUNTIME_MAX_WORKER <= EARLY_DISPATCH_CORE_MASK_WORDS * 64, "staged_core_mask too small for RUNTIME_MAX_WORKER cores" ); const char *SchedulerContext::shape_name(ResourceShape shape) { @@ -627,7 +626,7 @@ int32_t SchedulerContext::stage_consumer_blocks( // Stamp the real pre-stage time (NOT 0) so the swimlane shows these blocks // dispatched during the producer's run, not at trace start. uint64_t early_dispatch_ts = get_sys_cnt_aicpu(); - uint64_t my_cores[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; // cores gated by this staging pass + uint64_t my_cores[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; // cores gated by this staging pass int32_t staged = 0; int32_t block = start; // Mirror the normal flush_publish (scheduler_dispatch.cpp wmb()+publish loop): @@ -664,20 +663,20 @@ int32_t SchedulerContext::stage_consumer_blocks( } // Publish all this thread's gated cores into the shared mask in one OR per word // (vs one per subtask) so release sees them; seq_cst keeps the self-ring order. - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) if (my_cores[w] != 0) c->payload->staged_core_mask[w].fetch_or(my_cores[w], std::memory_order_seq_cst); // Full publication and release are independent events. The seq_cst // state/launch/count operations form a two-sided handshake. A released // block must ring before contributing to the publication count. - bool released = staged > 0 && - c->payload->early_dispatch_state.load(std::memory_order_seq_cst) == PTO2_EARLY_DISPATCH_DISPATCHED; + bool released = + staged > 0 && c->payload->early_dispatch_state.load(std::memory_order_seq_cst) == EARLY_DISPATCH_DISPATCHED; // Claim only bits the release path did not take. Local handles remain valid // even if the shared per-core table is reused before this thread resumes. if (released) { - uint64_t owned[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + uint64_t owned[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_cores[w] != 0) { owned[w] = SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); @@ -737,7 +736,7 @@ SchedulerContext::early_dispatch_shape(int32_t thread_idx, ResourceShape shape, for (int bi = 0; bi < got; bi++) { ChipTaskSlotState *c = batch[bi]; if (static_cast(c->task->task_id.raw) != task_id_snapshots[bi]) continue; - if (c->payload->early_dispatch_state.load(std::memory_order_acquire) != PTO2_EARLY_DISPATCH_STAGING) + if (c->payload->early_dispatch_state.load(std::memory_order_acquire) != EARLY_DISPATCH_STAGING) continue; // released // The single free-core bucket for this phase. For MIX, an active-mask-aware @@ -826,7 +825,7 @@ int32_t SchedulerContext::try_early_dispatch( bool current_sync_task = static_cast(c->task->task_id.raw) == sync_task_id_snapshot && c->task_attrs.requires_sync_start(); if (current_sync_task && SchedulerState::try_claim_early_sync_drain(*c->payload)) { - if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_STAGING) { + if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != EARLY_DISPATCH_STAGING) { sched_->cancel_early_sync_drain(*c); } else if (drain_state_.sync_start_pending.load(std::memory_order_acquire) == 0 && tracker.count_available_blocks( diff --git a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp index cc46987eb1..0d6ce3571a 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -86,9 +86,9 @@ SchedulerLayout SchedulerState::reserve_layout(DeviceArena &arena) { // ones. The big nine are the arena's last reservations so that the bytes bind // uploads stay one contiguous range no matter how much of them is in use. for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_EARLY_DISPATCH_QUEUE_SIZE); } - layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, CHIP_EARLY_DISPATCH_QUEUE_SIZE); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { layout.off_ready_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); } @@ -125,9 +125,9 @@ bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, Device ready_queue_init_data_from_layout(&sched->graph_ready_queue, layout.ready_queue_capacity); ready_queue_init_data_from_layout(&sched->graph_prepare_queue, layout.ready_queue_capacity); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - ready_queue_init_data_from_layout(&sched->early_dispatch_queues[i], PTO2_EARLY_DISPATCH_QUEUE_SIZE); + ready_queue_init_data_from_layout(&sched->early_dispatch_queues[i], CHIP_EARLY_DISPATCH_QUEUE_SIZE); } - ready_queue_init_data_from_layout(&sched->early_sync_start_queue, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + ready_queue_init_data_from_layout(&sched->early_sync_start_queue, CHIP_EARLY_DISPATCH_QUEUE_SIZE); // Polling: no dep_pool arena region to initialize. (void)arena; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 43c81f085e..a61f201358 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -91,7 +91,7 @@ // capacity tracks the publication burst rather than the simultaneously // stageable cohort: a 128-token EP8 routed-expert layer bursts 128 gate/up MM // candidates onto one shape, and this holds 2x that. -#define PTO2_EARLY_DISPATCH_QUEUE_SIZE 256 +#define CHIP_EARLY_DISPATCH_QUEUE_SIZE 256 // Fanin storage #define CHIP_FANIN_INLINE_CAP 64 @@ -237,25 +237,25 @@ static_assert(offsetof(TaskDescriptor, packed_buffer_base) == 24, "packed_buffer // ============================================================================= // Early-dispatch claim states for TaskPayload::early_dispatch_state. -enum PTO2EarlyDispatchState : uint8_t { - PTO2_EARLY_DISPATCH_NONE = 0, // not pre-staged - PTO2_EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress - PTO2_EARLY_DISPATCH_STAGED = 2, // reserved - PTO2_EARLY_DISPATCH_DISPATCHED = 3 // producers released; staged blocks may still be gated +enum EarlyDispatchState : uint8_t { + EARLY_DISPATCH_NONE = 0, // not pre-staged + EARLY_DISPATCH_STAGING = 1, // Hook 1 claimed it; staging in progress + EARLY_DISPATCH_STAGED = 2, // reserved + EARLY_DISPATCH_DISPATCHED = 3 // producers released; staged blocks may still be gated }; -enum PTO2EarlyDispatchLaunchState : uint8_t { - PTO2_EARLY_DISPATCH_LAUNCH_NONE = 0, - PTO2_EARLY_DISPATCH_LAUNCH_RINGING = 1, - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE = 2, +enum EarlyDispatchLaunchState : uint8_t { + EARLY_DISPATCH_LAUNCH_NONE = 0, + EARLY_DISPATCH_LAUNCH_RINGING = 1, + EARLY_DISPATCH_LAUNCH_COMPLETE = 2, }; -enum PTO2EarlySyncDrainState : uint8_t { - PTO2_EARLY_SYNC_DRAIN_NONE = 0, - PTO2_EARLY_SYNC_DRAIN_OWNER = 1 << 0, - PTO2_EARLY_SYNC_DRAIN_ARMED = 1 << 1, - PTO2_EARLY_SYNC_DRAIN_READY = 1 << 2, - PTO2_EARLY_SYNC_DRAIN_COMPLETE = 1 << 3, +enum EarlySyncDrainState : uint8_t { + EARLY_SYNC_DRAIN_NONE = 0, + EARLY_SYNC_DRAIN_OWNER = 1 << 0, + EARLY_SYNC_DRAIN_ARMED = 1 << 1, + EARLY_SYNC_DRAIN_READY = 1 << 2, + EARLY_SYNC_DRAIN_COMPLETE = 1 << 3, }; // A pre-staged consumer occupies one core per gated subtask block. WHICH cores @@ -265,7 +265,7 @@ enum PTO2EarlySyncDrainState : uint8_t { // chip's core count (RUNTIME_MAX_WORKER = 108 on a5; no two-level pre-dispatch means // gated cores in flight <= core count), NOT by block_num — so a wide SPMD // consumer can pre-stage all its idle cores. 2 words = 128 bits >= 108. -inline constexpr int PTO2_EARLY_DISPATCH_CORE_MASK_WORDS = 2; +inline constexpr int EARLY_DISPATCH_CORE_MASK_WORDS = 2; /** * Task payload data (cold path - only accessed during orchestration and dispatch) @@ -295,7 +295,7 @@ struct TaskPayload { // consumer destructively splits those bits between release and late-stager // owners; a sync_start cohort keeps the completed mask stable for one launch // owner across both local staging and the global-drain fallback. - std::atomic staged_core_mask[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS]{}; + std::atomic staged_core_mask[EARLY_DISPATCH_CORE_MASK_WORDS]{}; // Candidate detection is the event-driven dual of fanin_refcount. Wiring // seeds producers already complete, and flagged producers increment the // count only after all logical blocks are launch-visible. Equality with @@ -312,7 +312,7 @@ struct TaskPayload { std::atomic early_dispatch_state{0}; // COMPLETE is published only after every owned doorbell is visible, keeping // fanout private until all gated blocks have launched. - std::atomic early_dispatch_launch_state{PTO2_EARLY_DISPATCH_LAUNCH_NONE}; + std::atomic early_dispatch_launch_state{EARLY_DISPATCH_LAUNCH_NONE}; // Number of this sync_start task's gated cores occupying RUNNING slots. // Counting cores is shape-agnostic for MIX, whose cluster cores promote // independently. Doorbells launch only after this count matches the staged @@ -321,7 +321,7 @@ struct TaskPayload { // OWNER persists through ARMED and COMPLETE for one task lifetime. READY // records that producer release observed the owner; only cancellation clears // ownership before payload reinitialization. - std::atomic early_sync_drain_state{PTO2_EARLY_SYNC_DRAIN_NONE}; + std::atomic early_sync_drain_state{EARLY_SYNC_DRAIN_NONE}; // === Cache line 9 (byte 576) — dispatch predicate (AICPU-only) === // Offset is a fixed 576, independent of MAX_TENSOR_ARGS / MAX_SCALAR_ARGS. // AICore never reads it — args are materialized from the tensor_count / tensors @@ -401,14 +401,14 @@ struct TaskPayload { // are consumer-side state and must be initialized on every submit even // when the consumer itself does not allow early resolve, because one of // its flagged producers may still update them. - early_dispatch_state.store(PTO2_EARLY_DISPATCH_NONE, std::memory_order_relaxed); - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + early_dispatch_state.store(EARLY_DISPATCH_NONE, std::memory_order_relaxed); + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) staged_core_mask[w].store(0, std::memory_order_relaxed); dispatch_fanin.store(0, std::memory_order_relaxed); published_block_count.store(0, std::memory_order_relaxed); - early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); + early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); running_slot_count.store(0, std::memory_order_relaxed); - early_sync_drain_state.store(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); + early_sync_drain_state.store(EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); } }; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h index 50792d3f1f..eaf36a261b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler.h @@ -765,7 +765,7 @@ struct SchedulerState { uint64_t addr{0}; uint32_t token{0}; }; - EarlyDispatchDoorbell early_dispatch_doorbell_table[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; + EarlyDispatchDoorbell early_dispatch_doorbell_table[EARLY_DISPATCH_CORE_MASK_WORDS * 64]{}; ChipReadyQueue early_dispatch_queues[NUM_RESOURCE_SHAPES]; // Shape-agnostic sync_start candidates require an all-or-nothing owner // decision. The owner stages locally when one tracker can hold the complete @@ -797,7 +797,7 @@ struct SchedulerState { } static inline bool should_gate_early_dispatch(bool force_gate, uint8_t early_dispatch_state) { - return force_gate || early_dispatch_state == PTO2_EARLY_DISPATCH_STAGING; + return force_gate || early_dispatch_state == EARLY_DISPATCH_STAGING; } static inline bool @@ -808,9 +808,9 @@ struct SchedulerState { } static inline bool try_claim_early_dispatch_launch(TaskPayload &payload) { - uint8_t expected = PTO2_EARLY_DISPATCH_LAUNCH_NONE; + uint8_t expected = EARLY_DISPATCH_LAUNCH_NONE; return payload.early_dispatch_launch_state.compare_exchange_strong( - expected, PTO2_EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_DISPATCH_LAUNCH_RINGING, std::memory_order_seq_cst, std::memory_order_seq_cst ); } @@ -820,7 +820,7 @@ struct SchedulerState { } inline void ring_all_staged_doorbells(ChipTaskSlotState &slot_state) { - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { uint64_t bits = slot_state.payload->staged_core_mask[w].load(std::memory_order_seq_cst); while (bits != 0) { int core_id = w * 64 + __builtin_ctzll(bits); @@ -833,43 +833,42 @@ struct SchedulerState { } static inline bool try_claim_early_sync_drain(TaskPayload &payload) { - uint8_t expected = PTO2_EARLY_SYNC_DRAIN_NONE; + uint8_t expected = EARLY_SYNC_DRAIN_NONE; return payload.early_sync_drain_state.compare_exchange_strong( - expected, PTO2_EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_SYNC_DRAIN_OWNER, std::memory_order_seq_cst, std::memory_order_seq_cst ); } static inline bool owns_early_sync_drain(const TaskPayload &payload) { - return (payload.early_sync_drain_state.load(std::memory_order_acquire) & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; + return (payload.early_sync_drain_state.load(std::memory_order_acquire) & EARLY_SYNC_DRAIN_OWNER) != 0; } static inline void mark_early_sync_drain_armed(TaskPayload &payload) { - payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); + payload.early_sync_drain_state.fetch_or(EARLY_SYNC_DRAIN_ARMED, std::memory_order_seq_cst); } static inline bool publish_ready_to_early_sync_drain(TaskPayload &payload) { - uint8_t previous = - payload.early_sync_drain_state.fetch_or(PTO2_EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); - return (previous & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0; + uint8_t previous = payload.early_sync_drain_state.fetch_or(EARLY_SYNC_DRAIN_READY, std::memory_order_seq_cst); + return (previous & EARLY_SYNC_DRAIN_OWNER) != 0; } inline void cancel_early_sync_drain(ChipTaskSlotState &slot_state) { uint8_t previous = - slot_state.payload->early_sync_drain_state.exchange(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_seq_cst); - if ((previous & PTO2_EARLY_SYNC_DRAIN_OWNER) == 0) return; - if ((previous & PTO2_EARLY_SYNC_DRAIN_READY) != 0) { + slot_state.payload->early_sync_drain_state.exchange(EARLY_SYNC_DRAIN_NONE, std::memory_order_seq_cst); + if ((previous & EARLY_SYNC_DRAIN_OWNER) == 0) return; + if ((previous & EARLY_SYNC_DRAIN_READY) != 0) { push_ready_routed(&slot_state); return; } - if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) == PTO2_EARLY_DISPATCH_STAGING) { + if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) == EARLY_DISPATCH_STAGING) { early_sync_start_queue.push_tagged(&slot_state, static_cast(slot_state.task->task_id.raw)); } } static inline void finish_early_sync_drain(TaskPayload &payload) { uint8_t state = payload.early_sync_drain_state.load(std::memory_order_seq_cst); - while ((state & PTO2_EARLY_SYNC_DRAIN_OWNER) != 0 && (state & PTO2_EARLY_SYNC_DRAIN_COMPLETE) == 0) { - uint8_t desired = state | PTO2_EARLY_SYNC_DRAIN_COMPLETE; + while ((state & EARLY_SYNC_DRAIN_OWNER) != 0 && (state & EARLY_SYNC_DRAIN_COMPLETE) == 0) { + uint8_t desired = state | EARLY_SYNC_DRAIN_COMPLETE; if (payload.early_sync_drain_state.compare_exchange_weak( state, desired, std::memory_order_seq_cst, std::memory_order_seq_cst )) { @@ -885,18 +884,18 @@ struct SchedulerState { // could pair a partial mask with the later final seed and ring too early. int32_t running_cores = slot_state.payload->running_slot_count.load(std::memory_order_seq_cst); int32_t staged_cores = 0; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) staged_cores += __builtin_popcountll(slot_state.payload->staged_core_mask[w].load(std::memory_order_seq_cst)); if (staged_cores == 0) return false; if (running_cores != staged_cores) return false; - if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_DISPATCHED) + if (slot_state.payload->early_dispatch_state.load(std::memory_order_seq_cst) != EARLY_DISPATCH_DISPATCHED) return false; if (!try_claim_early_dispatch_launch(*slot_state.payload)) return false; ring_all_staged_doorbells(slot_state); wmb(); slot_state.payload->early_dispatch_launch_state.store( - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_release + EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_release ); return true; } @@ -914,9 +913,9 @@ struct SchedulerState { return; } - uint8_t expected = PTO2_EARLY_DISPATCH_NONE; + uint8_t expected = EARLY_DISPATCH_NONE; if (!consumer.payload->early_dispatch_state.compare_exchange_strong( - expected, PTO2_EARLY_DISPATCH_STAGING, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_DISPATCH_STAGING, std::memory_order_seq_cst, std::memory_order_seq_cst )) { return; } @@ -930,9 +929,9 @@ struct SchedulerState { // drop the speculative claim and let readiness take the ordinary // queue path. A concurrent producer release may already have moved // STAGING to DISPATCHED; that path then owns the transition. - expected = PTO2_EARLY_DISPATCH_STAGING; + expected = EARLY_DISPATCH_STAGING; consumer.payload->early_dispatch_state.compare_exchange_strong( - expected, PTO2_EARLY_DISPATCH_NONE, std::memory_order_seq_cst, std::memory_order_seq_cst + expected, EARLY_DISPATCH_NONE, std::memory_order_seq_cst, std::memory_order_seq_cst ); } } @@ -943,15 +942,15 @@ struct SchedulerState { void propagate_dispatch_fanin(ChipTaskSlotState &p) { if (!p.task_attrs.allow_early_resolve()) return; if (p.payload->published_block_count.load(std::memory_order_seq_cst) < p.logical_block_num) return; - if (p.payload->early_dispatch_state.load(std::memory_order_acquire) == PTO2_EARLY_DISPATCH_STAGING) return; + if (p.payload->early_dispatch_state.load(std::memory_order_acquire) == EARLY_DISPATCH_STAGING) return; uint8_t launch_state = p.payload->early_dispatch_launch_state.load(std::memory_order_acquire); - if (launch_state == PTO2_EARLY_DISPATCH_LAUNCH_RINGING) return; + if (launch_state == EARLY_DISPATCH_LAUNCH_RINGING) return; if (p.task_attrs.requires_sync_start()) { bool was_pre_staged = false; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { was_pre_staged |= p.payload->staged_core_mask[w].load(std::memory_order_acquire) != 0; } - if (was_pre_staged && launch_state != PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE) return; + if (was_pre_staged && launch_state != EARLY_DISPATCH_LAUNCH_COMPLETE) return; } if (p.has_dispatch_propagated()) return; p.lock_fanout(); @@ -983,30 +982,30 @@ struct SchedulerState { inline bool try_early_dispatch_release(ChipTaskSlotState &slot_state, EarlyDispatchReleaseSink *sink = nullptr) { if (slot_state.task_attrs.has_predicate()) return false; - uint8_t expect = PTO2_EARLY_DISPATCH_NONE; + uint8_t expect = EARLY_DISPATCH_NONE; if (slot_state.payload->early_dispatch_state.compare_exchange_strong( - expect, PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst, std::memory_order_seq_cst + expect, EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst, std::memory_order_seq_cst )) { return false; } - if (expect != PTO2_EARLY_DISPATCH_STAGING) return true; + if (expect != EARLY_DISPATCH_STAGING) return true; bool sync_start = slot_state.task_attrs.requires_sync_start(); if (!sync_start && !try_claim_early_dispatch_launch(*slot_state.payload)) return true; - expect = PTO2_EARLY_DISPATCH_STAGING; + expect = EARLY_DISPATCH_STAGING; slot_state.payload->early_dispatch_state.compare_exchange_strong( - expect, PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst, std::memory_order_seq_cst + expect, EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst, std::memory_order_seq_cst ); bool launched = true; if (sync_start) { launched = maybe_rendezvous_ring(slot_state); } else { - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { uint64_t owned = claim_all_staged_doorbell_bits(slot_state.payload->staged_core_mask[w]); ring_staged_doorbell_bits(w, owned); } wmb(); slot_state.payload->early_dispatch_launch_state.store( - PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_seq_cst + EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_seq_cst ); } if (launched) { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index af4c5f3178..2a4965f81c 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -289,7 +289,7 @@ void SchedulerContext::check_running_cores_for_completion( { ChipTaskSlotState *rs = core.running_slot_state; if (rs != nullptr && rs->payload != nullptr && - rs->payload->early_dispatch_state.load(std::memory_order_relaxed) == PTO2_EARLY_DISPATCH_STAGING) { + rs->payload->early_dispatch_state.load(std::memory_order_relaxed) == EARLY_DISPATCH_STAGING) { continue; } } @@ -320,12 +320,11 @@ void SchedulerContext::check_running_cores_for_completion( uint8_t pending_ss = (core.pending_slot_state != nullptr && core.pending_slot_state->payload != nullptr) ? core.pending_slot_state->payload->early_dispatch_state.load(std::memory_order_relaxed) : - static_cast(PTO2_EARLY_DISPATCH_NONE); + static_cast(EARLY_DISPATCH_NONE); bool pending_gated = (core.pending_slot_state != nullptr && core.pending_slot_state->payload != nullptr && - (pending_ss == PTO2_EARLY_DISPATCH_STAGING || - (pending_ss == PTO2_EARLY_DISPATCH_DISPATCHED && - core.pending_slot_state->task_attrs.requires_sync_start()))); + (pending_ss == EARLY_DISPATCH_STAGING || + (pending_ss == EARLY_DISPATCH_DISPATCHED && core.pending_slot_state->task_attrs.requires_sync_start()))); SlotTransition t = decide_slot_transition( reg_task_id, reg_state, core.running_reg_task_id, core.pending_reg_task_id, pending_gated ); @@ -533,7 +532,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( dispatch_ts = pub_t0 != 0 ? pub_t0 : get_sys_cnt_aicpu(); } #endif - uint64_t my_mask[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; + uint64_t my_mask[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; for (int i = 0; i < handle_count; i++) { publish_subtask_to_core(handles[i], dispatch_ts, thread_idx); if (gated) { @@ -544,7 +543,7 @@ SchedulerContext::SyncStartStageResult SchedulerContext::stage_sync_start_cores( } } if (gated) { - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_mask[w] != 0) { slot_state->payload->staged_core_mask[w].fetch_or(my_mask[w], std::memory_order_seq_cst); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 77863b4cd0..4a80e8e97d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -48,7 +48,7 @@ static_assert(offsetof(TaskPayload, scalar_count) == TASKPAYLOAD_SCALAR_COUNT_OF static_assert(offsetof(TaskPayload, tensors) == TASKPAYLOAD_TENSORS_OFFSET); static_assert(offsetof(TaskPayload, scalars) == TASKPAYLOAD_SCALARS_OFFSET); static_assert(sizeof(ChipTensor) == TASKPAYLOAD_TENSOR_STRIDE); -static_assert(RUNTIME_MAX_WORKER <= PTO2_EARLY_DISPATCH_CORE_MASK_WORDS * 64); +static_assert(RUNTIME_MAX_WORKER <= EARLY_DISPATCH_CORE_MASK_WORDS * 64); const char *SchedulerContext::shape_name(ResourceShape shape) { switch (shape) { @@ -584,7 +584,7 @@ int32_t SchedulerContext::stage_consumer_blocks( // Stamp the real pre-stage time (NOT 0) so the swimlane shows these blocks // dispatched during the producer's run, not at trace start. uint64_t early_dispatch_ts = get_sys_cnt_aicpu(); - uint64_t my_cores[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; // cores gated by this staging pass + uint64_t my_cores[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; // cores gated by this staging pass int32_t staged = 0; int32_t block = start; // Mirror the normal flush_publish (scheduler_dispatch.cpp wmb()+publish loop): @@ -621,20 +621,20 @@ int32_t SchedulerContext::stage_consumer_blocks( } // Publish all this thread's gated cores into the shared mask in one OR per word // (vs one per subtask) so release sees them; seq_cst keeps the self-ring order. - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) if (my_cores[w] != 0) c->payload->staged_core_mask[w].fetch_or(my_cores[w], std::memory_order_seq_cst); // Full publication and release are independent events. The seq_cst // state/launch/count operations form a two-sided handshake. A released // block must ring before contributing to the publication count. - bool released = staged > 0 && - c->payload->early_dispatch_state.load(std::memory_order_seq_cst) == PTO2_EARLY_DISPATCH_DISPATCHED; + bool released = + staged > 0 && c->payload->early_dispatch_state.load(std::memory_order_seq_cst) == EARLY_DISPATCH_DISPATCHED; // Claim only bits the release path did not take. Local handles remain valid // even if the shared per-core table is reused before this thread resumes. if (released) { - uint64_t owned[PTO2_EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { + uint64_t owned[EARLY_DISPATCH_CORE_MASK_WORDS] = {0}; + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; w++) { if (my_cores[w] != 0) { owned[w] = SchedulerState::claim_late_staged_doorbell_bits(c->payload->staged_core_mask[w], my_cores[w]); @@ -693,7 +693,7 @@ SchedulerContext::early_dispatch_shape(int32_t thread_idx, ResourceShape shape, for (int bi = 0; bi < got; bi++) { ChipTaskSlotState *c = batch[bi]; if (static_cast(c->task->task_id.raw) != task_id_snapshots[bi]) continue; - if (c->payload->early_dispatch_state.load(std::memory_order_acquire) != PTO2_EARLY_DISPATCH_STAGING) + if (c->payload->early_dispatch_state.load(std::memory_order_acquire) != EARLY_DISPATCH_STAGING) continue; // released // The single free-core bucket for this phase. For MIX, an active-mask-aware @@ -786,7 +786,7 @@ int32_t SchedulerContext::try_early_dispatch( bool current_sync_task = static_cast(c->task->task_id.raw) == sync_task_id_snapshot && c->task_attrs.requires_sync_start(); if (current_sync_task && SchedulerState::try_claim_early_sync_drain(*c->payload)) { - if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_STAGING) { + if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != EARLY_DISPATCH_STAGING) { sched_->cancel_early_sync_drain(*c); } else if (drain_state_.sync_start_pending.load(std::memory_order_acquire) == 0 && tracker.count_available_blocks( diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index a909a17a6b..0ebb948cfe 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -150,9 +150,9 @@ SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capaci } layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, CHIP_READY_QUEUE_SIZE); for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { - layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + layout.off_early_dispatch_queue_slots[i] = ready_queue_reserve_layout(arena, CHIP_EARLY_DISPATCH_QUEUE_SIZE); } - layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); + layout.off_early_sync_start_queue_slots = ready_queue_reserve_layout(arena, CHIP_EARLY_DISPATCH_QUEUE_SIZE); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { // Force a cache-line base so Orch-side dep_pool writes do not invalidate // adjacent multi-threaded regions like ready_queue.slots. @@ -201,14 +201,14 @@ bool SchedulerState::init_data_from_layout(const SchedulerLayout &layout, Device for (int i = 0; i < NUM_RESOURCE_SHAPES; i++) { if (!ready_queue_init_data_from_layout( &sched->early_dispatch_queues[i], arena, layout.off_early_dispatch_queue_slots[i], - PTO2_EARLY_DISPATCH_QUEUE_SIZE + CHIP_EARLY_DISPATCH_QUEUE_SIZE )) { return false; } } if (!ready_queue_init_data_from_layout( &sched->early_sync_start_queue, arena, layout.off_early_sync_start_queue_slots, - PTO2_EARLY_DISPATCH_QUEUE_SIZE + CHIP_EARLY_DISPATCH_QUEUE_SIZE )) { return false; } diff --git a/src/common/host_build_graph/graph_execution.cpp b/src/common/host_build_graph/graph_execution.cpp index e0bd7a0731..6a68011d9a 100644 --- a/src/common/host_build_graph/graph_execution.cpp +++ b/src/common/host_build_graph/graph_execution.cpp @@ -45,16 +45,16 @@ GraphExecution *acquire_execution_storage( void reset_graph_payload(TaskPayload &payload) { payload.fanin_count = 0; payload.predicate = DispatchPredicate{}; - payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_NONE, std::memory_order_relaxed); - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; ++w) { + payload.early_dispatch_state.store(EARLY_DISPATCH_NONE, std::memory_order_relaxed); + for (int w = 0; w < EARLY_DISPATCH_CORE_MASK_WORDS; ++w) { payload.staged_core_mask[w].store(0, std::memory_order_relaxed); } payload.dispatch_fanin.store(0, std::memory_order_relaxed); payload.dispatch_propagated.store(0, std::memory_order_relaxed); payload.published_block_count.store(0, std::memory_order_relaxed); - payload.early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); + payload.early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); payload.running_slot_count.store(0, std::memory_order_relaxed); - payload.early_sync_drain_state.store(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); + payload.early_sync_drain_state.store(EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); } bool bind_graph_topology(GraphExecution &execution) { diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index 31926a301e..5f7ac38254 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -454,7 +454,7 @@ TEST_F(WiringTest, EarlyDispatchQueueOverflowRollsBackStagingClaim) { sched.try_enqueue_early_dispatch_candidate(consumer); - EXPECT_EQ(consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_NONE); + EXPECT_EQ(consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_NONE); EXPECT_EQ(queue.size(), queue.capacity); } @@ -470,12 +470,12 @@ TEST_F(WiringTest, EarlyDispatchQueueOverflowFallsBackToNormalDispatch) { } sched.try_enqueue_early_dispatch_candidate(consumer); - ASSERT_EQ(consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_NONE); + ASSERT_EQ(consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_NONE); // The overflowed candidate carries no early-dispatch claim, so the producer // release routes every block through the ordinary ready queue. EXPECT_TRUE(sched.route_ready_once(consumer)); - EXPECT_EQ(consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); + EXPECT_EQ(consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_DISPATCHED); EXPECT_EQ(sched.ready_queues[static_cast(shape)].pop(), &consumer); } @@ -492,7 +492,7 @@ TEST_F(WiringTest, EarlyDispatchSyncStartQueueOverflowFallsBackToSyncReadyQueue) } sched.try_enqueue_early_dispatch_candidate(consumer); - EXPECT_EQ(consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_NONE); + EXPECT_EQ(consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_NONE); EXPECT_EQ(queue.size(), queue.capacity); // A sync_start cohort that never reached its drain falls back to the @@ -523,7 +523,7 @@ TEST_F(WiringTest, LateWiredFullyPublishedProducerStillSeedsEarlyDispatch) { wire_fanin(consumer, 1); EXPECT_EQ(consumer_payload.dispatch_fanin.load(), consumer_payload.fanin_actual_count); - EXPECT_EQ(consumer_payload.early_dispatch_state.load(), PTO2_EARLY_DISPATCH_STAGING); + EXPECT_EQ(consumer_payload.early_dispatch_state.load(), EARLY_DISPATCH_STAGING); auto shape = static_cast(consumer.active_mask.to_shape()); EXPECT_EQ(sched.early_dispatch_queues[shape].pop(), &consumer); EXPECT_NE(producer.fanout_head, nullptr); @@ -568,7 +568,7 @@ TEST_F(WiringTest, WiringSeedEnqueuesAfterConcurrentPropagation) { wiring.join(); EXPECT_EQ(consumer_payload.dispatch_fanin.load(), consumer_payload.fanin_actual_count); - EXPECT_EQ(consumer_payload.early_dispatch_state.load(), PTO2_EARLY_DISPATCH_STAGING); + EXPECT_EQ(consumer_payload.early_dispatch_state.load(), EARLY_DISPATCH_STAGING); auto shape = static_cast(consumer.active_mask.to_shape()); EXPECT_EQ(sched.early_dispatch_queues[shape].pop(), &consumer); } @@ -613,11 +613,11 @@ TEST_F(WiringTest, PartialStagedReleaseRoutesRemainderToReadyQueue) { init_slot(consumer, CHIP_TASK_PENDING, 1, 1); consumer.logical_block_num = 5; consumer.next_block_idx.store(2, std::memory_order_relaxed); - consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); consumer.payload->staged_core_mask[0].store(1, std::memory_order_relaxed); EXPECT_TRUE(sched.route_ready_once(consumer)); - EXPECT_EQ(consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); + EXPECT_EQ(consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_DISPATCHED); EXPECT_EQ(consumer.next_block_idx.load(), 2); ResourceShape shape = consumer.active_mask.to_shape(); @@ -657,9 +657,9 @@ TEST_F(WiringTest, EarlyDispatchDoorbellBitsHaveOneOwner) { } TEST_F(WiringTest, EarlyDispatchClaimStaysGatedAfterRelease) { - EXPECT_TRUE(SchedulerState::should_gate_early_dispatch(true, PTO2_EARLY_DISPATCH_DISPATCHED)); - EXPECT_TRUE(SchedulerState::should_gate_early_dispatch(false, PTO2_EARLY_DISPATCH_STAGING)); - EXPECT_FALSE(SchedulerState::should_gate_early_dispatch(false, PTO2_EARLY_DISPATCH_DISPATCHED)); + EXPECT_TRUE(SchedulerState::should_gate_early_dispatch(true, EARLY_DISPATCH_DISPATCHED)); + EXPECT_TRUE(SchedulerState::should_gate_early_dispatch(false, EARLY_DISPATCH_STAGING)); + EXPECT_FALSE(SchedulerState::should_gate_early_dispatch(false, EARLY_DISPATCH_DISPATCHED)); } TEST_F(WiringTest, EarlyDispatchLaunchHasSingleOwner) { @@ -679,7 +679,7 @@ TEST_F(WiringTest, EarlyDispatchLaunchHasSingleOwner) { contender.join(); EXPECT_NE(won[0], won[1]); - EXPECT_EQ(payload.early_dispatch_launch_state.load(), PTO2_EARLY_DISPATCH_LAUNCH_RINGING); + EXPECT_EQ(payload.early_dispatch_launch_state.load(), EARLY_DISPATCH_LAUNCH_RINGING); } TEST_F(WiringTest, EarlyDispatchFanoutWaitsForDoorbellPass) { @@ -695,18 +695,18 @@ TEST_F(WiringTest, EarlyDispatchFanoutWaitsForDoorbellPass) { dep.slot_state = &consumer; producer.fanout_head = &dep; - producer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + producer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); sched.propagate_dispatch_fanin(producer); EXPECT_EQ(consumer.payload->dispatch_fanin.load(), 0); EXPECT_FALSE(producer.has_dispatch_propagated()); - producer.payload->early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_RINGING); - producer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_release); + producer.payload->early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_RINGING); + producer.payload->early_dispatch_state.store(EARLY_DISPATCH_DISPATCHED, std::memory_order_release); sched.propagate_dispatch_fanin(producer); EXPECT_EQ(consumer.payload->dispatch_fanin.load(), 0); EXPECT_FALSE(producer.has_dispatch_propagated()); - producer.payload->early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_seq_cst); + producer.payload->early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_COMPLETE, std::memory_order_seq_cst); sched.propagate_dispatch_fanin(producer); EXPECT_EQ(consumer.payload->dispatch_fanin.load(), 1); EXPECT_TRUE(producer.has_dispatch_propagated()); @@ -737,18 +737,18 @@ TEST_F(WiringTest, EarlyDispatchReleaseConsumesDoorbellMask) { init_slot(task, CHIP_TASK_PENDING, 1, 1); task.task_attrs.set_early_resolve(true); task.next_block_idx.store(1, std::memory_order_relaxed); - task.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + task.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); task.payload->staged_core_mask[0].store(1ULL << core_id, std::memory_order_relaxed); sched.early_dispatch_doorbell_table[core_id].addr = reg_addr; sched.early_dispatch_doorbell_table[core_id].token = token; bool released_before = - task.payload->early_dispatch_state.load(std::memory_order_seq_cst) == PTO2_EARLY_DISPATCH_DISPATCHED; + task.payload->early_dispatch_state.load(std::memory_order_seq_cst) == EARLY_DISPATCH_DISPATCHED; reset_test_reg_stub(); EXPECT_FALSE(released_before); EXPECT_TRUE(sched.try_early_dispatch_release(task)); - EXPECT_EQ(task.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); - EXPECT_EQ(task.payload->early_dispatch_launch_state.load(), PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE); + EXPECT_EQ(task.payload->early_dispatch_state.load(), EARLY_DISPATCH_DISPATCHED); + EXPECT_EQ(task.payload->early_dispatch_launch_state.load(), EARLY_DISPATCH_LAUNCH_COMPLETE); EXPECT_EQ(task.payload->staged_core_mask[0].load(), 0); EXPECT_EQ(get_test_reg_stub_base_addr(), reg_addr); EXPECT_EQ(get_test_reg_stub_value(), (static_cast(token) << 32) | token); @@ -772,7 +772,7 @@ TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { TaskPayload payload{}; for (int iteration = 0; iteration < 1000; iteration++) { - payload.early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); + payload.early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); std::atomic start{false}; bool first_won = false; bool second_won = false; @@ -791,9 +791,7 @@ TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { second.join(); EXPECT_NE(first_won, second_won); - EXPECT_EQ( - payload.early_dispatch_launch_state.load(std::memory_order_acquire), PTO2_EARLY_DISPATCH_LAUNCH_RINGING - ); + EXPECT_EQ(payload.early_dispatch_launch_state.load(std::memory_order_acquire), EARLY_DISPATCH_LAUNCH_RINGING); } } @@ -810,7 +808,7 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { sched.record_published_blocks(sync_consumer, sync_consumer.logical_block_num); sync_consumer.payload->staged_core_mask[0].store(0b11, std::memory_order_relaxed); sync_consumer.payload->running_slot_count.store(0, std::memory_order_relaxed); - sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); downstream.payload->fanin_actual_count = 1; DepListEntry dep{}; @@ -820,13 +818,13 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { // Producer release wins before staging publishes its running-slot seed. With no pending // promotions, staging finalize is the only remaining rendezvous retry. EXPECT_TRUE(sched.try_early_dispatch_release(sync_consumer)); - EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); - EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), PTO2_EARLY_DISPATCH_LAUNCH_NONE); + EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_DISPATCHED); + EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), EARLY_DISPATCH_LAUNCH_NONE); EXPECT_EQ(downstream.payload->dispatch_fanin.load(), 0); sync_consumer.payload->running_slot_count.store(2, std::memory_order_seq_cst); EXPECT_TRUE(sched.retry_sync_start_rendezvous_after_staging(sync_consumer)); - EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE); + EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), EARLY_DISPATCH_LAUNCH_COMPLETE); EXPECT_TRUE(sync_consumer.has_dispatch_propagated()); EXPECT_EQ(downstream.payload->dispatch_fanin.load(), 1); @@ -847,7 +845,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { sched.record_published_blocks(sync_consumer, sync_consumer.logical_block_num); sync_consumer.payload->staged_core_mask[0].store(0b11, std::memory_order_relaxed); sync_consumer.payload->running_slot_count.store(2, std::memory_order_relaxed); - sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); downstream.payload->fanin_actual_count = 1; DepListEntry dep{}; @@ -856,8 +854,8 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { EXPECT_FALSE(sched.retry_sync_start_rendezvous_after_staging(sync_consumer)); EXPECT_TRUE(sched.try_early_dispatch_release(sync_consumer)); - EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); - EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE); + EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_DISPATCHED); + EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), EARLY_DISPATCH_LAUNCH_COMPLETE); EXPECT_TRUE(sync_consumer.has_dispatch_propagated()); EXPECT_EQ(downstream.payload->dispatch_fanin.load(), 1); @@ -871,23 +869,23 @@ TEST_F(WiringTest, ArmedEarlySyncDrainOwnsFinalReadyRoute) { sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; - sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); SchedulerState::mark_early_sync_drain_armed(*sync_consumer.payload); EXPECT_TRUE(sched.release_fanin_and_check_ready(sync_consumer)); auto shape = static_cast(sync_consumer.active_mask.to_shape()); EXPECT_EQ(sched.ready_sync_queues[shape].pop(), nullptr); - EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); + EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_DISPATCHED); EXPECT_EQ( sync_consumer.payload->early_sync_drain_state.load(), - PTO2_EARLY_SYNC_DRAIN_OWNER | PTO2_EARLY_SYNC_DRAIN_ARMED | PTO2_EARLY_SYNC_DRAIN_READY + EARLY_SYNC_DRAIN_OWNER | EARLY_SYNC_DRAIN_ARMED | EARLY_SYNC_DRAIN_READY ); } TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { alignas(64) TaskPayload payload{}; - payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + payload.early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(payload)); SchedulerState::mark_early_sync_drain_armed(payload); @@ -912,7 +910,7 @@ TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { } start.store(true, std::memory_order_release); while (before_ready.load(std::memory_order_acquire) != 2) {} - payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst); + payload.early_dispatch_state.store(EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst); bool ready_has_owner = SchedulerState::publish_ready_to_early_sync_drain(payload); ready_published.store(true, std::memory_order_release); for (auto &stager : stagers) @@ -923,7 +921,7 @@ TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { EXPECT_TRUE(was_gated); alignas(64) TaskPayload normal_ready{}; - normal_ready.early_dispatch_state.store(PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_relaxed); + normal_ready.early_dispatch_state.store(EARLY_DISPATCH_DISPATCHED, std::memory_order_relaxed); EXPECT_FALSE(SchedulerState::publish_ready_to_early_sync_drain(normal_ready)); EXPECT_FALSE(SchedulerState::owns_early_sync_drain(normal_ready)); EXPECT_FALSE( @@ -940,7 +938,7 @@ TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; - sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); SchedulerState::mark_early_sync_drain_armed(*sync_consumer.payload); @@ -953,8 +951,7 @@ TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) EXPECT_TRUE(SchedulerState::publish_ready_to_early_sync_drain(*sync_consumer.payload)); EXPECT_EQ( sync_consumer.payload->early_sync_drain_state.load(), - PTO2_EARLY_SYNC_DRAIN_OWNER | PTO2_EARLY_SYNC_DRAIN_ARMED | PTO2_EARLY_SYNC_DRAIN_READY | - PTO2_EARLY_SYNC_DRAIN_COMPLETE + EARLY_SYNC_DRAIN_OWNER | EARLY_SYNC_DRAIN_ARMED | EARLY_SYNC_DRAIN_READY | EARLY_SYNC_DRAIN_COMPLETE ); } @@ -964,7 +961,7 @@ TEST_F(WiringTest, CancelledEarlySyncDrainRoutesProducerRelease) { sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; - sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); sched.cancel_early_sync_drain(sync_consumer); @@ -982,7 +979,7 @@ TEST_F(WiringTest, ProducerReleaseTransfersReadyRouteToCancellingDrain) { sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; - sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); EXPECT_TRUE(sched.release_fanin_and_check_ready(sync_consumer)); diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index c87ff6e9e2..1a57ae67ef 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -283,7 +283,7 @@ TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { TaskPayload payload{}; for (int iteration = 0; iteration < 1000; iteration++) { - payload.early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); + payload.early_dispatch_launch_state.store(EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); std::atomic start{false}; bool first_won = false; bool second_won = false; @@ -302,9 +302,7 @@ TEST_F(WiringTest, SyncStartDoorbellPassHasOneOwner) { second.join(); EXPECT_NE(first_won, second_won); - EXPECT_EQ( - payload.early_dispatch_launch_state.load(std::memory_order_acquire), PTO2_EARLY_DISPATCH_LAUNCH_RINGING - ); + EXPECT_EQ(payload.early_dispatch_launch_state.load(std::memory_order_acquire), EARLY_DISPATCH_LAUNCH_RINGING); } } @@ -321,7 +319,7 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { sched.record_published_blocks(sync_consumer, sync_consumer.logical_block_num); sync_consumer.payload->staged_core_mask[0].store(0b11, std::memory_order_relaxed); sync_consumer.payload->running_slot_count.store(0, std::memory_order_relaxed); - sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); downstream.payload->fanin_actual_count = 1; DepListEntry dep{}; @@ -329,13 +327,13 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { sync_consumer.fanout_head = &dep; EXPECT_TRUE(sched.try_early_dispatch_release(sync_consumer)); - EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); - EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), PTO2_EARLY_DISPATCH_LAUNCH_NONE); + EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_DISPATCHED); + EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), EARLY_DISPATCH_LAUNCH_NONE); EXPECT_EQ(downstream.payload->dispatch_fanin.load(), 0); sync_consumer.payload->running_slot_count.store(2, std::memory_order_seq_cst); EXPECT_TRUE(sched.retry_sync_start_rendezvous_after_staging(sync_consumer)); - EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE); + EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), EARLY_DISPATCH_LAUNCH_COMPLETE); EXPECT_TRUE(sync_consumer.has_dispatch_propagated()); EXPECT_EQ(downstream.payload->dispatch_fanin.load(), 1); @@ -356,7 +354,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { sched.record_published_blocks(sync_consumer, sync_consumer.logical_block_num); sync_consumer.payload->staged_core_mask[0].store(0b11, std::memory_order_relaxed); sync_consumer.payload->running_slot_count.store(2, std::memory_order_relaxed); - sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); downstream.payload->fanin_actual_count = 1; DepListEntry dep{}; @@ -365,8 +363,8 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { EXPECT_FALSE(sched.retry_sync_start_rendezvous_after_staging(sync_consumer)); EXPECT_TRUE(sched.try_early_dispatch_release(sync_consumer)); - EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); - EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), PTO2_EARLY_DISPATCH_LAUNCH_COMPLETE); + EXPECT_EQ(sync_consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_DISPATCHED); + EXPECT_EQ(sync_consumer.payload->early_dispatch_launch_state.load(), EARLY_DISPATCH_LAUNCH_COMPLETE); EXPECT_TRUE(sync_consumer.has_dispatch_propagated()); EXPECT_EQ(downstream.payload->dispatch_fanin.load(), 1); @@ -380,7 +378,7 @@ TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; - sync_consumer.payload->early_dispatch_state.store(PTO2_EARLY_DISPATCH_STAGING, std::memory_order_relaxed); + sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); ASSERT_TRUE(SchedulerState::try_claim_early_sync_drain(*sync_consumer.payload)); SchedulerState::mark_early_sync_drain_armed(*sync_consumer.payload); @@ -392,8 +390,7 @@ TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) EXPECT_TRUE(SchedulerState::publish_ready_to_early_sync_drain(*sync_consumer.payload)); EXPECT_EQ( sync_consumer.payload->early_sync_drain_state.load(), - PTO2_EARLY_SYNC_DRAIN_OWNER | PTO2_EARLY_SYNC_DRAIN_ARMED | PTO2_EARLY_SYNC_DRAIN_READY | - PTO2_EARLY_SYNC_DRAIN_COMPLETE + EARLY_SYNC_DRAIN_OWNER | EARLY_SYNC_DRAIN_ARMED | EARLY_SYNC_DRAIN_READY | EARLY_SYNC_DRAIN_COMPLETE ); } @@ -1001,12 +998,12 @@ TEST_F(WiringTest, EarlyDispatchQueueOverflowFallsBackToNormalDispatch) { } sched.try_enqueue_early_dispatch_candidate(consumer); - ASSERT_EQ(consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_NONE); + ASSERT_EQ(consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_NONE); // The overflowed candidate carries no early-dispatch claim, so the producer // release routes every block through the ordinary ready queue. EXPECT_TRUE(sched.route_ready_once(consumer)); - EXPECT_EQ(consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_DISPATCHED); + EXPECT_EQ(consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_DISPATCHED); EXPECT_EQ(sched.ready_queues[static_cast(shape)].pop(), &consumer); } @@ -1023,7 +1020,7 @@ TEST_F(WiringTest, EarlyDispatchSyncStartQueueOverflowFallsBackToSyncReadyQueue) } sched.try_enqueue_early_dispatch_candidate(consumer); - EXPECT_EQ(consumer.payload->early_dispatch_state.load(), PTO2_EARLY_DISPATCH_NONE); + EXPECT_EQ(consumer.payload->early_dispatch_state.load(), EARLY_DISPATCH_NONE); EXPECT_EQ(queue.size(), queue.capacity); // A sync_start cohort that never reached its drain falls back to the From 3c0c0aae2234c210df1e96313c9495c6d1d774d1 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:47:19 -0700 Subject: [PATCH 14/22] Refactor: retire the PTO2 prefix on the subtask and dispatch types `PTO2SubtaskSlot` -> `SubtaskSlot` with `SUBTASK_SLOT_COUNT` and the three `SUBTASK_MASK_AIC` / `SUBTASK_MASK_AIV0` / `SUBTASK_MASK_AIV1` selectors, and `PTO2DispatchPayload` -> `DispatchPayload` with `DISPATCH_MAX_ARGS`. 442 occurrences across 72 files. All five constants here are `inline constexpr` in `runtime/submit_types.h` rather than macros, so they can take the bare name: a clash would be a redeclaration the compiler reports, not a silent textual rewrite. The masks name the slots, so they move with the slot type; the payload is what a dispatch writes into one, so its argument cap moves with it. Co-Authored-By: Claude Opus 5 (1M context) --- ...026-07-aicore-fills-all-args-ready-path.md | 4 +-- .../aicore/aicore_executor.cpp | 12 +++---- .../runtime/host_build_graph/build_config.py | 2 +- .../host_build_graph/common/intrinsic.h | 4 +-- .../runtime/dispatch_payload.h | 16 +++++----- .../orchestrator_core/orchestrator.cpp | 32 +++++++++---------- .../host_build_graph/runtime/runtime.h | 10 +++--- .../host_build_graph/runtime/runtime_types.h | 6 ++-- .../runtime/scheduler/scheduler_cold_path.cpp | 6 ++-- .../scheduler/scheduler_completion.cpp | 6 ++-- .../runtime/scheduler/scheduler_context.h | 22 ++++++------- .../runtime/scheduler/scheduler_dispatch.cpp | 26 +++++++-------- .../runtime/scheduler/scheduler_types.h | 16 +++++----- .../host_build_graph/runtime/submit_types.h | 20 ++++++------ .../aicore/aicore_executor.cpp | 12 +++---- .../tensormap_and_ringbuffer/build_config.py | 2 +- .../common/intrinsic.h | 4 +-- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 8 ++--- .../runtime/dispatch_payload.h | 16 +++++----- .../runtime/orchestrator.cpp | 20 ++++++------ .../runtime/runtime.h | 10 +++--- .../runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler_cold_path.cpp | 8 ++--- .../scheduler/scheduler_completion.cpp | 6 ++-- .../runtime/scheduler/scheduler_context.h | 22 ++++++------- .../runtime/scheduler/scheduler_dispatch.cpp | 26 +++++++-------- .../runtime/scheduler/scheduler_types.h | 16 +++++----- .../runtime/submit_types.h | 20 ++++++------ .../aicore/aicore_executor.cpp | 12 +++---- .../runtime/host_build_graph/build_config.py | 2 +- .../host_build_graph/common/intrinsic.h | 4 +-- .../runtime/dispatch_payload.h | 16 +++++----- .../orchestrator_core/orchestrator.cpp | 32 +++++++++---------- .../host_build_graph/runtime/runtime.h | 10 +++--- .../host_build_graph/runtime/runtime_types.h | 6 ++-- .../runtime/scheduler/scheduler_cold_path.cpp | 6 ++-- .../scheduler/scheduler_completion.cpp | 6 ++-- .../runtime/scheduler/scheduler_context.h | 22 ++++++------- .../runtime/scheduler/scheduler_dispatch.cpp | 26 +++++++-------- .../runtime/scheduler/scheduler_types.h | 16 +++++----- .../host_build_graph/runtime/submit_types.h | 20 ++++++------ .../aicore/aicore_executor.cpp | 12 +++---- .../tensormap_and_ringbuffer/build_config.py | 2 +- .../common/intrinsic.h | 4 +-- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 8 ++--- .../runtime/dispatch_payload.h | 16 +++++----- .../runtime/orchestrator.cpp | 20 ++++++------ .../runtime/runtime.h | 10 +++--- .../runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler_cold_path.cpp | 8 ++--- .../scheduler/scheduler_completion.cpp | 6 ++-- .../runtime/scheduler/scheduler_context.h | 22 ++++++------- .../runtime/scheduler/scheduler_dispatch.cpp | 26 +++++++-------- .../runtime/scheduler/scheduler_types.h | 16 +++++----- .../runtime/submit_types.h | 20 ++++++------ .../host_build_graph/graph_execution.cpp | 6 ++-- src/common/host_build_graph/graph_execution.h | 2 +- .../platform/include/aicpu/args_dump_aicpu.h | 2 +- .../platform/include/common/args_dump.h | 2 +- tests/ut/cpp/a2a3/test_graph_activation.cpp | 2 +- tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp | 2 +- tests/ut/cpp/a2a3/test_scheduler_state.cpp | 26 +++++++-------- tests/ut/cpp/a2a3/test_task_state.cpp | 2 +- tests/ut/cpp/a2a3/test_wiring.cpp | 18 +++++------ tests/ut/cpp/a5/test_graph_activation.cpp | 2 +- tests/ut/cpp/a5/test_hbg_submit_poison.cpp | 2 +- tests/ut/cpp/a5/test_scheduler_state.cpp | 26 +++++++-------- tests/ut/cpp/a5/test_task_state.cpp | 2 +- tests/ut/cpp/a5/test_wiring.cpp | 12 +++---- tests/ut/cpp/common/test_hbg_core_tracker.cpp | 2 +- 72 files changed, 408 insertions(+), 408 deletions(-) diff --git a/docs/investigations/2026-07-aicore-fills-all-args-ready-path.md b/docs/investigations/2026-07-aicore-fills-all-args-ready-path.md index 68c725f39d..4763354237 100644 --- a/docs/investigations/2026-07-aicore-fills-all-args-ready-path.md +++ b/docs/investigations/2026-07-aicore-fills-all-args-ready-path.md @@ -12,7 +12,7 @@ AICPU filling `args[]` for ready tasks and offloads only gated ones. ## Question The tmr dispatch handoff writes the kernel `args[]` (tensor GM pointers + -scalar values) into the per-core `PTO2DispatchPayload` on the AICPU. For +scalar values) into the per-core `DispatchPayload` on the AICPU. For gated (`not_ready` / early-dispatch) tasks we already offload that fill to the AICore — it does it during its doorbell wait, off the AICPU dispatch path (see the folded-gate `src_payload` design in the dispatch cold-write @@ -25,7 +25,7 @@ Natural follow-on: why not offload the fill for **all** tasks and take the An experiment build (not merged) that: -- Adds a separate `gated` flag to `PTO2DispatchPayload` — with all tasks +- Adds a separate `gated` flag to `DispatchPayload` — with all tasks filling from source, `src_payload` is always non-zero and can no longer double as the gate flag (the shipped design folds `not_ready` into `src_payload == 0`). diff --git a/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp b/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp index 61da824b68..968b97640b 100644 --- a/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp +++ b/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp @@ -27,13 +27,13 @@ typedef void (*UnifiedKernelFunc)(__gm__ int64_t *); /** - * Execute task from PTO2DispatchPayload. + * Execute task from DispatchPayload. * * Reads function_bin_addr and args from the dispatch payload. * - * @param payload Pointer to PTO2DispatchPayload in global memory + * @param payload Pointer to DispatchPayload in global memory */ -__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ PTO2DispatchPayload *payload) { +__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ DispatchPayload *payload) { if (payload == nullptr || payload->function_bin_addr == 0) { return; } @@ -49,7 +49,7 @@ __aicore__ __attribute__((always_inline)) static void execute_task(__gm__ PTO2Di * Implements the AICPU-AICore register-based dispatch protocol: * 1. Report physical core ID and core type, signal aicore_done (no AICPU wait) * 2. Wait for the AICPU to open our register window (DATA_MAIN_BASE != 0) - * 3. Cache per-core PTO2DispatchPayload pointer from my_hank->task + * 3. Cache per-core DispatchPayload pointer from my_hank->task * 4. Poll DATA_MAIN_BASE register for task dispatch until exit signal * * AICore reports on launch; the AICPU writes &s_payload_per_core[i] to @@ -101,7 +101,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // above cannot clobber it) and before opening the window; dcci to read its // fresh value here. dcci(my_hank, SINGLE_CACHE_LINE); - __gm__ PTO2DispatchPayload *payload = reinterpret_cast<__gm__ PTO2DispatchPayload *>(my_hank->task); + __gm__ DispatchPayload *payload = reinterpret_cast<__gm__ DispatchPayload *>(my_hank->task); uint32_t enable_profiling_flag = get_aicore_profiling_flag(); bool chip_swimlane_enabled = SIMPLER_GET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_CHIP_SWIMLANE); @@ -166,7 +166,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in uint32_t task_id = reg_val; // Decode: register holds task_id directly // Select dual-buffer slot: same bit as AICPU used when writing payload - __gm__ PTO2DispatchPayload *exec_payload = payload + (task_id & 1u); + __gm__ DispatchPayload *exec_payload = payload + (task_id & 1u); // Invalidate payload buffer (AICPU updates its content each dispatch) dcci(exec_payload, ENTIRE_DATA_CACHE); diff --git a/src/a2a3/runtime/host_build_graph/build_config.py b/src/a2a3/runtime/host_build_graph/build_config.py index 8caf89cd3a..40b6524dd3 100644 --- a/src/a2a3/runtime/host_build_graph/build_config.py +++ b/src/a2a3/runtime/host_build_graph/build_config.py @@ -16,7 +16,7 @@ # the image to device (vs tensormap, where AICPU thread N-1 orchestrates on # device concurrently with the scheduler threads) # - AICPU threads 0..N-1 all run schedulers (no on-device orchestrator thread) -# - AICore executes tasks via an aligned PTO2DispatchPayload + pre-built dispatch_args +# - AICore executes tasks via an aligned DispatchPayload + pre-built dispatch_args # # The "orchestration" directory contains source files compiled into both # runtime targets AND the orchestration .so (e.g., tensor methods needed diff --git a/src/a2a3/runtime/host_build_graph/common/intrinsic.h b/src/a2a3/runtime/host_build_graph/common/intrinsic.h index 72bb241374..8dcd4ca5fd 100644 --- a/src/a2a3/runtime/host_build_graph/common/intrinsic.h +++ b/src/a2a3/runtime/host_build_graph/common/intrinsic.h @@ -106,7 +106,7 @@ static constexpr int32_t PAYLOAD_LOCAL_CONTEXT_INDEX = SPMD_LOCAL_CONTEXT_INDEX; static constexpr int32_t PAYLOAD_GLOBAL_CONTEXT_INDEX = SPMD_GLOBAL_CONTEXT_INDEX; /** - * Per-core global context, stored in PTO2DispatchPayload. + * Per-core global context, stored in DispatchPayload. * Initialized once at runtime startup (init_global_context) based on each * core's cluster position. Never modified after initialization. */ @@ -140,7 +140,7 @@ struct AsyncCtx { }; /** - * Per-dispatch local context, stored in PTO2DispatchPayload. + * Per-dispatch local context, stored in DispatchPayload. * Written by build_payload() before each dispatch. Different blocks of the * same task receive different block_idx values but the same block_num. * diff --git a/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h b/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h index 554e797d17..13169f89bc 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h +++ b/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h @@ -13,7 +13,7 @@ * @file dispatch_payload.h * @brief Per-core dispatch payload for AICore kernel execution * - * PTO2DispatchPayload holds the kernel function address, a per-core args[] + * DispatchPayload holds the kernel function address, a per-core args[] * array, and embedded SPMD context (LocalContext + GlobalContext). AICPU * maintains a static array of these (one per core). * @@ -42,8 +42,8 @@ #include "intrinsic.h" /** Max dispatch arguments: 16 scalars + up to 32 tensor pointers + ext params */ -#ifndef PTO2_DISPATCH_MAX_ARGS -#define PTO2_DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) +#ifndef DISPATCH_MAX_ARGS +#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) #endif #ifndef PTO2_ALIGN_UP @@ -86,7 +86,7 @@ constexpr uint32_t TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) * The struct is cache-line aligned to prevent false sharing across * concurrently dispatched cores. */ -struct alignas(64) PTO2DispatchPayload { +struct alignas(64) DispatchPayload { // === Cache line 0 (64B): control block, the only line written per dispatch === // function_bin_addr, local_context.{block_idx,block_num,async_ctx.task_token} // and src_payload are the per-dispatch writes; async_ctx's slab pointers + @@ -114,7 +114,7 @@ struct alignas(64) PTO2DispatchPayload { * = &local_context, [SPMD_GLOBAL_CONTEXT_INDEX] = &global_context (both prefilled * once at init). On the gated path the AICPU leaves [0..num_args) unwritten and * the idle AICore fills them from src_payload during its gate wait. */ - uint64_t args[PTO2_DISPATCH_MAX_ARGS]; + uint64_t args[DISPATCH_MAX_ARGS]; /** Per-core global context: sub_block_id (AIV lane identity). Cold: written once * at init, never per dispatch, so it lives in the tail (not the CL0 control @@ -129,6 +129,6 @@ struct alignas(64) PTO2DispatchPayload { ); }; -static_assert(sizeof(PTO2DispatchPayload) == 512, "PTO2DispatchPayload hardware ABI size drift"); -static_assert(offsetof(PTO2DispatchPayload, args) == 64, "args[] must start at cache line 1 (control block = CL0)"); -static_assert(offsetof(PTO2DispatchPayload, src_payload) < 64, "src_payload (gate) must live on the CL0 control block"); +static_assert(sizeof(DispatchPayload) == 512, "DispatchPayload hardware ABI size drift"); +static_assert(offsetof(DispatchPayload, args) == 64, "args[] must start at cache line 1 (control block = CL0)"); +static_assert(offsetof(DispatchPayload, src_payload) < 64, "src_payload (gate) must live on the CL0 control block"); diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index ea9fefd4e1..f1d47ae637 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -315,7 +315,7 @@ struct GraphRecordedPredicate { }; struct GraphRecordedNode { - std::array kernel_ids{}; + std::array kernel_ids{}; ActiveMask active_mask{}; TaskAttrs task_attrs{}; int16_t logical_block_num{1}; @@ -1473,7 +1473,7 @@ static TaskOutputTensors submit_task_common( // fanout now that the swimlane hot path no longer records it. const bool capture_dep_graph = dep_gen_host_graph_enabled(); if (capture_dep_graph) { - const std::array kernel_ids_capture{ + const std::array kernel_ids_capture{ aic_kernel_id, aiv0_kernel_id, aiv1_kernel_id, @@ -1566,9 +1566,9 @@ static TaskOutputTensors submit_task_common( // evicted by TensorMap lookup/insert cache pressure. __builtin_prefetch(&task, 1, 1); task.task_id = task_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = aic_kernel_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = aiv0_kernel_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = aiv1_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIC)] = aic_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIV0)] = aiv0_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIV1)] = aiv1_kernel_id; task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; @@ -1860,7 +1860,7 @@ bool graph_submit_outer( // the args it consumes and the edges those produce. const bool capture_dep_graph = dep_gen_host_graph_enabled(); if (capture_dep_graph) { - const std::array kernel_ids_capture{ + const std::array kernel_ids_capture{ INVALID_KERNEL_ID, INVALID_KERNEL_ID, INVALID_KERNEL_ID, @@ -2001,9 +2001,9 @@ TaskOutputTensors graph_record_submit_node( if (node_index >= recording.nodes.size()) recording.nodes.emplace_back(); GraphRecordedNode &node = recording.nodes[node_index]; node.reset(); - node.kernel_ids[static_cast(PTO2SubtaskSlot::AIC)] = aic_kernel_id; - node.kernel_ids[static_cast(PTO2SubtaskSlot::AIV0)] = aiv0_kernel_id; - node.kernel_ids[static_cast(PTO2SubtaskSlot::AIV1)] = aiv1_kernel_id; + node.kernel_ids[static_cast(SubtaskSlot::AIC)] = aic_kernel_id; + node.kernel_ids[static_cast(SubtaskSlot::AIV0)] = aiv0_kernel_id; + node.kernel_ids[static_cast(SubtaskSlot::AIV1)] = aiv1_kernel_id; node.active_mask = active_mask; node.task_attrs = task_attrs; node.task_attrs.set_early_resolve(false); @@ -2551,13 +2551,13 @@ TaskOutputTensors OrchestratorState::submit_task(const MixedKernels &mixed_kerne // Normalize single-AIV tasks: if only aiv1 is set (no aic, no aiv0), move // it to the aiv0 slot. This guarantees the dispatch path can always use - // PTO2SubtaskSlot::AIV0 for single-AIV shapes without inspecting active_mask. + // SubtaskSlot::AIV0 for single-AIV shapes without inspecting active_mask. // Mixed tasks (AIC+AIV) keep their original AIV identity so the correct // hardware channel (AIV0→AIC vs AIV1→AIC) is used at dispatch time. MixedKernels normalized = mixed_kernels; - bool has_aic = active_mask.has_mask(PTO2_SUBTASK_MASK_AIC); - bool has_aiv0 = active_mask.has_mask(PTO2_SUBTASK_MASK_AIV0); - bool has_aiv1 = active_mask.has_mask(PTO2_SUBTASK_MASK_AIV1); + bool has_aic = active_mask.has_mask(SUBTASK_MASK_AIC); + bool has_aiv0 = active_mask.has_mask(SUBTASK_MASK_AIV0); + bool has_aiv1 = active_mask.has_mask(SUBTASK_MASK_AIV1); if (!has_aic && has_aiv1 && !has_aiv0) { normalized.aiv0_kernel_id = normalized.aiv1_kernel_id; normalized.aiv1_kernel_id = INVALID_KERNEL_ID; @@ -2712,9 +2712,9 @@ TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { #endif task.task_id = prepared.task_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = INVALID_KERNEL_ID; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = INVALID_KERNEL_ID; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIC)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIV0)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIV1)] = INVALID_KERNEL_ID; task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime.h b/src/a2a3/runtime/host_build_graph/runtime/runtime.h index 442b7a9a7b..608b6c069f 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime.h @@ -20,7 +20,7 @@ * - Device orchestration state (gm_sm_ptr_, orch_args_) * - Function address mapping (func_id_to_addr_) * - * Task dispatch uses a per-core PTO2DispatchPayload written by the scheduler. + * Task dispatch uses a per-core DispatchPayload written by the scheduler. * At dispatch time, build_payload() copies tensor pointers and scalars from * the task payload into the per-core args[], populates SPMD context, then * signals AICore via DATA_MAIN_BASE. @@ -71,7 +71,7 @@ constexpr int RUNTIME_DEFAULT_READY_QUEUE_SHARDS = PLATFORM_MAX_AICPU_THREADS - * 2. AICPU publishes the task pointer and opens the register window with DATA_MAIN_BASE=IDLE * 3. AICore observes window-open, reports initial idle state, and reads the task pointer * 4. Task Dispatch: AICPU writes DATA_MAIN_BASE after updating the per-core payload - * 5. Task Execution: AICore reads the cached PTO2DispatchPayload and executes + * 5. Task Execution: AICore reads the cached DispatchPayload and executes * 6. Task Completion: AICore writes FIN to COND; AICPU observes completion * 7. Shutdown: AICPU writes the exit signal to DATA_MAIN_BASE; AICore exits * @@ -97,7 +97,7 @@ constexpr int RUNTIME_DEFAULT_READY_QUEUE_SHARDS = PLATFORM_MAX_AICPU_THREADS - struct Handshake { volatile uint32_t aicpu_ready; // Legacy layout field; unused by the current handshake volatile uint32_t aicore_done; // AICore ready signal: 0=not ready, core_id+1=ready - volatile uint64_t task; // PTO2DispatchPayload* published before register window-open + volatile uint64_t task; // DispatchPayload* published before register window-open volatile CoreType core_type; // Core type: CoreType::AIC or CoreType::AIV (reported by AICore with aicore_done) volatile uint32_t physical_core_id; // Physical core ID (reported by AICore with aicore_done) } __attribute__((aligned(64))); @@ -119,7 +119,7 @@ struct TensorPair { /** * Task structure - Compatibility stub for platform layer * - * RT2 uses PTO2DispatchPayload instead of Task for task dispatch. + * RT2 uses DispatchPayload instead of Task for task dispatch. * This stub exists only for API compatibility with device_runner.cpp. * Since get_task_count() returns 0, this struct is never actually used. */ @@ -296,7 +296,7 @@ class Runtime { /** @deprecated Task count is now in PTO2 shared memory */ int get_task_count() const { return 0; } - /** @deprecated RT2 uses PTO2DispatchPayload, not Task. Always returns nullptr. */ + /** @deprecated RT2 uses DispatchPayload, not Task. Always returns nullptr. */ Task *get_task(int) { return nullptr; } // Host-side tensor ledger for D2H copy-back at finalize. Populated by diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index 774b33bdcc..d7f76ac405 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -39,9 +39,9 @@ // dispatch_payload.h include is intentionally dropped here. This header is // pulled in by the platform's args_dump.h via a hardcoded // "host_build_graph/runtime/runtime_types.h" path, and dispatch_payload.h -// uses #pragma once (path-keyed), so leaving it in double-defines PTO2DispatchPayload +// uses #pragma once (path-keyed), so leaving it in double-defines DispatchPayload // against tensormap_and_ringbuffer's copy inside the shared host-dispatcher TU. -// runtime_types.h never references PTO2DispatchPayload itself; consumers that +// runtime_types.h never references DispatchPayload itself; consumers that // need it include it via runtime.h directly. #include "aicore_completion_mailbox.h" #include "common/args_dump_task_metadata.h" @@ -237,7 +237,7 @@ struct TaskDescriptor { TaskId task_id; // raw: (ring_id << 32) | local_id // Per-slot kernel IDs (INVALID_KERNEL_ID = inactive) - int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT]; + int32_t kernel_id[SUBTASK_SLOT_COUNT]; // Packed output buffer (all outputs packed into single contiguous buffer) void *packed_buffer_base; // Start of packed buffer in GM Heap diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index c08161b4dc..8956105da6 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -384,13 +384,13 @@ int32_t SchedulerContext::handle_timeout_exit( // Capture the in-flight kernels' partial output before signalling the // cores to exit, so the dump reflects the live stuck state. if (is_dump_args_enabled()) { - dump_running_task_outputs( + dump_running_task_outputs( thread_idx, cores_total_num_, [this](int32_t cid) { return core_exec_states_[cid].running_slot_state; }, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); @@ -958,7 +958,7 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // live on a later line). for (int32_t core_id = 0; core_id < RUNTIME_MAX_WORKER; core_id++) { for (int32_t buf = 0; buf < 2; buf++) { - PTO2DispatchPayload &dp = payload_per_core_[core_id][buf]; + DispatchPayload &dp = payload_per_core_[core_id][buf]; AsyncCtx &ac = dp.local_context.async_ctx; volatile DeferredCompletionSlab *slab = &deferred_slab_per_core_[core_id][buf]; ac.completion_count = &slab->count; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index eee285fa90..dc4d65c7ed 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -81,7 +81,7 @@ SlotTransition SchedulerContext::decide_slot_transition( // Complete one slot's task: subtask counting, mixed completion, deferred release, profiling. void SchedulerContext::complete_slot_task( - ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, [[maybe_unused]] PTO2SubtaskSlot subslot, + ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, [[maybe_unused]] SubtaskSlot subslot, int32_t thread_idx, int32_t core_id, Handshake *hank, [[maybe_unused]] int32_t &completed_this_turn #if SIMPLER_DFX , @@ -174,10 +174,10 @@ void SchedulerContext::complete_slot_task( if (task_complete && !defer_completion_to_consumer) { #if SIMPLER_DFX if (is_dump_args_enabled()) { - dump_args_for_task( + dump_args_for_task( thread_idx, slot_state, ArgsDumpStage::AFTER_COMPLETION, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index 41019cf4cc..43965a8ca5 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -199,7 +199,7 @@ class SchedulerContext { // Per-core dispatch payload storage: dual-buffer for pipelining. // buf_idx = reg_task_id & 1; adjacent dispatches alternate automatically. - PTO2DispatchPayload payload_per_core_[RUNTIME_MAX_WORKER][2]; + DispatchPayload payload_per_core_[RUNTIME_MAX_WORKER][2]; // Per-core deferred-completion software registration storage. This has // the same runtime lifetime as payload_per_core_, but is kept out of the @@ -283,16 +283,16 @@ class SchedulerContext { static const char *shape_name(ResourceShape shape); - // Lower-case rendering of PTO2SubtaskSlot, used by dispatch and stall logs. + // Lower-case rendering of SubtaskSlot, used by dispatch and stall logs. // Kept lower-case to match the `kernels=[aic:N aiv0:N aiv1:N]` field // convention already established in the stall log family. - static inline const char *subslot_name(PTO2SubtaskSlot s) { + static inline const char *subslot_name(SubtaskSlot s) { switch (s) { - case PTO2SubtaskSlot::AIC: + case SubtaskSlot::AIC: return "aic"; - case PTO2SubtaskSlot::AIV0: + case SubtaskSlot::AIV0: return "aiv0"; - case PTO2SubtaskSlot::AIV1: + case SubtaskSlot::AIV1: return "aiv1"; } return "?"; @@ -303,8 +303,8 @@ class SchedulerContext { ); void build_payload( - PTO2DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, - int32_t block_idx, bool force_gate + DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, SubtaskSlot subslot, int32_t block_idx, + bool force_gate ); // Batched-dispatch primitives. prepare_* builds the payload and per-core @@ -325,8 +325,8 @@ class SchedulerContext { }; PublishHandle prepare_subtask_to_core( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, - bool to_pending, int32_t block_idx, bool force_gate + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, SubtaskSlot subslot, bool to_pending, + int32_t block_idx, bool force_gate ); // `thread_idx` is the publishing Scheduler thread's index, used to select the @@ -482,7 +482,7 @@ class SchedulerContext { ); void complete_slot_task( - ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, PTO2SubtaskSlot subslot, int32_t thread_idx, + ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, SubtaskSlot subslot, int32_t thread_idx, int32_t core_id, Handshake *hank, int32_t &completed_this_turn #if SIMPLER_DFX , diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index d0d0db44f5..61f5ca99dc 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -120,7 +120,7 @@ int SchedulerContext::pop_ready_tasks_batch( } void SchedulerContext::build_payload( - PTO2DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, int32_t block_idx, + DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, SubtaskSlot subslot, int32_t block_idx, bool force_gate ) { int32_t slot_idx = static_cast(subslot); @@ -168,7 +168,7 @@ void SchedulerContext::build_payload( } SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, bool to_pending, + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, SubtaskSlot subslot, bool to_pending, int32_t block_idx, bool force_gate ) { CoreTracker &tracker = core_trackers_[thread_idx]; @@ -186,7 +186,7 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( } uint32_t buf_idx = reg_task_id & 1u; - PTO2DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; + DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; // The deferred_slab is NOT reset here: it is init-cleared once and the // completion path re-clears count only after a task actually recorded a // deferred completion (count > 0), which is rare. A non-deferred task never @@ -246,10 +246,10 @@ int SchedulerContext::prepare_block_for_dispatch( ) { #if SIMPLER_DFX if (is_dump_args_enabled()) { - dump_args_for_task( + dump_args_for_task( thread_idx, slot_state, ArgsDumpStage::BEFORE_DISPATCH, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); @@ -267,24 +267,24 @@ int SchedulerContext::prepare_block_for_dispatch( // (promoted on completion). Gated sync_start staging relies on this — it passes // to_pending=true so every busy core opts into a pending slot; the non-zero // src_payload gate keeps the whole cohort waiting for the rendezvous. - if (cmask & PTO2_SUBTASK_MASK_AIC) { + if (cmask & SUBTASK_MASK_AIC) { bool p = to_pending && !tracker.is_aic_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aic_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIC, p, block_idx, + thread_idx, tracker.get_aic_core_offset(core_offset), slot_state, SubtaskSlot::AIC, p, block_idx, force_gate ); } - if (cmask & PTO2_SUBTASK_MASK_AIV0) { + if (cmask & SUBTASK_MASK_AIV0) { bool p = to_pending && !tracker.is_aiv0_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aiv0_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIV0, p, block_idx, + thread_idx, tracker.get_aiv0_core_offset(core_offset), slot_state, SubtaskSlot::AIV0, p, block_idx, force_gate ); } - if (cmask & PTO2_SUBTASK_MASK_AIV1) { + if (cmask & SUBTASK_MASK_AIV1) { bool p = to_pending && !tracker.is_aiv1_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aiv1_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIV1, p, block_idx, + thread_idx, tracker.get_aiv1_core_offset(core_offset), slot_state, SubtaskSlot::AIV1, p, block_idx, force_gate ); } @@ -294,7 +294,7 @@ int SchedulerContext::prepare_block_for_dispatch( return n; } else if (shape == ResourceShape::AIC) { out_handles[0] = prepare_subtask_to_core( - thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIC, to_pending, block_idx, force_gate + thread_idx, core_offset, slot_state, SubtaskSlot::AIC, to_pending, block_idx, force_gate ); #if SIMPLER_DFX sched_chip_swimlane_[thread_idx].phase_dispatch_count += 1; @@ -302,7 +302,7 @@ int SchedulerContext::prepare_block_for_dispatch( return 1; } else { out_handles[0] = prepare_subtask_to_core( - thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIV0, to_pending, block_idx, force_gate + thread_idx, core_offset, slot_state, SubtaskSlot::AIV0, to_pending, block_idx, force_gate ); #if SIMPLER_DFX sched_chip_swimlane_[thread_idx].phase_dispatch_count += 1; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h index 367ae824eb..7ace4472ad 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_types.h @@ -111,8 +111,8 @@ struct alignas(64) CoreExecState { int32_t running_reg_task_id; // offset 24: register task ID (AICPU_TASK_INVALID = idle) int32_t pending_reg_task_id; // offset 28: pending register task ID (AICPU_TASK_INVALID = none) uint32_t dispatch_seq; // offset 32: monotonic dispatch counter - PTO2SubtaskSlot running_subslot; // offset 36: which subtask slot is running - PTO2SubtaskSlot pending_subslot; // offset 37: which subtask slot is pending + SubtaskSlot running_subslot; // offset 36: which subtask slot is running + SubtaskSlot pending_subslot; // offset 37: which subtask slot is pending uint8_t pad0_[2]; // offset 38: alignment padding // Precomputed COND register pointer; resolved once in handshake so the // hot completion poll does a single volatile load instead of recomputing @@ -370,13 +370,13 @@ class alignas(64) CoreTracker { // slot) or the mask is empty. MixPlacement classify_mix_cluster(int32_t cluster_offset, uint8_t core_mask) const { BitStates used; - if (core_mask & PTO2_SUBTASK_MASK_AIC) { + if (core_mask & SUBTASK_MASK_AIC) { used |= BitStates::bit(cluster_offset); } - if (core_mask & PTO2_SUBTASK_MASK_AIV0) { + if (core_mask & SUBTASK_MASK_AIV0) { used |= BitStates::bit(cluster_offset + 1); } - if (core_mask & PTO2_SUBTASK_MASK_AIV1) { + if (core_mask & SUBTASK_MASK_AIV1) { used |= BitStates::bit(cluster_offset + 2); } if (!used.has_value() || (pending_occupied_ & used).has_value()) { @@ -412,9 +412,9 @@ class alignas(64) CoreTracker { // Cores of `cluster_offset` named by core_mask. BitStates mix_used_cores(int32_t cluster_offset, uint8_t core_mask) const { BitStates used; - if (core_mask & PTO2_SUBTASK_MASK_AIC) used |= BitStates::bit(cluster_offset); - if (core_mask & PTO2_SUBTASK_MASK_AIV0) used |= BitStates::bit(cluster_offset + 1); - if (core_mask & PTO2_SUBTASK_MASK_AIV1) used |= BitStates::bit(cluster_offset + 2); + if (core_mask & SUBTASK_MASK_AIC) used |= BitStates::bit(cluster_offset); + if (core_mask & SUBTASK_MASK_AIV0) used |= BitStates::bit(cluster_offset + 1); + if (core_mask & SUBTASK_MASK_AIV1) used |= BitStates::bit(cluster_offset + 2); return used; } diff --git a/src/a2a3/runtime/host_build_graph/runtime/submit_types.h b/src/a2a3/runtime/host_build_graph/runtime/submit_types.h index daeabb2776..21756a3e8b 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/submit_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/submit_types.h @@ -25,12 +25,12 @@ inline constexpr int32_t INVALID_KERNEL_ID = -1; /** * Subtask slot count: AIC, AIV0, AIV1 */ -inline constexpr int32_t PTO2_SUBTASK_SLOT_COUNT = 3; +inline constexpr int32_t SUBTASK_SLOT_COUNT = 3; /** * Subtask slot indices */ -enum class PTO2SubtaskSlot : uint8_t { +enum class SubtaskSlot : uint8_t { AIC = 0, AIV0 = 1, AIV1 = 2, @@ -39,9 +39,9 @@ enum class PTO2SubtaskSlot : uint8_t { /** * Subtask mask bits (for ActiveMask) */ -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIC = (1u << 0); // 0x1 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 +inline constexpr uint8_t SUBTASK_MASK_AIC = (1u << 0); // 0x1 +inline constexpr uint8_t SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 +inline constexpr uint8_t SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 // Dispatch-predicate comparison operator. The scheduler evaluates the predicate // at the dispatch point — the task is ready (fanin satisfied), so the predicate @@ -132,7 +132,7 @@ class ActiveMask { uint8_t raw() const { return raw_; } - bool subtask_active(PTO2SubtaskSlot slot) const { return (raw_ & (1u << static_cast(slot))) != 0; } + bool subtask_active(SubtaskSlot slot) const { return (raw_ & (1u << static_cast(slot))) != 0; } uint8_t core_mask() const { return raw_ & 0x07u; } @@ -141,7 +141,7 @@ class ActiveMask { if (cmask == 0) return ResourceShape::DUMMY; int bit_count = __builtin_popcount(cmask); if (bit_count >= 2) return ResourceShape::MIX; - if (cmask & PTO2_SUBTASK_MASK_AIC) return ResourceShape::AIC; + if (cmask & SUBTASK_MASK_AIC) return ResourceShape::AIC; return ResourceShape::AIV; } @@ -244,9 +244,9 @@ struct MixedKernels { ActiveMask to_active_mask() const { uint8_t mask = 0; - if (aic_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIC; - if (aiv0_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIV0; - if (aiv1_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIV1; + if (aic_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIC; + if (aiv0_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIV0; + if (aiv1_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIV1; return ActiveMask(mask); } }; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp index a08f71df99..32e5818568 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp @@ -27,13 +27,13 @@ typedef void (*UnifiedKernelFunc)(__gm__ int64_t *); /** - * Execute task from PTO2DispatchPayload. + * Execute task from DispatchPayload. * * Reads function_bin_addr and args from the dispatch payload. * - * @param payload Pointer to PTO2DispatchPayload in global memory + * @param payload Pointer to DispatchPayload in global memory */ -__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ PTO2DispatchPayload *payload) { +__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ DispatchPayload *payload) { if (payload == nullptr || payload->function_bin_addr == 0) { return; } @@ -49,7 +49,7 @@ __aicore__ __attribute__((always_inline)) static void execute_task(__gm__ PTO2Di * Implements the AICPU-AICore register-based dispatch protocol: * 1. Report physical core ID and core type, signal aicore_done (no AICPU wait) * 2. Wait for the AICPU to open our register window (DATA_MAIN_BASE != 0) - * 3. Cache per-core PTO2DispatchPayload pointer from my_hank->task + * 3. Cache per-core DispatchPayload pointer from my_hank->task * 4. Poll DATA_MAIN_BASE register for task dispatch until exit signal * * AICore reports on launch; the AICPU writes &s_payload_per_core[i] to @@ -101,7 +101,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // above cannot clobber it) and before opening the window; dcci to read its // fresh value here. dcci(my_hank, SINGLE_CACHE_LINE); - __gm__ PTO2DispatchPayload *payload = reinterpret_cast<__gm__ PTO2DispatchPayload *>(my_hank->task); + __gm__ DispatchPayload *payload = reinterpret_cast<__gm__ DispatchPayload *>(my_hank->task); uint32_t enable_profiling_flag = get_aicore_profiling_flag(); bool chip_swimlane_enabled = SIMPLER_GET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_CHIP_SWIMLANE); @@ -158,7 +158,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in } // Select dual-buffer slot: same bit as AICPU used when writing payload - __gm__ PTO2DispatchPayload *exec_payload = payload + (task_id & 1u); + __gm__ DispatchPayload *exec_payload = payload + (task_id & 1u); // Invalidate payload buffer (AICPU updates its content each dispatch) dcci(exec_payload, ENTIRE_DATA_CACHE); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/build_config.py b/src/a2a3/runtime/tensormap_and_ringbuffer/build_config.py index 2cfba734d4..9641f85873 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/build_config.py +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/build_config.py @@ -12,7 +12,7 @@ # This is a device-orchestration runtime where: # - AICPU thread 3 runs the orchestrator (builds task graph on device) # - AICPU threads 0/1/2 run schedulers (dispatch tasks to AICore) -# - AICore executes tasks via an aligned PTO2DispatchPayload + pre-built dispatch_args +# - AICore executes tasks via an aligned DispatchPayload + pre-built dispatch_args # # The "orchestration" directory contains source files compiled into both # runtime targets AND the orchestration .so (e.g., tensor methods needed diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/common/intrinsic.h b/src/a2a3/runtime/tensormap_and_ringbuffer/common/intrinsic.h index ef238be694..5229098552 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/common/intrinsic.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/common/intrinsic.h @@ -110,7 +110,7 @@ static constexpr int32_t PAYLOAD_LOCAL_CONTEXT_INDEX = SPMD_LOCAL_CONTEXT_INDEX; static constexpr int32_t PAYLOAD_GLOBAL_CONTEXT_INDEX = SPMD_GLOBAL_CONTEXT_INDEX; /** - * Per-core global context, stored in PTO2DispatchPayload. + * Per-core global context, stored in DispatchPayload. * Initialized during scheduler cold start from each core's cluster position * and the resident per-device async-DMA config. Stable for that scheduler * instance; unavailable workspace slots remain zero. @@ -150,7 +150,7 @@ struct AsyncCtx { }; /** - * Per-dispatch local context, stored in PTO2DispatchPayload. + * Per-dispatch local context, stored in DispatchPayload. * Written by build_payload() before each dispatch. Different blocks of the * same task receive different block_idx values but the same block_num. * diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 7c54f0753c..2df9338cc2 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -168,7 +168,7 @@ bool entry_valid(const ChipTensorMapEntry& e) { | `DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | | `TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | | `ChipReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | -| `PTO2DispatchPayload` | Built per-dispatch, no ring state needed | +| `DispatchPayload` | Built per-dispatch, no ring state needed | ## 5. Reclamation diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 8f9f7b6997..e571436f67 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -557,7 +557,7 @@ Each scheduler thread runs a tight loop with two main phases: - Service each source (normal ready ▸ speculative early) in occupancy order — `sync_start` Tier-0 ▸ MIX ▸ AIC/AIV, idle ▸ pending — popping from the matching shape-based ready queue (lock-free MPMC Vyukov queue, one per resource shape) -- Build `PTO2DispatchPayload` from `TaskDescriptor` with `task_id`, `subslot`, `kernel_id`, and `core_type` +- Build `DispatchPayload` from `TaskDescriptor` with `task_id`, `subslot`, `kernel_id`, and `core_type` - Write task pointer to `Handshake.task`, signal AICore via register `DATA_MAIN_BASE` After these phases, the scheduler updates profiling headers and checks for termination (all tasks completed and orchestrator done). @@ -725,7 +725,7 @@ Each AICore worker has a `Handshake` struct in shared memory: | Field | Direction | Purpose | | ----- | --------- | ------- | -| `task` | AICPU→AICore | Pointer to `PTO2DispatchPayload` | +| `task` | AICPU→AICore | Pointer to `DispatchPayload` | | `control` | AICPU→AICore | 0=normal, 1=shutdown | | `perf_records_addr` | AICPU→AICore | Performance buffer address | @@ -748,7 +748,7 @@ Instead of polling a shared-memory status flag, the production protocol uses har 4. Execute kernel function via `func_id_to_addr` lookup 5. Write FIN to `COND` -### 9.3 PTO2DispatchPayload +### 9.3 DispatchPayload Built by the scheduler from `TaskDescriptor`: @@ -778,7 +778,7 @@ Built by the scheduler from `TaskDescriptor`: and stores it in `Runtime.dev.func_id_to_addr_[child_func_id(i)]`. 4. When dispatching, the scheduler reads `func_id_to_addr_[fid]`, casts to `const CoreCallable*`, reads `resolved_addr_`, and copies that into - `PTO2DispatchPayload.function_bin_addr`. + `DispatchPayload.function_bin_addr`. ### 10.2 Orchestration SO Loading diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h index ac9a2b9008..81db71c5bd 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h @@ -13,7 +13,7 @@ * @file dispatch_payload.h * @brief Per-core dispatch payload for AICore kernel execution * - * PTO2DispatchPayload holds the kernel function address, a per-core args[] + * DispatchPayload holds the kernel function address, a per-core args[] * array, and embedded SPMD context (LocalContext + GlobalContext). AICPU * maintains a static array of these (one per core). * @@ -43,8 +43,8 @@ #include "intrinsic.h" /** Max dispatch arguments: 16 scalars + up to 32 tensor pointers + ext params */ -#ifndef PTO2_DISPATCH_MAX_ARGS -#define PTO2_DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) +#ifndef DISPATCH_MAX_ARGS +#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) #endif #ifndef PTO2_ALIGN_UP @@ -83,7 +83,7 @@ constexpr uint32_t TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) * The struct is cache-line aligned to prevent false sharing across * concurrently dispatched cores. */ -struct alignas(64) PTO2DispatchPayload { +struct alignas(64) DispatchPayload { // === Cache line 0 (64B): control block, the only line written per dispatch === // function_bin_addr, local_context.{block_idx,block_num,async_ctx.task_token} // and src_payload are the per-dispatch writes; async_ctx's slab pointers + @@ -111,7 +111,7 @@ struct alignas(64) PTO2DispatchPayload { * = &local_context, [SPMD_GLOBAL_CONTEXT_INDEX] = &global_context (both prefilled * once at init). On the gated path the AICPU leaves [0..num_args) unwritten and * the idle AICore fills them from src_payload during its gate wait. */ - uint64_t args[PTO2_DISPATCH_MAX_ARGS]; + uint64_t args[DISPATCH_MAX_ARGS]; /** Per-core global context: sub_block_id (AIV lane identity) plus optional * async-DMA workspace addresses. Cold: written during scheduler init, never @@ -127,6 +127,6 @@ struct alignas(64) PTO2DispatchPayload { ); }; -static_assert(sizeof(PTO2DispatchPayload) == 512, "PTO2DispatchPayload hardware ABI size drift"); -static_assert(offsetof(PTO2DispatchPayload, args) == 64, "args[] must start at cache line 1 (control block = CL0)"); -static_assert(offsetof(PTO2DispatchPayload, src_payload) < 64, "src_payload (gate) must live on the CL0 control block"); +static_assert(sizeof(DispatchPayload) == 512, "DispatchPayload hardware ABI size drift"); +static_assert(offsetof(DispatchPayload, args) == 64, "args[] must start at cache line 1 (control block = CL0)"); +static_assert(offsetof(DispatchPayload, src_payload) < 64, "src_payload (gate) must live on the CL0 control block"); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 8ed2dc174a..51a2c8f7da 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -1021,9 +1021,9 @@ static TaskOutputTensors submit_task_common( // evicted by TensorMap lookup/insert cache pressure. __builtin_prefetch(&task, 1, 1); task.task_id = task_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = aic_kernel_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = aiv0_kernel_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = aiv1_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIC)] = aic_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIV0)] = aiv0_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIV1)] = aiv1_kernel_id; task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; @@ -1174,13 +1174,13 @@ TaskOutputTensors OrchestratorState::submit_task(const MixedKernels &mixed_kerne // Normalize single-AIV tasks: if only aiv1 is set (no aic, no aiv0), move // it to the aiv0 slot. This guarantees the dispatch path can always use - // PTO2SubtaskSlot::AIV0 for single-AIV shapes without inspecting active_mask. + // SubtaskSlot::AIV0 for single-AIV shapes without inspecting active_mask. // Mixed tasks (AIC+AIV) keep their original AIV identity so the correct // hardware channel (AIV0→AIC vs AIV1→AIC) is used at dispatch time. MixedKernels normalized = mixed_kernels; - bool has_aic = active_mask.has_mask(PTO2_SUBTASK_MASK_AIC); - bool has_aiv0 = active_mask.has_mask(PTO2_SUBTASK_MASK_AIV0); - bool has_aiv1 = active_mask.has_mask(PTO2_SUBTASK_MASK_AIV1); + bool has_aic = active_mask.has_mask(SUBTASK_MASK_AIC); + bool has_aiv0 = active_mask.has_mask(SUBTASK_MASK_AIV0); + bool has_aiv1 = active_mask.has_mask(SUBTASK_MASK_AIV1); if (!has_aic && has_aiv1 && !has_aiv0) { normalized.aiv0_kernel_id = normalized.aiv1_kernel_id; normalized.aiv1_kernel_id = INVALID_KERNEL_ID; @@ -1311,9 +1311,9 @@ TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { #endif task.task_id = prepared.task_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = INVALID_KERNEL_ID; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = INVALID_KERNEL_ID; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIC)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIV0)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIV1)] = INVALID_KERNEL_ID; task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime.h index 73419744db..7f89df8daa 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime.h @@ -20,7 +20,7 @@ * - Device orchestration state (gm_sm_ptr_, orch_args_) * - Function address mapping (func_id_to_addr_) * - * Task dispatch uses a per-core PTO2DispatchPayload written by the scheduler. + * Task dispatch uses a per-core DispatchPayload written by the scheduler. * At dispatch time, build_payload() copies tensor pointers and scalars from * the task payload into the per-core args[], populates SPMD context, then * signals AICore via DATA_MAIN_BASE. @@ -73,7 +73,7 @@ constexpr int RUNTIME_DEFAULT_READY_QUEUE_SHARDS = PLATFORM_MAX_AICPU_THREADS - * 2. AICPU publishes the task pointer and opens the register window with DATA_MAIN_BASE=IDLE * 3. AICore observes window-open, reports initial idle state, and reads the task pointer * 4. Task Dispatch: AICPU writes DATA_MAIN_BASE after updating the per-core payload - * 5. Task Execution: AICore reads the cached PTO2DispatchPayload and executes + * 5. Task Execution: AICore reads the cached DispatchPayload and executes * 6. Task Completion: AICore writes FIN to COND; AICPU observes completion * 7. Shutdown: AICPU writes the exit signal to DATA_MAIN_BASE; AICore exits * @@ -99,7 +99,7 @@ constexpr int RUNTIME_DEFAULT_READY_QUEUE_SHARDS = PLATFORM_MAX_AICPU_THREADS - struct Handshake { volatile uint32_t aicpu_ready; // Legacy layout field; unused by the current handshake volatile uint32_t aicore_done; // AICore ready signal: 0=not ready, core_id+1=ready - volatile uint64_t task; // PTO2DispatchPayload* published before register window-open + volatile uint64_t task; // DispatchPayload* published before register window-open volatile CoreType core_type; // Core type: CoreType::AIC or CoreType::AIV (reported by AICore with aicore_done) volatile uint32_t physical_core_id; // Physical core ID (reported by AICore with aicore_done) } __attribute__((aligned(64))); @@ -127,7 +127,7 @@ struct TensorLease { /** * Task structure - Compatibility stub for platform layer * - * RT2 uses PTO2DispatchPayload instead of Task for task dispatch. + * RT2 uses DispatchPayload instead of Task for task dispatch. * This stub exists only for API compatibility with device_runner.cpp. * Since get_task_count() returns 0, this struct is never actually used. */ @@ -309,7 +309,7 @@ class Runtime { /** @deprecated Task count is now in PTO2 shared memory */ int get_task_count() const { return 0; } - /** @deprecated RT2 uses PTO2DispatchPayload, not Task. Always returns nullptr. */ + /** @deprecated RT2 uses DispatchPayload, not Task. Always returns nullptr. */ Task *get_task(int) { return nullptr; } // ========================================================================= diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 4f853215e0..7059f5349b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -220,7 +220,7 @@ struct TaskDescriptor { TaskId task_id; // raw: (ring_id << 32) | local_id // Per-slot kernel IDs (INVALID_KERNEL_ID = inactive) - int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT]; + int32_t kernel_id[SUBTASK_SLOT_COUNT]; // Packed output buffer (all outputs packed into single contiguous buffer) void *packed_buffer_base; // Start of packed buffer in GM Heap diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 697fc0580c..d0a4ff73fa 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -451,13 +451,13 @@ int32_t SchedulerContext::handle_timeout_exit( // Capture the in-flight kernels' partial output before signalling the // cores to exit, so the dump reflects the live stuck state. if (is_dump_args_enabled()) { - dump_running_task_outputs( + dump_running_task_outputs( thread_idx, cores_total_num_, [this](int32_t cid) { return core_exec_states_[cid].running_slot_state; }, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); @@ -944,7 +944,7 @@ void SchedulerContext::assign_own_clusters(int32_t tidx) { for (int32_t sub = 0; sub < PLATFORM_CORES_PER_BLOCKDIM; sub++) { int32_t core_id = tracker.get_core_id_by_offset(c * PLATFORM_CORES_PER_BLOCKDIM + sub); for (int32_t buf = 0; buf < 2; buf++) { - PTO2DispatchPayload &dp = payload_per_core_[core_id][buf]; + DispatchPayload &dp = payload_per_core_[core_id][buf]; AsyncCtx &ac = dp.local_context.async_ctx; volatile DeferredCompletionSlab *slab = &deferred_slab_per_core_[core_id][buf]; ac.completion_count = &slab->count; @@ -1283,7 +1283,7 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // live on a later line). for (int32_t core_id = 0; core_id < RUNTIME_MAX_WORKER; core_id++) { for (int32_t buf = 0; buf < 2; buf++) { - PTO2DispatchPayload &dp = payload_per_core_[core_id][buf]; + DispatchPayload &dp = payload_per_core_[core_id][buf]; AsyncCtx &ac = dp.local_context.async_ctx; volatile DeferredCompletionSlab *slab = &deferred_slab_per_core_[core_id][buf]; ac.completion_count = &slab->count; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index 5bbe448327..fa51e936c6 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -84,7 +84,7 @@ SlotTransition SchedulerContext::decide_slot_transition( // Complete one slot's task: subtask counting, mixed completion, deferred release, profiling. void SchedulerContext::complete_slot_task( - ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, [[maybe_unused]] PTO2SubtaskSlot subslot, + ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, [[maybe_unused]] SubtaskSlot subslot, [[maybe_unused]] int32_t thread_idx, int32_t core_id, Handshake *hank, int32_t &completed_this_turn, ChipTaskSlotState *deferred_release_slot_states[], int32_t &deferred_release_count #if SIMPLER_DFX @@ -178,10 +178,10 @@ void SchedulerContext::complete_slot_task( if (task_complete && !defer_completion_to_consumer) { #if SIMPLER_DFX if (is_dump_args_enabled()) { - dump_args_for_task( + dump_args_for_task( thread_idx, slot_state, ArgsDumpStage::AFTER_COMPLETION, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index e30245dfb2..7085728c97 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -154,7 +154,7 @@ class SchedulerContext { // Per-core dispatch payload storage: dual-buffer for pipelining. // buf_idx = reg_task_id & 1; adjacent dispatches alternate automatically. - PTO2DispatchPayload payload_per_core_[RUNTIME_MAX_WORKER][2]; + DispatchPayload payload_per_core_[RUNTIME_MAX_WORKER][2]; // Per-core deferred-completion software registration storage. This has // the same runtime lifetime as payload_per_core_, but is kept out of the @@ -233,16 +233,16 @@ class SchedulerContext { static const char *shape_name(ResourceShape shape); - // Lower-case rendering of PTO2SubtaskSlot, used by dispatch and stall logs. + // Lower-case rendering of SubtaskSlot, used by dispatch and stall logs. // Kept lower-case to match the `kernels=[aic:N aiv0:N aiv1:N]` field // convention already established in the stall log family. - static inline const char *subslot_name(PTO2SubtaskSlot s) { + static inline const char *subslot_name(SubtaskSlot s) { switch (s) { - case PTO2SubtaskSlot::AIC: + case SubtaskSlot::AIC: return "aic"; - case PTO2SubtaskSlot::AIV0: + case SubtaskSlot::AIV0: return "aiv0"; - case PTO2SubtaskSlot::AIV1: + case SubtaskSlot::AIV1: return "aiv1"; } return "?"; @@ -253,8 +253,8 @@ class SchedulerContext { ); void build_payload( - PTO2DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, - int32_t block_idx, bool force_gate + DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, SubtaskSlot subslot, int32_t block_idx, + bool force_gate ); // Batched-dispatch primitives. prepare_* builds the payload and per-core @@ -275,8 +275,8 @@ class SchedulerContext { }; PublishHandle prepare_subtask_to_core( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, - bool to_pending, int32_t block_idx, bool force_gate + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, SubtaskSlot subslot, bool to_pending, + int32_t block_idx, bool force_gate ); // `thread_idx` is the publishing Scheduler thread's index, used to select the @@ -432,7 +432,7 @@ class SchedulerContext { ); void complete_slot_task( - ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, PTO2SubtaskSlot subslot, int32_t thread_idx, + ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, SubtaskSlot subslot, int32_t thread_idx, int32_t core_id, Handshake *hank, int32_t &completed_this_turn, ChipTaskSlotState *deferred_release_slot_states[], int32_t &deferred_release_count #if SIMPLER_DFX diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 48f79d934d..c73fc463eb 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -122,7 +122,7 @@ int SchedulerContext::pop_ready_tasks_batch( } void SchedulerContext::build_payload( - PTO2DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, int32_t block_idx, + DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, SubtaskSlot subslot, int32_t block_idx, bool force_gate ) { int32_t slot_idx = static_cast(subslot); @@ -163,7 +163,7 @@ void SchedulerContext::build_payload( } SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, bool to_pending, + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, SubtaskSlot subslot, bool to_pending, int32_t block_idx, bool force_gate ) { CoreTracker &tracker = core_trackers_[thread_idx]; @@ -181,7 +181,7 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( } uint32_t buf_idx = reg_task_id & 1u; - PTO2DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; + DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; // The deferred_slab is NOT reset here: it is init-cleared once and the // completion path re-clears count only after a task actually recorded a // deferred completion (count > 0), which is rare. A non-deferred task never @@ -242,10 +242,10 @@ int SchedulerContext::prepare_block_for_dispatch( ) { #if SIMPLER_DFX if (is_dump_args_enabled()) { - dump_args_for_task( + dump_args_for_task( thread_idx, slot_state, ArgsDumpStage::BEFORE_DISPATCH, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); @@ -262,24 +262,24 @@ int SchedulerContext::prepare_block_for_dispatch( // (promoted on completion). Gated sync_start staging relies on this — it passes // to_pending=true so every busy core opts into a pending slot; the non-zero // src_payload gate keeps the whole cohort waiting for the rendezvous. - if (cmask & PTO2_SUBTASK_MASK_AIC) { + if (cmask & SUBTASK_MASK_AIC) { bool p = to_pending && !tracker.is_aic_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aic_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIC, p, block_idx, + thread_idx, tracker.get_aic_core_offset(core_offset), slot_state, SubtaskSlot::AIC, p, block_idx, force_gate ); } - if (cmask & PTO2_SUBTASK_MASK_AIV0) { + if (cmask & SUBTASK_MASK_AIV0) { bool p = to_pending && !tracker.is_aiv0_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aiv0_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIV0, p, block_idx, + thread_idx, tracker.get_aiv0_core_offset(core_offset), slot_state, SubtaskSlot::AIV0, p, block_idx, force_gate ); } - if (cmask & PTO2_SUBTASK_MASK_AIV1) { + if (cmask & SUBTASK_MASK_AIV1) { bool p = to_pending && !tracker.is_aiv1_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aiv1_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIV1, p, block_idx, + thread_idx, tracker.get_aiv1_core_offset(core_offset), slot_state, SubtaskSlot::AIV1, p, block_idx, force_gate ); } @@ -289,7 +289,7 @@ int SchedulerContext::prepare_block_for_dispatch( return n; } else if (shape == ResourceShape::AIC) { out_handles[0] = prepare_subtask_to_core( - thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIC, to_pending, block_idx, force_gate + thread_idx, core_offset, slot_state, SubtaskSlot::AIC, to_pending, block_idx, force_gate ); #if SIMPLER_DFX sched_chip_swimlane_[thread_idx].phase_dispatch_count += 1; @@ -297,7 +297,7 @@ int SchedulerContext::prepare_block_for_dispatch( return 1; } else { out_handles[0] = prepare_subtask_to_core( - thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIV0, to_pending, block_idx, force_gate + thread_idx, core_offset, slot_state, SubtaskSlot::AIV0, to_pending, block_idx, force_gate ); #if SIMPLER_DFX sched_chip_swimlane_[thread_idx].phase_dispatch_count += 1; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h index e83f92cbe7..a0b3f1d9fd 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h @@ -100,8 +100,8 @@ struct alignas(64) CoreExecState { int32_t running_reg_task_id; // offset 24: register task ID (AICPU_TASK_INVALID = idle) int32_t pending_reg_task_id; // offset 28: pending register task ID (AICPU_TASK_INVALID = none) uint32_t dispatch_seq; // offset 32: monotonic dispatch counter - PTO2SubtaskSlot running_subslot; // offset 36: which subtask slot is running - PTO2SubtaskSlot pending_subslot; // offset 37: which subtask slot is pending + SubtaskSlot running_subslot; // offset 36: which subtask slot is running + SubtaskSlot pending_subslot; // offset 37: which subtask slot is pending uint8_t pad0_[2]; // offset 38: alignment padding // Precomputed COND register pointer; resolved once in handshake so the // hot completion poll does a single volatile load instead of recomputing @@ -346,13 +346,13 @@ class alignas(64) CoreTracker { // slot) or the mask is empty. MixPlacement classify_mix_cluster(int32_t cluster_offset, uint8_t core_mask) const { BitStates used; - if (core_mask & PTO2_SUBTASK_MASK_AIC) { + if (core_mask & SUBTASK_MASK_AIC) { used |= BitStates::bit(cluster_offset); } - if (core_mask & PTO2_SUBTASK_MASK_AIV0) { + if (core_mask & SUBTASK_MASK_AIV0) { used |= BitStates::bit(cluster_offset + 1); } - if (core_mask & PTO2_SUBTASK_MASK_AIV1) { + if (core_mask & SUBTASK_MASK_AIV1) { used |= BitStates::bit(cluster_offset + 2); } if (!used.has_value() || (pending_occupied_ & used).has_value()) { @@ -392,9 +392,9 @@ class alignas(64) CoreTracker { // Cores of `cluster_offset` named by core_mask. BitStates mix_used_cores(int32_t cluster_offset, uint8_t core_mask) const { BitStates used; - if (core_mask & PTO2_SUBTASK_MASK_AIC) used |= BitStates::bit(cluster_offset); - if (core_mask & PTO2_SUBTASK_MASK_AIV0) used |= BitStates::bit(cluster_offset + 1); - if (core_mask & PTO2_SUBTASK_MASK_AIV1) used |= BitStates::bit(cluster_offset + 2); + if (core_mask & SUBTASK_MASK_AIC) used |= BitStates::bit(cluster_offset); + if (core_mask & SUBTASK_MASK_AIV0) used |= BitStates::bit(cluster_offset + 1); + if (core_mask & SUBTASK_MASK_AIV1) used |= BitStates::bit(cluster_offset + 2); return used; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h index e0f01abe0d..fb5feb3abc 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h @@ -25,12 +25,12 @@ inline constexpr int32_t INVALID_KERNEL_ID = -1; /** * Subtask slot count: AIC, AIV0, AIV1 */ -inline constexpr int32_t PTO2_SUBTASK_SLOT_COUNT = 3; +inline constexpr int32_t SUBTASK_SLOT_COUNT = 3; /** * Subtask slot indices */ -enum class PTO2SubtaskSlot : uint8_t { +enum class SubtaskSlot : uint8_t { AIC = 0, AIV0 = 1, AIV1 = 2, @@ -39,9 +39,9 @@ enum class PTO2SubtaskSlot : uint8_t { /** * Subtask mask bits (for ActiveMask) */ -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIC = (1u << 0); // 0x1 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 +inline constexpr uint8_t SUBTASK_MASK_AIC = (1u << 0); // 0x1 +inline constexpr uint8_t SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 +inline constexpr uint8_t SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 // Dispatch-predicate comparison operator. The scheduler evaluates the predicate // at the dispatch point — the task is ready (fanin satisfied), so the predicate @@ -132,7 +132,7 @@ class ActiveMask { uint8_t raw() const { return raw_; } - bool subtask_active(PTO2SubtaskSlot slot) const { return (raw_ & (1u << static_cast(slot))) != 0; } + bool subtask_active(SubtaskSlot slot) const { return (raw_ & (1u << static_cast(slot))) != 0; } uint8_t core_mask() const { return raw_ & 0x07u; } @@ -141,7 +141,7 @@ class ActiveMask { if (cmask == 0) return ResourceShape::DUMMY; int bit_count = __builtin_popcount(cmask); if (bit_count >= 2) return ResourceShape::MIX; - if (cmask & PTO2_SUBTASK_MASK_AIC) return ResourceShape::AIC; + if (cmask & SUBTASK_MASK_AIC) return ResourceShape::AIC; return ResourceShape::AIV; } @@ -240,9 +240,9 @@ struct MixedKernels { ActiveMask to_active_mask() const { uint8_t mask = 0; - if (aic_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIC; - if (aiv0_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIV0; - if (aiv1_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIV1; + if (aic_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIC; + if (aiv0_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIV0; + if (aiv1_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIV1; return ActiveMask(mask); } }; diff --git a/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp b/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp index 61da824b68..968b97640b 100644 --- a/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp +++ b/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp @@ -27,13 +27,13 @@ typedef void (*UnifiedKernelFunc)(__gm__ int64_t *); /** - * Execute task from PTO2DispatchPayload. + * Execute task from DispatchPayload. * * Reads function_bin_addr and args from the dispatch payload. * - * @param payload Pointer to PTO2DispatchPayload in global memory + * @param payload Pointer to DispatchPayload in global memory */ -__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ PTO2DispatchPayload *payload) { +__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ DispatchPayload *payload) { if (payload == nullptr || payload->function_bin_addr == 0) { return; } @@ -49,7 +49,7 @@ __aicore__ __attribute__((always_inline)) static void execute_task(__gm__ PTO2Di * Implements the AICPU-AICore register-based dispatch protocol: * 1. Report physical core ID and core type, signal aicore_done (no AICPU wait) * 2. Wait for the AICPU to open our register window (DATA_MAIN_BASE != 0) - * 3. Cache per-core PTO2DispatchPayload pointer from my_hank->task + * 3. Cache per-core DispatchPayload pointer from my_hank->task * 4. Poll DATA_MAIN_BASE register for task dispatch until exit signal * * AICore reports on launch; the AICPU writes &s_payload_per_core[i] to @@ -101,7 +101,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // above cannot clobber it) and before opening the window; dcci to read its // fresh value here. dcci(my_hank, SINGLE_CACHE_LINE); - __gm__ PTO2DispatchPayload *payload = reinterpret_cast<__gm__ PTO2DispatchPayload *>(my_hank->task); + __gm__ DispatchPayload *payload = reinterpret_cast<__gm__ DispatchPayload *>(my_hank->task); uint32_t enable_profiling_flag = get_aicore_profiling_flag(); bool chip_swimlane_enabled = SIMPLER_GET_DFX_FLAG(enable_profiling_flag, SIMPLER_DFX_FLAG_CHIP_SWIMLANE); @@ -166,7 +166,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in uint32_t task_id = reg_val; // Decode: register holds task_id directly // Select dual-buffer slot: same bit as AICPU used when writing payload - __gm__ PTO2DispatchPayload *exec_payload = payload + (task_id & 1u); + __gm__ DispatchPayload *exec_payload = payload + (task_id & 1u); // Invalidate payload buffer (AICPU updates its content each dispatch) dcci(exec_payload, ENTIRE_DATA_CACHE); diff --git a/src/a5/runtime/host_build_graph/build_config.py b/src/a5/runtime/host_build_graph/build_config.py index 8caf89cd3a..40b6524dd3 100644 --- a/src/a5/runtime/host_build_graph/build_config.py +++ b/src/a5/runtime/host_build_graph/build_config.py @@ -16,7 +16,7 @@ # the image to device (vs tensormap, where AICPU thread N-1 orchestrates on # device concurrently with the scheduler threads) # - AICPU threads 0..N-1 all run schedulers (no on-device orchestrator thread) -# - AICore executes tasks via an aligned PTO2DispatchPayload + pre-built dispatch_args +# - AICore executes tasks via an aligned DispatchPayload + pre-built dispatch_args # # The "orchestration" directory contains source files compiled into both # runtime targets AND the orchestration .so (e.g., tensor methods needed diff --git a/src/a5/runtime/host_build_graph/common/intrinsic.h b/src/a5/runtime/host_build_graph/common/intrinsic.h index 72bb241374..8dcd4ca5fd 100644 --- a/src/a5/runtime/host_build_graph/common/intrinsic.h +++ b/src/a5/runtime/host_build_graph/common/intrinsic.h @@ -106,7 +106,7 @@ static constexpr int32_t PAYLOAD_LOCAL_CONTEXT_INDEX = SPMD_LOCAL_CONTEXT_INDEX; static constexpr int32_t PAYLOAD_GLOBAL_CONTEXT_INDEX = SPMD_GLOBAL_CONTEXT_INDEX; /** - * Per-core global context, stored in PTO2DispatchPayload. + * Per-core global context, stored in DispatchPayload. * Initialized once at runtime startup (init_global_context) based on each * core's cluster position. Never modified after initialization. */ @@ -140,7 +140,7 @@ struct AsyncCtx { }; /** - * Per-dispatch local context, stored in PTO2DispatchPayload. + * Per-dispatch local context, stored in DispatchPayload. * Written by build_payload() before each dispatch. Different blocks of the * same task receive different block_idx values but the same block_num. * diff --git a/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h b/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h index 554e797d17..13169f89bc 100644 --- a/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h +++ b/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h @@ -13,7 +13,7 @@ * @file dispatch_payload.h * @brief Per-core dispatch payload for AICore kernel execution * - * PTO2DispatchPayload holds the kernel function address, a per-core args[] + * DispatchPayload holds the kernel function address, a per-core args[] * array, and embedded SPMD context (LocalContext + GlobalContext). AICPU * maintains a static array of these (one per core). * @@ -42,8 +42,8 @@ #include "intrinsic.h" /** Max dispatch arguments: 16 scalars + up to 32 tensor pointers + ext params */ -#ifndef PTO2_DISPATCH_MAX_ARGS -#define PTO2_DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) +#ifndef DISPATCH_MAX_ARGS +#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) #endif #ifndef PTO2_ALIGN_UP @@ -86,7 +86,7 @@ constexpr uint32_t TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) * The struct is cache-line aligned to prevent false sharing across * concurrently dispatched cores. */ -struct alignas(64) PTO2DispatchPayload { +struct alignas(64) DispatchPayload { // === Cache line 0 (64B): control block, the only line written per dispatch === // function_bin_addr, local_context.{block_idx,block_num,async_ctx.task_token} // and src_payload are the per-dispatch writes; async_ctx's slab pointers + @@ -114,7 +114,7 @@ struct alignas(64) PTO2DispatchPayload { * = &local_context, [SPMD_GLOBAL_CONTEXT_INDEX] = &global_context (both prefilled * once at init). On the gated path the AICPU leaves [0..num_args) unwritten and * the idle AICore fills them from src_payload during its gate wait. */ - uint64_t args[PTO2_DISPATCH_MAX_ARGS]; + uint64_t args[DISPATCH_MAX_ARGS]; /** Per-core global context: sub_block_id (AIV lane identity). Cold: written once * at init, never per dispatch, so it lives in the tail (not the CL0 control @@ -129,6 +129,6 @@ struct alignas(64) PTO2DispatchPayload { ); }; -static_assert(sizeof(PTO2DispatchPayload) == 512, "PTO2DispatchPayload hardware ABI size drift"); -static_assert(offsetof(PTO2DispatchPayload, args) == 64, "args[] must start at cache line 1 (control block = CL0)"); -static_assert(offsetof(PTO2DispatchPayload, src_payload) < 64, "src_payload (gate) must live on the CL0 control block"); +static_assert(sizeof(DispatchPayload) == 512, "DispatchPayload hardware ABI size drift"); +static_assert(offsetof(DispatchPayload, args) == 64, "args[] must start at cache line 1 (control block = CL0)"); +static_assert(offsetof(DispatchPayload, src_payload) < 64, "src_payload (gate) must live on the CL0 control block"); diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index ea9fefd4e1..f1d47ae637 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -315,7 +315,7 @@ struct GraphRecordedPredicate { }; struct GraphRecordedNode { - std::array kernel_ids{}; + std::array kernel_ids{}; ActiveMask active_mask{}; TaskAttrs task_attrs{}; int16_t logical_block_num{1}; @@ -1473,7 +1473,7 @@ static TaskOutputTensors submit_task_common( // fanout now that the swimlane hot path no longer records it. const bool capture_dep_graph = dep_gen_host_graph_enabled(); if (capture_dep_graph) { - const std::array kernel_ids_capture{ + const std::array kernel_ids_capture{ aic_kernel_id, aiv0_kernel_id, aiv1_kernel_id, @@ -1566,9 +1566,9 @@ static TaskOutputTensors submit_task_common( // evicted by TensorMap lookup/insert cache pressure. __builtin_prefetch(&task, 1, 1); task.task_id = task_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = aic_kernel_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = aiv0_kernel_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = aiv1_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIC)] = aic_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIV0)] = aiv0_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIV1)] = aiv1_kernel_id; task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; @@ -1860,7 +1860,7 @@ bool graph_submit_outer( // the args it consumes and the edges those produce. const bool capture_dep_graph = dep_gen_host_graph_enabled(); if (capture_dep_graph) { - const std::array kernel_ids_capture{ + const std::array kernel_ids_capture{ INVALID_KERNEL_ID, INVALID_KERNEL_ID, INVALID_KERNEL_ID, @@ -2001,9 +2001,9 @@ TaskOutputTensors graph_record_submit_node( if (node_index >= recording.nodes.size()) recording.nodes.emplace_back(); GraphRecordedNode &node = recording.nodes[node_index]; node.reset(); - node.kernel_ids[static_cast(PTO2SubtaskSlot::AIC)] = aic_kernel_id; - node.kernel_ids[static_cast(PTO2SubtaskSlot::AIV0)] = aiv0_kernel_id; - node.kernel_ids[static_cast(PTO2SubtaskSlot::AIV1)] = aiv1_kernel_id; + node.kernel_ids[static_cast(SubtaskSlot::AIC)] = aic_kernel_id; + node.kernel_ids[static_cast(SubtaskSlot::AIV0)] = aiv0_kernel_id; + node.kernel_ids[static_cast(SubtaskSlot::AIV1)] = aiv1_kernel_id; node.active_mask = active_mask; node.task_attrs = task_attrs; node.task_attrs.set_early_resolve(false); @@ -2551,13 +2551,13 @@ TaskOutputTensors OrchestratorState::submit_task(const MixedKernels &mixed_kerne // Normalize single-AIV tasks: if only aiv1 is set (no aic, no aiv0), move // it to the aiv0 slot. This guarantees the dispatch path can always use - // PTO2SubtaskSlot::AIV0 for single-AIV shapes without inspecting active_mask. + // SubtaskSlot::AIV0 for single-AIV shapes without inspecting active_mask. // Mixed tasks (AIC+AIV) keep their original AIV identity so the correct // hardware channel (AIV0→AIC vs AIV1→AIC) is used at dispatch time. MixedKernels normalized = mixed_kernels; - bool has_aic = active_mask.has_mask(PTO2_SUBTASK_MASK_AIC); - bool has_aiv0 = active_mask.has_mask(PTO2_SUBTASK_MASK_AIV0); - bool has_aiv1 = active_mask.has_mask(PTO2_SUBTASK_MASK_AIV1); + bool has_aic = active_mask.has_mask(SUBTASK_MASK_AIC); + bool has_aiv0 = active_mask.has_mask(SUBTASK_MASK_AIV0); + bool has_aiv1 = active_mask.has_mask(SUBTASK_MASK_AIV1); if (!has_aic && has_aiv1 && !has_aiv0) { normalized.aiv0_kernel_id = normalized.aiv1_kernel_id; normalized.aiv1_kernel_id = INVALID_KERNEL_ID; @@ -2712,9 +2712,9 @@ TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { #endif task.task_id = prepared.task_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = INVALID_KERNEL_ID; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = INVALID_KERNEL_ID; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIC)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIV0)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIV1)] = INVALID_KERNEL_ID; task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; diff --git a/src/a5/runtime/host_build_graph/runtime/runtime.h b/src/a5/runtime/host_build_graph/runtime/runtime.h index 556a16b18a..6449c56591 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime.h @@ -20,7 +20,7 @@ * - Device orchestration state (gm_sm_ptr_, orch_args_) * - Function address mapping (func_id_to_addr_) * - * Task dispatch uses a per-core PTO2DispatchPayload written by the scheduler. + * Task dispatch uses a per-core DispatchPayload written by the scheduler. * At dispatch time, build_payload() copies tensor pointers and scalars from * the task payload into the per-core args[], populates SPMD context, then * signals AICore via DATA_MAIN_BASE. @@ -70,7 +70,7 @@ constexpr int RUNTIME_DEFAULT_READY_QUEUE_SHARDS = PLATFORM_MAX_AICPU_THREADS - * 2. AICPU publishes the task pointer and opens the register window with DATA_MAIN_BASE=IDLE * 3. AICore observes window-open, reports initial idle state, and reads the task pointer * 4. Task Dispatch: AICPU writes DATA_MAIN_BASE after updating the per-core payload - * 5. Task Execution: AICore reads the cached PTO2DispatchPayload and executes + * 5. Task Execution: AICore reads the cached DispatchPayload and executes * 6. Task Completion: AICore writes FIN to COND; AICPU observes completion * 7. Shutdown: AICPU writes the exit signal to DATA_MAIN_BASE; AICore exits * @@ -96,7 +96,7 @@ constexpr int RUNTIME_DEFAULT_READY_QUEUE_SHARDS = PLATFORM_MAX_AICPU_THREADS - struct Handshake { volatile uint32_t aicpu_ready; // Legacy layout field; unused by the current handshake volatile uint32_t aicore_done; // AICore ready signal: 0=not ready, core_id+1=ready - volatile uint64_t task; // PTO2DispatchPayload* published before register window-open + volatile uint64_t task; // DispatchPayload* published before register window-open volatile CoreType core_type; // Core type: CoreType::AIC or CoreType::AIV (reported by AICore with aicore_done) volatile uint32_t physical_core_id; // Physical core ID (reported by AICore with aicore_done) } __attribute__((aligned(64))); @@ -118,7 +118,7 @@ struct TensorPair { /** * Task structure - Compatibility stub for platform layer * - * RT2 uses PTO2DispatchPayload instead of Task for task dispatch. + * RT2 uses DispatchPayload instead of Task for task dispatch. * This stub exists only for API compatibility with device_runner.cpp. * Since get_task_count() returns 0, this struct is never actually used. */ @@ -295,7 +295,7 @@ class Runtime { /** @deprecated Task count is now in PTO2 shared memory */ int get_task_count() const { return 0; } - /** @deprecated RT2 uses PTO2DispatchPayload, not Task. Always returns nullptr. */ + /** @deprecated RT2 uses DispatchPayload, not Task. Always returns nullptr. */ Task *get_task(int) { return nullptr; } // Host-side tensor ledger for D2H copy-back at finalize. Populated by diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 57aadc907d..8e71c2553b 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -39,9 +39,9 @@ // dispatch_payload.h include is intentionally dropped here. This header is // pulled in by the platform's args_dump.h via a hardcoded // "host_build_graph/runtime/runtime_types.h" path, and dispatch_payload.h -// uses #pragma once (path-keyed), so leaving it in double-defines PTO2DispatchPayload +// uses #pragma once (path-keyed), so leaving it in double-defines DispatchPayload // against tensormap_and_ringbuffer's copy inside the shared host-dispatcher TU. -// runtime_types.h never references PTO2DispatchPayload itself; consumers that +// runtime_types.h never references DispatchPayload itself; consumers that // need it include it via runtime.h directly. #include "aicore_completion_mailbox.h" #include "common/args_dump_task_metadata.h" @@ -238,7 +238,7 @@ struct TaskDescriptor { TaskId task_id; // raw: (ring_id << 32) | local_id // Per-slot kernel IDs (INVALID_KERNEL_ID = inactive) - int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT]; + int32_t kernel_id[SUBTASK_SLOT_COUNT]; // Packed output buffer (all outputs packed into single contiguous buffer) void *packed_buffer_base; // Start of packed buffer in GM Heap diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index c08161b4dc..8956105da6 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -384,13 +384,13 @@ int32_t SchedulerContext::handle_timeout_exit( // Capture the in-flight kernels' partial output before signalling the // cores to exit, so the dump reflects the live stuck state. if (is_dump_args_enabled()) { - dump_running_task_outputs( + dump_running_task_outputs( thread_idx, cores_total_num_, [this](int32_t cid) { return core_exec_states_[cid].running_slot_state; }, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); @@ -958,7 +958,7 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // live on a later line). for (int32_t core_id = 0; core_id < RUNTIME_MAX_WORKER; core_id++) { for (int32_t buf = 0; buf < 2; buf++) { - PTO2DispatchPayload &dp = payload_per_core_[core_id][buf]; + DispatchPayload &dp = payload_per_core_[core_id][buf]; AsyncCtx &ac = dp.local_context.async_ctx; volatile DeferredCompletionSlab *slab = &deferred_slab_per_core_[core_id][buf]; ac.completion_count = &slab->count; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index 7612353f24..99cecc119d 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -81,7 +81,7 @@ SlotTransition SchedulerContext::decide_slot_transition( // Complete one slot's task: subtask counting, mixed completion, deferred release, profiling. void SchedulerContext::complete_slot_task( - ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, [[maybe_unused]] PTO2SubtaskSlot subslot, + ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, [[maybe_unused]] SubtaskSlot subslot, int32_t thread_idx, int32_t core_id, Handshake *hank, [[maybe_unused]] int32_t &completed_this_turn #if SIMPLER_DFX , @@ -174,10 +174,10 @@ void SchedulerContext::complete_slot_task( if (task_complete && !defer_completion_to_consumer) { #if SIMPLER_DFX if (is_dump_args_enabled()) { - dump_args_for_task( + dump_args_for_task( thread_idx, slot_state, ArgsDumpStage::AFTER_COMPLETION, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index 19d2265309..de8c5fdf41 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -199,7 +199,7 @@ class SchedulerContext { // Per-core dispatch payload storage: dual-buffer for pipelining. // buf_idx = reg_task_id & 1; adjacent dispatches alternate automatically. - PTO2DispatchPayload payload_per_core_[RUNTIME_MAX_WORKER][2]; + DispatchPayload payload_per_core_[RUNTIME_MAX_WORKER][2]; // Per-core deferred-completion software registration storage. This has // the same runtime lifetime as payload_per_core_, but is kept out of the @@ -283,16 +283,16 @@ class SchedulerContext { static const char *shape_name(ResourceShape shape); - // Lower-case rendering of PTO2SubtaskSlot, used by dispatch and stall logs. + // Lower-case rendering of SubtaskSlot, used by dispatch and stall logs. // Kept lower-case to match the `kernels=[aic:N aiv0:N aiv1:N]` field // convention already established in the stall log family. - static inline const char *subslot_name(PTO2SubtaskSlot s) { + static inline const char *subslot_name(SubtaskSlot s) { switch (s) { - case PTO2SubtaskSlot::AIC: + case SubtaskSlot::AIC: return "aic"; - case PTO2SubtaskSlot::AIV0: + case SubtaskSlot::AIV0: return "aiv0"; - case PTO2SubtaskSlot::AIV1: + case SubtaskSlot::AIV1: return "aiv1"; } return "?"; @@ -303,8 +303,8 @@ class SchedulerContext { ); void build_payload( - PTO2DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, - int32_t block_idx, bool force_gate + DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, SubtaskSlot subslot, int32_t block_idx, + bool force_gate ); // Batched-dispatch primitives. prepare_* builds the payload and per-core @@ -325,8 +325,8 @@ class SchedulerContext { }; PublishHandle prepare_subtask_to_core( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, - bool to_pending, int32_t block_idx, bool force_gate + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, SubtaskSlot subslot, bool to_pending, + int32_t block_idx, bool force_gate ); // `thread_idx` is the publishing Scheduler thread's index, used to select the @@ -482,7 +482,7 @@ class SchedulerContext { ); void complete_slot_task( - ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, PTO2SubtaskSlot subslot, int32_t thread_idx, + ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, SubtaskSlot subslot, int32_t thread_idx, int32_t core_id, Handshake *hank, int32_t &completed_this_turn #if SIMPLER_DFX , diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index d0d0db44f5..61f5ca99dc 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -120,7 +120,7 @@ int SchedulerContext::pop_ready_tasks_batch( } void SchedulerContext::build_payload( - PTO2DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, int32_t block_idx, + DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, SubtaskSlot subslot, int32_t block_idx, bool force_gate ) { int32_t slot_idx = static_cast(subslot); @@ -168,7 +168,7 @@ void SchedulerContext::build_payload( } SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, bool to_pending, + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, SubtaskSlot subslot, bool to_pending, int32_t block_idx, bool force_gate ) { CoreTracker &tracker = core_trackers_[thread_idx]; @@ -186,7 +186,7 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( } uint32_t buf_idx = reg_task_id & 1u; - PTO2DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; + DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; // The deferred_slab is NOT reset here: it is init-cleared once and the // completion path re-clears count only after a task actually recorded a // deferred completion (count > 0), which is rare. A non-deferred task never @@ -246,10 +246,10 @@ int SchedulerContext::prepare_block_for_dispatch( ) { #if SIMPLER_DFX if (is_dump_args_enabled()) { - dump_args_for_task( + dump_args_for_task( thread_idx, slot_state, ArgsDumpStage::BEFORE_DISPATCH, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); @@ -267,24 +267,24 @@ int SchedulerContext::prepare_block_for_dispatch( // (promoted on completion). Gated sync_start staging relies on this — it passes // to_pending=true so every busy core opts into a pending slot; the non-zero // src_payload gate keeps the whole cohort waiting for the rendezvous. - if (cmask & PTO2_SUBTASK_MASK_AIC) { + if (cmask & SUBTASK_MASK_AIC) { bool p = to_pending && !tracker.is_aic_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aic_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIC, p, block_idx, + thread_idx, tracker.get_aic_core_offset(core_offset), slot_state, SubtaskSlot::AIC, p, block_idx, force_gate ); } - if (cmask & PTO2_SUBTASK_MASK_AIV0) { + if (cmask & SUBTASK_MASK_AIV0) { bool p = to_pending && !tracker.is_aiv0_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aiv0_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIV0, p, block_idx, + thread_idx, tracker.get_aiv0_core_offset(core_offset), slot_state, SubtaskSlot::AIV0, p, block_idx, force_gate ); } - if (cmask & PTO2_SUBTASK_MASK_AIV1) { + if (cmask & SUBTASK_MASK_AIV1) { bool p = to_pending && !tracker.is_aiv1_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aiv1_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIV1, p, block_idx, + thread_idx, tracker.get_aiv1_core_offset(core_offset), slot_state, SubtaskSlot::AIV1, p, block_idx, force_gate ); } @@ -294,7 +294,7 @@ int SchedulerContext::prepare_block_for_dispatch( return n; } else if (shape == ResourceShape::AIC) { out_handles[0] = prepare_subtask_to_core( - thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIC, to_pending, block_idx, force_gate + thread_idx, core_offset, slot_state, SubtaskSlot::AIC, to_pending, block_idx, force_gate ); #if SIMPLER_DFX sched_chip_swimlane_[thread_idx].phase_dispatch_count += 1; @@ -302,7 +302,7 @@ int SchedulerContext::prepare_block_for_dispatch( return 1; } else { out_handles[0] = prepare_subtask_to_core( - thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIV0, to_pending, block_idx, force_gate + thread_idx, core_offset, slot_state, SubtaskSlot::AIV0, to_pending, block_idx, force_gate ); #if SIMPLER_DFX sched_chip_swimlane_[thread_idx].phase_dispatch_count += 1; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h index 367ae824eb..7ace4472ad 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_types.h @@ -111,8 +111,8 @@ struct alignas(64) CoreExecState { int32_t running_reg_task_id; // offset 24: register task ID (AICPU_TASK_INVALID = idle) int32_t pending_reg_task_id; // offset 28: pending register task ID (AICPU_TASK_INVALID = none) uint32_t dispatch_seq; // offset 32: monotonic dispatch counter - PTO2SubtaskSlot running_subslot; // offset 36: which subtask slot is running - PTO2SubtaskSlot pending_subslot; // offset 37: which subtask slot is pending + SubtaskSlot running_subslot; // offset 36: which subtask slot is running + SubtaskSlot pending_subslot; // offset 37: which subtask slot is pending uint8_t pad0_[2]; // offset 38: alignment padding // Precomputed COND register pointer; resolved once in handshake so the // hot completion poll does a single volatile load instead of recomputing @@ -370,13 +370,13 @@ class alignas(64) CoreTracker { // slot) or the mask is empty. MixPlacement classify_mix_cluster(int32_t cluster_offset, uint8_t core_mask) const { BitStates used; - if (core_mask & PTO2_SUBTASK_MASK_AIC) { + if (core_mask & SUBTASK_MASK_AIC) { used |= BitStates::bit(cluster_offset); } - if (core_mask & PTO2_SUBTASK_MASK_AIV0) { + if (core_mask & SUBTASK_MASK_AIV0) { used |= BitStates::bit(cluster_offset + 1); } - if (core_mask & PTO2_SUBTASK_MASK_AIV1) { + if (core_mask & SUBTASK_MASK_AIV1) { used |= BitStates::bit(cluster_offset + 2); } if (!used.has_value() || (pending_occupied_ & used).has_value()) { @@ -412,9 +412,9 @@ class alignas(64) CoreTracker { // Cores of `cluster_offset` named by core_mask. BitStates mix_used_cores(int32_t cluster_offset, uint8_t core_mask) const { BitStates used; - if (core_mask & PTO2_SUBTASK_MASK_AIC) used |= BitStates::bit(cluster_offset); - if (core_mask & PTO2_SUBTASK_MASK_AIV0) used |= BitStates::bit(cluster_offset + 1); - if (core_mask & PTO2_SUBTASK_MASK_AIV1) used |= BitStates::bit(cluster_offset + 2); + if (core_mask & SUBTASK_MASK_AIC) used |= BitStates::bit(cluster_offset); + if (core_mask & SUBTASK_MASK_AIV0) used |= BitStates::bit(cluster_offset + 1); + if (core_mask & SUBTASK_MASK_AIV1) used |= BitStates::bit(cluster_offset + 2); return used; } diff --git a/src/a5/runtime/host_build_graph/runtime/submit_types.h b/src/a5/runtime/host_build_graph/runtime/submit_types.h index daeabb2776..21756a3e8b 100644 --- a/src/a5/runtime/host_build_graph/runtime/submit_types.h +++ b/src/a5/runtime/host_build_graph/runtime/submit_types.h @@ -25,12 +25,12 @@ inline constexpr int32_t INVALID_KERNEL_ID = -1; /** * Subtask slot count: AIC, AIV0, AIV1 */ -inline constexpr int32_t PTO2_SUBTASK_SLOT_COUNT = 3; +inline constexpr int32_t SUBTASK_SLOT_COUNT = 3; /** * Subtask slot indices */ -enum class PTO2SubtaskSlot : uint8_t { +enum class SubtaskSlot : uint8_t { AIC = 0, AIV0 = 1, AIV1 = 2, @@ -39,9 +39,9 @@ enum class PTO2SubtaskSlot : uint8_t { /** * Subtask mask bits (for ActiveMask) */ -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIC = (1u << 0); // 0x1 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 +inline constexpr uint8_t SUBTASK_MASK_AIC = (1u << 0); // 0x1 +inline constexpr uint8_t SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 +inline constexpr uint8_t SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 // Dispatch-predicate comparison operator. The scheduler evaluates the predicate // at the dispatch point — the task is ready (fanin satisfied), so the predicate @@ -132,7 +132,7 @@ class ActiveMask { uint8_t raw() const { return raw_; } - bool subtask_active(PTO2SubtaskSlot slot) const { return (raw_ & (1u << static_cast(slot))) != 0; } + bool subtask_active(SubtaskSlot slot) const { return (raw_ & (1u << static_cast(slot))) != 0; } uint8_t core_mask() const { return raw_ & 0x07u; } @@ -141,7 +141,7 @@ class ActiveMask { if (cmask == 0) return ResourceShape::DUMMY; int bit_count = __builtin_popcount(cmask); if (bit_count >= 2) return ResourceShape::MIX; - if (cmask & PTO2_SUBTASK_MASK_AIC) return ResourceShape::AIC; + if (cmask & SUBTASK_MASK_AIC) return ResourceShape::AIC; return ResourceShape::AIV; } @@ -244,9 +244,9 @@ struct MixedKernels { ActiveMask to_active_mask() const { uint8_t mask = 0; - if (aic_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIC; - if (aiv0_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIV0; - if (aiv1_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIV1; + if (aic_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIC; + if (aiv0_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIV0; + if (aiv1_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIV1; return ActiveMask(mask); } }; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp b/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp index 173bfa9775..803227de2a 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp @@ -28,13 +28,13 @@ typedef void (*UnifiedKernelFunc)(__gm__ int64_t *); /** - * Execute task from PTO2DispatchPayload. + * Execute task from DispatchPayload. * * Reads function_bin_addr and args from the dispatch payload. * - * @param payload Pointer to PTO2DispatchPayload in global memory + * @param payload Pointer to DispatchPayload in global memory */ -__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ PTO2DispatchPayload *payload) { +__aicore__ __attribute__((always_inline)) static void execute_task(__gm__ DispatchPayload *payload) { if (payload == nullptr || payload->function_bin_addr == 0) { return; } @@ -50,7 +50,7 @@ __aicore__ __attribute__((always_inline)) static void execute_task(__gm__ PTO2Di * Implements the AICPU-AICore register-based dispatch protocol: * 1. Report physical core ID and core type, signal aicore_done (no AICPU wait) * 2. Wait for the AICPU to open our register window (DATA_MAIN_BASE != 0) - * 3. Cache per-core PTO2DispatchPayload pointer from my_hank->task + * 3. Cache per-core DispatchPayload pointer from my_hank->task * 4. Poll DATA_MAIN_BASE register for task dispatch until exit signal * * AICore reports on launch; the AICPU writes &s_payload_per_core[i] to @@ -98,7 +98,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // above cannot clobber it) and before opening the window; dcci to read its // fresh value here. dcci(my_hank, SINGLE_CACHE_LINE); - __gm__ PTO2DispatchPayload *payload = reinterpret_cast<__gm__ PTO2DispatchPayload *>(my_hank->task); + __gm__ DispatchPayload *payload = reinterpret_cast<__gm__ DispatchPayload *>(my_hank->task); // Cache profiling state once after Phase 2. The L2 / PMU rings and the // PMU MMIO base are all stable for the entire run (host-resolved at @@ -156,7 +156,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in } // Select dual-buffer slot: same bit as AICPU used when writing payload - __gm__ PTO2DispatchPayload *exec_payload = payload + (task_id & 1u); + __gm__ DispatchPayload *exec_payload = payload + (task_id & 1u); // Invalidate payload buffer (AICPU updates its content each dispatch) dcci(exec_payload, ENTIRE_DATA_CACHE); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/build_config.py b/src/a5/runtime/tensormap_and_ringbuffer/build_config.py index 2cfba734d4..9641f85873 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/build_config.py +++ b/src/a5/runtime/tensormap_and_ringbuffer/build_config.py @@ -12,7 +12,7 @@ # This is a device-orchestration runtime where: # - AICPU thread 3 runs the orchestrator (builds task graph on device) # - AICPU threads 0/1/2 run schedulers (dispatch tasks to AICore) -# - AICore executes tasks via an aligned PTO2DispatchPayload + pre-built dispatch_args +# - AICore executes tasks via an aligned DispatchPayload + pre-built dispatch_args # # The "orchestration" directory contains source files compiled into both # runtime targets AND the orchestration .so (e.g., tensor methods needed diff --git a/src/a5/runtime/tensormap_and_ringbuffer/common/intrinsic.h b/src/a5/runtime/tensormap_and_ringbuffer/common/intrinsic.h index d33196f2cb..7d80e44cef 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/common/intrinsic.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/common/intrinsic.h @@ -110,7 +110,7 @@ static constexpr int32_t PAYLOAD_LOCAL_CONTEXT_INDEX = SPMD_LOCAL_CONTEXT_INDEX; static constexpr int32_t PAYLOAD_GLOBAL_CONTEXT_INDEX = SPMD_GLOBAL_CONTEXT_INDEX; /** - * Per-core global context, stored in PTO2DispatchPayload. + * Per-core global context, stored in DispatchPayload. * Initialized during scheduler cold start from each core's cluster position * and the resident per-device async-DMA config. Stable for that scheduler * instance; unavailable workspace slots remain zero. @@ -150,7 +150,7 @@ struct AsyncCtx { }; /** - * Per-dispatch local context, stored in PTO2DispatchPayload. + * Per-dispatch local context, stored in DispatchPayload. * Written by build_payload() before each dispatch. Different blocks of the * same task receive different s_block_idx values but the same s_block_num. * diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index d4642f0a5d..615e946db3 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -168,7 +168,7 @@ bool entry_valid(const ChipTensorMapEntry& e) { | `DepListEntry` | Stores `ChipTaskSlotState*` pointer — naturally crosses ring boundaries | | `TaskPayload` | `fanin_slot_states[]` are pointers — no ring coupling | | `ChipReadyQueue` | Global ready queues shared across all rings (tasks ready to dispatch regardless of origin ring) | -| `PTO2DispatchPayload` | Built per-dispatch, no ring state needed | +| `DispatchPayload` | Built per-dispatch, no ring state needed | ## 5. Reclamation diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 0007559d09..0ea43e79a7 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -571,7 +571,7 @@ Each scheduler thread runs a tight loop with two main phases: - Service each source (normal ready ▸ speculative early) in occupancy order — `sync_start` Tier-0 ▸ MIX ▸ AIC/AIV, idle ▸ pending — popping from the matching shape-based ready queue (lock-free MPMC Vyukov queue, one per resource shape) -- Build `PTO2DispatchPayload` from `TaskDescriptor` with `task_id`, `subslot`, `kernel_id`, and `core_type` +- Build `DispatchPayload` from `TaskDescriptor` with `task_id`, `subslot`, `kernel_id`, and `core_type` - Write task pointer to `Handshake.task`, signal AICore via register `DATA_MAIN_BASE` (DMB offset `0xD0` on a5) - After normal ready queues are empty, Phase **4b** may stage speculative early-dispatch candidates onto spare slots (`early_dispatch_queues[]` / `early_sync_start_queue`) @@ -745,7 +745,7 @@ Each AICore worker has a `Handshake` struct in shared memory: | Field | Direction | Purpose | | ----- | --------- | ------- | -| `task` | AICPU→AICore | Pointer to `PTO2DispatchPayload` | +| `task` | AICPU→AICore | Pointer to `DispatchPayload` | | `control` | AICPU→AICore | 0=normal, 1=shutdown | | `perf_records_addr` | AICPU→AICore | Performance buffer address | @@ -770,7 +770,7 @@ Instead of polling a shared-memory status flag, the production protocol uses har 5. Execute kernel function via `func_id_to_addr` lookup 6. Write FIN to `COND` -### 9.3 PTO2DispatchPayload +### 9.3 DispatchPayload Built by the scheduler from `TaskDescriptor`: @@ -801,7 +801,7 @@ Built by the scheduler from `TaskDescriptor`: and stores it in `Runtime.dev.func_id_to_addr_[child_func_id(i)]`. 4. When dispatching, the scheduler reads `func_id_to_addr_[fid]`, casts to `const CoreCallable*`, reads `resolved_addr_`, and copies that into - `PTO2DispatchPayload.function_bin_addr`. + `DispatchPayload.function_bin_addr`. ### 10.2 Orchestration SO Loading diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h index fa6b471f82..a147665ca4 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h @@ -13,7 +13,7 @@ * @file dispatch_payload.h * @brief Per-core dispatch payload for AICore kernel execution * - * PTO2DispatchPayload holds the kernel function address, a per-core args[] + * DispatchPayload holds the kernel function address, a per-core args[] * array, and embedded SPMD context (LocalContext + GlobalContext). AICPU * maintains a static array of these (one per core). * @@ -43,8 +43,8 @@ #include "intrinsic.h" /** Max dispatch arguments: 16 scalars + up to 32 tensor pointers + ext params */ -#ifndef PTO2_DISPATCH_MAX_ARGS -#define PTO2_DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) +#ifndef DISPATCH_MAX_ARGS +#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) #endif #ifndef PTO2_ALIGN_UP @@ -83,7 +83,7 @@ constexpr uint32_t TASKPAYLOAD_TENSOR_STRIDE = 128; // sizeof(ChipTensor) * The struct is cache-line aligned to prevent false sharing across * concurrently dispatched cores. */ -struct alignas(64) PTO2DispatchPayload { +struct alignas(64) DispatchPayload { // === Cache line 0 (64B): control block, the only line written per dispatch === // function_bin_addr, local_context.{s_block_idx,s_block_num,async_ctx.task_token} // and src_payload are the per-dispatch writes; async_ctx's slab pointers + @@ -111,7 +111,7 @@ struct alignas(64) PTO2DispatchPayload { * = &local_context, [SPMD_GLOBAL_CONTEXT_INDEX] = &global_context (both prefilled * once at init). On the gated path the AICPU leaves [0..num_args) unwritten and * the idle AICore fills them from src_payload during its gate wait. */ - uint64_t args[PTO2_DISPATCH_MAX_ARGS]; + uint64_t args[DISPATCH_MAX_ARGS]; /** Per-core global context: sub_block_id (AIV lane identity) plus optional * async-DMA workspace addresses. Cold: written during scheduler init, never @@ -127,6 +127,6 @@ struct alignas(64) PTO2DispatchPayload { ); }; -static_assert(sizeof(PTO2DispatchPayload) == 512, "PTO2DispatchPayload hardware ABI size drift"); -static_assert(offsetof(PTO2DispatchPayload, args) == 64, "args[] must start at cache line 1 (control block = CL0)"); -static_assert(offsetof(PTO2DispatchPayload, src_payload) < 64, "src_payload (gate) must live on the CL0 control block"); +static_assert(sizeof(DispatchPayload) == 512, "DispatchPayload hardware ABI size drift"); +static_assert(offsetof(DispatchPayload, args) == 64, "args[] must start at cache line 1 (control block = CL0)"); +static_assert(offsetof(DispatchPayload, src_payload) < 64, "src_payload (gate) must live on the CL0 control block"); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 38bce0d8f2..cebe9f4b84 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -1023,9 +1023,9 @@ static TaskOutputTensors submit_task_common( // evicted by TensorMap lookup/insert cache pressure. __builtin_prefetch(&task, 1, 1); task.task_id = task_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = aic_kernel_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = aiv0_kernel_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = aiv1_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIC)] = aic_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIV0)] = aiv0_kernel_id; + task.kernel_id[static_cast(SubtaskSlot::AIV1)] = aiv1_kernel_id; task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; @@ -1176,13 +1176,13 @@ TaskOutputTensors OrchestratorState::submit_task(const MixedKernels &mixed_kerne // Normalize single-AIV tasks: if only aiv1 is set (no aic, no aiv0), move // it to the aiv0 slot. This guarantees the dispatch path can always use - // PTO2SubtaskSlot::AIV0 for single-AIV shapes without inspecting active_mask. + // SubtaskSlot::AIV0 for single-AIV shapes without inspecting active_mask. // Mixed tasks (AIC+AIV) keep their original AIV identity so the correct // hardware channel (AIV0→AIC vs AIV1→AIC) is used at dispatch time. MixedKernels normalized = mixed_kernels; - bool has_aic = active_mask.has_mask(PTO2_SUBTASK_MASK_AIC); - bool has_aiv0 = active_mask.has_mask(PTO2_SUBTASK_MASK_AIV0); - bool has_aiv1 = active_mask.has_mask(PTO2_SUBTASK_MASK_AIV1); + bool has_aic = active_mask.has_mask(SUBTASK_MASK_AIC); + bool has_aiv0 = active_mask.has_mask(SUBTASK_MASK_AIV0); + bool has_aiv1 = active_mask.has_mask(SUBTASK_MASK_AIV1); if (!has_aic && has_aiv1 && !has_aiv0) { normalized.aiv0_kernel_id = normalized.aiv1_kernel_id; normalized.aiv1_kernel_id = INVALID_KERNEL_ID; @@ -1313,9 +1313,9 @@ TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { #endif task.task_id = prepared.task_id; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = INVALID_KERNEL_ID; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = INVALID_KERNEL_ID; - task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIC)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIV0)] = INVALID_KERNEL_ID; + task.kernel_id[static_cast(SubtaskSlot::AIV1)] = INVALID_KERNEL_ID; task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime.h index 8d27ef8608..ac78ded0ae 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime.h @@ -20,7 +20,7 @@ * - Device orchestration state (gm_sm_ptr_, orch_args_) * - Function address mapping (func_id_to_addr_) * - * Task dispatch uses a per-core PTO2DispatchPayload written by the scheduler. + * Task dispatch uses a per-core DispatchPayload written by the scheduler. * At dispatch time, build_payload() copies tensor pointers and scalars from * the task payload into the per-core args[], populates SPMD context, then * signals AICore via DATA_MAIN_BASE. @@ -72,7 +72,7 @@ constexpr int RUNTIME_DEFAULT_READY_QUEUE_SHARDS = PLATFORM_MAX_AICPU_THREADS - * 2. AICPU publishes the task pointer and opens the register window with DATA_MAIN_BASE=IDLE * 3. AICore observes window-open, reports initial idle state, and reads the task pointer * 4. Task Dispatch: AICPU writes DATA_MAIN_BASE after updating the per-core payload - * 5. Task Execution: AICore reads the cached PTO2DispatchPayload and executes + * 5. Task Execution: AICore reads the cached DispatchPayload and executes * 6. Task Completion: AICore writes FIN to COND; AICPU observes completion * 7. Shutdown: AICPU writes the exit signal to DATA_MAIN_BASE; AICore exits * @@ -105,7 +105,7 @@ constexpr int RUNTIME_DEFAULT_READY_QUEUE_SHARDS = PLATFORM_MAX_AICPU_THREADS - struct Handshake { volatile uint32_t aicpu_ready; // Legacy layout field; unused by the current handshake volatile uint32_t aicore_done; // AICore ready signal: 0=not ready, core_id+1=ready - volatile uint64_t task; // PTO2DispatchPayload* published before register window-open + volatile uint64_t task; // DispatchPayload* published before register window-open volatile CoreType core_type; // Core type: CoreType::AIC or CoreType::AIV (reported by AICore with aicore_done) volatile uint32_t physical_core_id; // Physical core ID (reported by AICore with aicore_done) } __attribute__((aligned(64))); @@ -133,7 +133,7 @@ struct TensorLease { /** * Task structure - Compatibility stub for platform layer * - * RT2 uses PTO2DispatchPayload instead of Task for task dispatch. + * RT2 uses DispatchPayload instead of Task for task dispatch. * This stub exists only for API compatibility with device_runner.cpp. * Since get_task_count() returns 0, this struct is never actually used. */ @@ -322,7 +322,7 @@ class Runtime { /** @deprecated Task count is now in PTO2 shared memory */ int get_task_count() const { return 0; } - /** @deprecated RT2 uses PTO2DispatchPayload, not Task. Always returns nullptr. */ + /** @deprecated RT2 uses DispatchPayload, not Task. Always returns nullptr. */ Task *get_task(int) { return nullptr; } // ========================================================================= diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index a61f201358..c662966e50 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -220,7 +220,7 @@ struct TaskDescriptor { TaskId task_id; // raw: (ring_id << 32) | local_id // Per-slot kernel IDs (INVALID_KERNEL_ID = inactive) - int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT]; + int32_t kernel_id[SUBTASK_SLOT_COUNT]; // Packed output buffer (all outputs packed into single contiguous buffer) void *packed_buffer_base; // Start of packed buffer in GM Heap diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index ac68db51c3..364b72adb9 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -457,13 +457,13 @@ int32_t SchedulerContext::handle_timeout_exit( // Capture the in-flight kernels' partial output before signalling the // cores to exit, so the dump reflects the live stuck state. if (is_dump_args_enabled()) { - dump_running_task_outputs( + dump_running_task_outputs( thread_idx, cores_total_num_, [this](int32_t cid) { return core_exec_states_[cid].running_slot_state; }, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); @@ -941,7 +941,7 @@ void SchedulerContext::assign_own_clusters(int32_t tidx) { for (int32_t sub = 0; sub < PLATFORM_CORES_PER_BLOCKDIM; sub++) { int32_t core_id = tracker.get_core_id_by_offset(c * PLATFORM_CORES_PER_BLOCKDIM + sub); for (int32_t buf = 0; buf < 2; buf++) { - PTO2DispatchPayload &dp = payload_per_core_[core_id][buf]; + DispatchPayload &dp = payload_per_core_[core_id][buf]; AsyncCtx &ac = dp.local_context.async_ctx; volatile DeferredCompletionSlab *slab = &deferred_slab_per_core_[core_id][buf]; ac.completion_count = &slab->count; @@ -1257,7 +1257,7 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // live on a later line). for (int32_t core_id = 0; core_id < RUNTIME_MAX_WORKER; core_id++) { for (int32_t buf = 0; buf < 2; buf++) { - PTO2DispatchPayload &dp = payload_per_core_[core_id][buf]; + DispatchPayload &dp = payload_per_core_[core_id][buf]; AsyncCtx &ac = dp.local_context.async_ctx; volatile DeferredCompletionSlab *slab = &deferred_slab_per_core_[core_id][buf]; ac.completion_count = &slab->count; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index 2a4965f81c..3085111aea 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -76,7 +76,7 @@ SlotTransition SchedulerContext::decide_slot_transition( // Complete one slot's task: subtask counting, mixed completion, deferred release, profiling. void SchedulerContext::complete_slot_task( - ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, [[maybe_unused]] PTO2SubtaskSlot subslot, + ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, [[maybe_unused]] SubtaskSlot subslot, [[maybe_unused]] int32_t thread_idx, int32_t core_id, Handshake *hank, int32_t &completed_this_turn, ChipTaskSlotState *deferred_release_slot_states[], int32_t &deferred_release_count #if SIMPLER_DFX @@ -161,10 +161,10 @@ void SchedulerContext::complete_slot_task( if (task_complete && !defer_completion_to_consumer) { #if SIMPLER_DFX if (is_dump_args_enabled()) { - dump_args_for_task( + dump_args_for_task( thread_idx, slot_state, ArgsDumpStage::AFTER_COMPLETION, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 7be805decb..32349f4797 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -153,7 +153,7 @@ class SchedulerContext { // Per-core dispatch payload storage: dual-buffer for pipelining. // buf_idx = reg_task_id & 1; adjacent dispatches alternate automatically. - PTO2DispatchPayload payload_per_core_[RUNTIME_MAX_WORKER][2]; + DispatchPayload payload_per_core_[RUNTIME_MAX_WORKER][2]; // Per-core deferred-completion software registration storage. This has // the same runtime lifetime as payload_per_core_, but is kept out of the @@ -231,16 +231,16 @@ class SchedulerContext { static const char *shape_name(ResourceShape shape); - // Lower-case rendering of PTO2SubtaskSlot, used by dispatch and stall logs. + // Lower-case rendering of SubtaskSlot, used by dispatch and stall logs. // Kept lower-case to match the `kernels=[aic:N aiv0:N aiv1:N]` field // convention already established in the stall log family. - static inline const char *subslot_name(PTO2SubtaskSlot s) { + static inline const char *subslot_name(SubtaskSlot s) { switch (s) { - case PTO2SubtaskSlot::AIC: + case SubtaskSlot::AIC: return "aic"; - case PTO2SubtaskSlot::AIV0: + case SubtaskSlot::AIV0: return "aiv0"; - case PTO2SubtaskSlot::AIV1: + case SubtaskSlot::AIV1: return "aiv1"; } return "?"; @@ -251,8 +251,8 @@ class SchedulerContext { ); void build_payload( - PTO2DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, - int32_t block_idx, bool force_gate = false + DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, SubtaskSlot subslot, int32_t block_idx, + bool force_gate = false ); // Batched-dispatch primitives. prepare_* builds the payload and per-core @@ -273,8 +273,8 @@ class SchedulerContext { }; PublishHandle prepare_subtask_to_core( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, - bool to_pending, int32_t block_idx, bool force_gate = false + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, SubtaskSlot subslot, bool to_pending, + int32_t block_idx, bool force_gate = false ); // `thread_idx` is the publishing Scheduler thread's index, used to select the @@ -373,7 +373,7 @@ class SchedulerContext { ); void complete_slot_task( - ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, PTO2SubtaskSlot subslot, int32_t thread_idx, + ChipTaskSlotState &slot_state, int32_t expected_reg_task_id, SubtaskSlot subslot, int32_t thread_idx, int32_t core_id, Handshake *hank, int32_t &completed_this_turn, ChipTaskSlotState *deferred_release_slot_states[], int32_t &deferred_release_count #if SIMPLER_DFX diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 4a80e8e97d..5f59a2c610 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -112,7 +112,7 @@ int SchedulerContext::pop_ready_tasks_batch( } void SchedulerContext::build_payload( - PTO2DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, int32_t block_idx, + DispatchPayload &dispatch_payload, ChipTaskSlotState &slot_state, SubtaskSlot subslot, int32_t block_idx, bool force_gate ) { int32_t slot_idx = static_cast(subslot); @@ -140,7 +140,7 @@ void SchedulerContext::build_payload( } SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( - int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, PTO2SubtaskSlot subslot, bool to_pending, + int32_t thread_idx, int32_t core_offset, ChipTaskSlotState &slot_state, SubtaskSlot subslot, bool to_pending, int32_t block_idx, bool force_gate ) { CoreTracker &tracker = core_trackers_[thread_idx]; @@ -158,7 +158,7 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( } uint32_t buf_idx = reg_task_id & 1u; - PTO2DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; + DispatchPayload &payload = payload_per_core_[core_id][buf_idx]; build_payload(payload, slot_state, subslot, block_idx, force_gate); if (to_pending) { @@ -211,10 +211,10 @@ int SchedulerContext::prepare_block_for_dispatch( ) { #if SIMPLER_DFX if (is_dump_args_enabled()) { - dump_args_for_task( + dump_args_for_task( thread_idx, slot_state, ArgsDumpStage::BEFORE_DISPATCH, [](ActiveMask active_mask, int raw_subtask_id) { - return active_mask.subtask_active(static_cast(raw_subtask_id)); + return active_mask.subtask_active(static_cast(raw_subtask_id)); }, [this](int32_t func_id) { return get_function_bin_addr(func_id); @@ -230,24 +230,24 @@ int SchedulerContext::prepare_block_for_dispatch( // running slot; busy used cores take pending when to_pending. Cluster // selection remains classify_mix_cluster (uniform RUNNING/PENDING), not // a2a3 gated MIX split. - if (cmask & PTO2_SUBTASK_MASK_AIC) { + if (cmask & SUBTASK_MASK_AIC) { bool p = to_pending && !tracker.is_aic_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aic_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIC, p, block_idx, + thread_idx, tracker.get_aic_core_offset(core_offset), slot_state, SubtaskSlot::AIC, p, block_idx, force_gate ); } - if (cmask & PTO2_SUBTASK_MASK_AIV0) { + if (cmask & SUBTASK_MASK_AIV0) { bool p = to_pending && !tracker.is_aiv0_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aiv0_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIV0, p, block_idx, + thread_idx, tracker.get_aiv0_core_offset(core_offset), slot_state, SubtaskSlot::AIV0, p, block_idx, force_gate ); } - if (cmask & PTO2_SUBTASK_MASK_AIV1) { + if (cmask & SUBTASK_MASK_AIV1) { bool p = to_pending && !tracker.is_aiv1_core_idle(core_offset); out_handles[n++] = prepare_subtask_to_core( - thread_idx, tracker.get_aiv1_core_offset(core_offset), slot_state, PTO2SubtaskSlot::AIV1, p, block_idx, + thread_idx, tracker.get_aiv1_core_offset(core_offset), slot_state, SubtaskSlot::AIV1, p, block_idx, force_gate ); } @@ -257,7 +257,7 @@ int SchedulerContext::prepare_block_for_dispatch( return n; } else if (shape == ResourceShape::AIC) { out_handles[0] = prepare_subtask_to_core( - thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIC, to_pending, block_idx, force_gate + thread_idx, core_offset, slot_state, SubtaskSlot::AIC, to_pending, block_idx, force_gate ); #if SIMPLER_DFX sched_chip_swimlane_[thread_idx].phase_dispatch_count += 1; @@ -265,7 +265,7 @@ int SchedulerContext::prepare_block_for_dispatch( return 1; } else { out_handles[0] = prepare_subtask_to_core( - thread_idx, core_offset, slot_state, PTO2SubtaskSlot::AIV0, to_pending, block_idx, force_gate + thread_idx, core_offset, slot_state, SubtaskSlot::AIV0, to_pending, block_idx, force_gate ); #if SIMPLER_DFX sched_chip_swimlane_[thread_idx].phase_dispatch_count += 1; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h index e83f92cbe7..a0b3f1d9fd 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h @@ -100,8 +100,8 @@ struct alignas(64) CoreExecState { int32_t running_reg_task_id; // offset 24: register task ID (AICPU_TASK_INVALID = idle) int32_t pending_reg_task_id; // offset 28: pending register task ID (AICPU_TASK_INVALID = none) uint32_t dispatch_seq; // offset 32: monotonic dispatch counter - PTO2SubtaskSlot running_subslot; // offset 36: which subtask slot is running - PTO2SubtaskSlot pending_subslot; // offset 37: which subtask slot is pending + SubtaskSlot running_subslot; // offset 36: which subtask slot is running + SubtaskSlot pending_subslot; // offset 37: which subtask slot is pending uint8_t pad0_[2]; // offset 38: alignment padding // Precomputed COND register pointer; resolved once in handshake so the // hot completion poll does a single volatile load instead of recomputing @@ -346,13 +346,13 @@ class alignas(64) CoreTracker { // slot) or the mask is empty. MixPlacement classify_mix_cluster(int32_t cluster_offset, uint8_t core_mask) const { BitStates used; - if (core_mask & PTO2_SUBTASK_MASK_AIC) { + if (core_mask & SUBTASK_MASK_AIC) { used |= BitStates::bit(cluster_offset); } - if (core_mask & PTO2_SUBTASK_MASK_AIV0) { + if (core_mask & SUBTASK_MASK_AIV0) { used |= BitStates::bit(cluster_offset + 1); } - if (core_mask & PTO2_SUBTASK_MASK_AIV1) { + if (core_mask & SUBTASK_MASK_AIV1) { used |= BitStates::bit(cluster_offset + 2); } if (!used.has_value() || (pending_occupied_ & used).has_value()) { @@ -392,9 +392,9 @@ class alignas(64) CoreTracker { // Cores of `cluster_offset` named by core_mask. BitStates mix_used_cores(int32_t cluster_offset, uint8_t core_mask) const { BitStates used; - if (core_mask & PTO2_SUBTASK_MASK_AIC) used |= BitStates::bit(cluster_offset); - if (core_mask & PTO2_SUBTASK_MASK_AIV0) used |= BitStates::bit(cluster_offset + 1); - if (core_mask & PTO2_SUBTASK_MASK_AIV1) used |= BitStates::bit(cluster_offset + 2); + if (core_mask & SUBTASK_MASK_AIC) used |= BitStates::bit(cluster_offset); + if (core_mask & SUBTASK_MASK_AIV0) used |= BitStates::bit(cluster_offset + 1); + if (core_mask & SUBTASK_MASK_AIV1) used |= BitStates::bit(cluster_offset + 2); return used; } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h index 5a49a0b717..df671e0f88 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h @@ -25,12 +25,12 @@ inline constexpr int32_t INVALID_KERNEL_ID = -1; /** * Subtask slot count: AIC, AIV0, AIV1 */ -inline constexpr int32_t PTO2_SUBTASK_SLOT_COUNT = 3; +inline constexpr int32_t SUBTASK_SLOT_COUNT = 3; /** * Subtask slot indices */ -enum class PTO2SubtaskSlot : uint8_t { +enum class SubtaskSlot : uint8_t { AIC = 0, AIV0 = 1, AIV1 = 2, @@ -39,9 +39,9 @@ enum class PTO2SubtaskSlot : uint8_t { /** * Subtask mask bits (for ActiveMask) */ -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIC = (1u << 0); // 0x1 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 +inline constexpr uint8_t SUBTASK_MASK_AIC = (1u << 0); // 0x1 +inline constexpr uint8_t SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 +inline constexpr uint8_t SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 // Dispatch-predicate comparison operator. The scheduler evaluates the predicate // at the dispatch point — the task is ready (fanin satisfied), so the predicate @@ -132,7 +132,7 @@ class ActiveMask { uint8_t raw() const { return raw_; } - bool subtask_active(PTO2SubtaskSlot slot) const { return (raw_ & (1u << static_cast(slot))) != 0; } + bool subtask_active(SubtaskSlot slot) const { return (raw_ & (1u << static_cast(slot))) != 0; } uint8_t core_mask() const { return raw_ & 0x07u; } @@ -141,7 +141,7 @@ class ActiveMask { if (cmask == 0) return ResourceShape::DUMMY; int bit_count = __builtin_popcount(cmask); if (bit_count >= 2) return ResourceShape::MIX; - if (cmask & PTO2_SUBTASK_MASK_AIC) return ResourceShape::AIC; + if (cmask & SUBTASK_MASK_AIC) return ResourceShape::AIC; return ResourceShape::AIV; } @@ -240,9 +240,9 @@ struct MixedKernels { ActiveMask to_active_mask() const { uint8_t mask = 0; - if (aic_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIC; - if (aiv0_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIV0; - if (aiv1_kernel_id != INVALID_KERNEL_ID) mask |= PTO2_SUBTASK_MASK_AIV1; + if (aic_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIC; + if (aiv0_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIV0; + if (aiv1_kernel_id != INVALID_KERNEL_ID) mask |= SUBTASK_MASK_AIV1; return ActiveMask(mask); } }; diff --git a/src/common/host_build_graph/graph_execution.cpp b/src/common/host_build_graph/graph_execution.cpp index 6a68011d9a..398c3a414f 100644 --- a/src/common/host_build_graph/graph_execution.cpp +++ b/src/common/host_build_graph/graph_execution.cpp @@ -96,7 +96,7 @@ bool bind_graph_topology(GraphExecution &execution) { } uint64_t required_heap = 0; - constexpr uint8_t VALID_ACTIVE_MASK = (1U << PTO2_SUBTASK_SLOT_COUNT) - 1U; + constexpr uint8_t VALID_ACTIVE_MASK = (1U << SUBTASK_SLOT_COUNT) - 1U; for (uint32_t i = 0; i < definition.task_count; ++i) { const GraphNodeDefinition &node = nodes[i]; if (node_offsets[i] != required_heap || node.total_output_size < 0 || node.tensor_count < 0 || @@ -109,7 +109,7 @@ bool bind_graph_topology(GraphExecution &execution) { node.total_required_subtasks < 0) { return false; } - for (int32_t slot = 0; slot < PTO2_SUBTASK_SLOT_COUNT; ++slot) { + for (int32_t slot = 0; slot < SUBTASK_SLOT_COUNT; ++slot) { const bool active = (node.active_mask & (1U << slot)) != 0; if (active != (node.kernel_id[slot] != INVALID_KERNEL_ID)) return false; } @@ -444,7 +444,7 @@ GraphMaterializeResult graph_execution_materialize_slice( const GraphNodeDefinition &source = nodes[i]; const uint64_t node_offset = node_offsets[i]; const uint64_t output_bytes = PTO2_ALIGN_UP(static_cast(source.total_output_size), PTO2_ALIGN_SIZE); - for (int k = 0; k < PTO2_SUBTASK_SLOT_COUNT; ++k) + for (int k = 0; k < SUBTASK_SLOT_COUNT; ++k) task.kernel_id[k] = source.kernel_id[k]; task.packed_buffer_base = reinterpret_cast(outer_base + node_offset); task.packed_buffer_end = reinterpret_cast(outer_base + node_offset + output_bytes); diff --git a/src/common/host_build_graph/graph_execution.h b/src/common/host_build_graph/graph_execution.h index caff55de64..29df6660ad 100644 --- a/src/common/host_build_graph/graph_execution.h +++ b/src/common/host_build_graph/graph_execution.h @@ -93,7 +93,7 @@ struct GraphPredicate { }; struct GraphNodeDefinition { - int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT]; + int32_t kernel_id[SUBTASK_SLOT_COUNT]; uint8_t active_mask; uint8_t task_attrs; int16_t logical_block_num; diff --git a/src/common/platform/include/aicpu/args_dump_aicpu.h b/src/common/platform/include/aicpu/args_dump_aicpu.h index a135a85fa3..8e6c410af6 100644 --- a/src/common/platform/include/aicpu/args_dump_aicpu.h +++ b/src/common/platform/include/aicpu/args_dump_aicpu.h @@ -99,7 +99,7 @@ inline void dump_args_for_task( GetFunctionBinAddrFn get_function_bin_addr, const ArgsDumpTaskMetadata *task_metadata = nullptr ) { // The record's func_ids[] must hold every active subtask's id. MaxSubtaskSlots - // is PTO2_SUBTASK_SLOT_COUNT at every call site, so this ties the record array + // is SUBTASK_SLOT_COUNT at every call site, so this ties the record array // size (platform layer) to the runtime subtask cap and catches any drift. static_assert(MaxSubtaskSlots <= ARGS_DUMP_MAX_FUNC_IDS, "ARGS_DUMP_MAX_FUNC_IDS must cover MaxSubtaskSlots"); const auto &pl = *slot_state.payload; diff --git a/src/common/platform/include/common/args_dump.h b/src/common/platform/include/common/args_dump.h index 3744e74910..5729cd9500 100644 --- a/src/common/platform/include/common/args_dump.h +++ b/src/common/platform/include/common/args_dump.h @@ -77,7 +77,7 @@ enum class ArgsDumpKind : uint8_t { }; // Max kernel ids a record carries: one per active subtask of a task (its mix -// membership). Must equal the runtime's PTO2_SUBTASK_SLOT_COUNT (1C2V => 3); +// membership). Must equal the runtime's SUBTASK_SLOT_COUNT (1C2V => 3); // can't reference it here (platform layer doesn't include the tmap+ring runtime // header), so a static_assert in dump_args_for_task ties the two together. constexpr uint8_t ARGS_DUMP_MAX_FUNC_IDS = 3; diff --git a/tests/ut/cpp/a2a3/test_graph_activation.cpp b/tests/ut/cpp/a2a3/test_graph_activation.cpp index 885c7420dc..670e98a5e5 100644 --- a/tests/ut/cpp/a2a3/test_graph_activation.cpp +++ b/tests/ut/cpp/a2a3/test_graph_activation.cpp @@ -62,7 +62,7 @@ class GraphActivationTest : public ::testing::Test { memset(&node, 0, sizeof(GraphNodeStorage)); node.slot.task_state.store(state); node.slot.graph_node_index = node_index; - node.slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + node.slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); node.slot.task_kind = TaskKind::KERNEL; node.slot.total_required_subtasks = 1; node.slot.logical_block_num = 1; diff --git a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp index 9afc901a64..e0557bbe94 100644 --- a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp @@ -173,7 +173,7 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { EXPECT_EQ(root_pl.dump_metadata.scalar_dtypes[0], static_cast(DataType::FLOAT32)); EXPECT_NE(root_desc.packed_buffer_base, POISON_PTR); EXPECT_NE(root_desc.packed_buffer_base, nullptr); - EXPECT_EQ(root_desc.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)], 0); + EXPECT_EQ(root_desc.kernel_id[static_cast(SubtaskSlot::AIV0)], 0); // The consumer's fanin is written: two duplicate deps dedupe to one. const TaskPayload &cons_pl = ring.task_payloads[ring.get_slot_by_task_id(consumer.task_id().local())]; diff --git a/tests/ut/cpp/a2a3/test_scheduler_state.cpp b/tests/ut/cpp/a2a3/test_scheduler_state.cpp index 292529f637..98403bf1d2 100644 --- a/tests/ut/cpp/a2a3/test_scheduler_state.cpp +++ b/tests/ut/cpp/a2a3/test_scheduler_state.cpp @@ -92,7 +92,7 @@ class SchedulerStateTest : public ::testing::Test { slot.fanout_lock.store(0); slot.fanout_head = nullptr; slot.ring_id = ring_id; - slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; @@ -119,7 +119,7 @@ class SchedulerStateTest : public ::testing::Test { slot.fanout_lock.store(0, std::memory_order_relaxed); slot.fanout_head = nullptr; slot.ring_id = ring_id; - slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); slot.completed_subtasks.store(0, std::memory_order_relaxed); slot.total_required_subtasks = 1; slot.logical_block_num = 1; @@ -389,7 +389,7 @@ TEST(CoreTrackerTest, MixPartiallyRunningClusterAdmittedAsPerCorePlacement) { // Per-core dispatch then puts the idle AIC/AIV1 on their running slots (marked // running so the completion poller tracks them) and the busy AIV0 on its pending // slot, executing after the in-flight AIV-only task. - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0 | PTO2_SUBTASK_MASK_AIV1; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0 | SUBTASK_MASK_AIV1; EXPECT_EQ(tracker.classify_mix_cluster(cluster_offset, used_mask), CoreTracker::MixPlacement::PENDING); // Not all used cores are idle, so the IDLE phase skips this cluster; it is @@ -452,7 +452,7 @@ TEST(CoreTrackerTest, MixClassifyIgnoresUnusedBusyCoreForRunningPlacement) { constexpr int32_t cluster_offset = 0; tracker.change_core_state(cluster_offset + 2); // AIV1 running, unused by this 1c1v task - auto placement = tracker.classify_mix_cluster(cluster_offset, PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0); + auto placement = tracker.classify_mix_cluster(cluster_offset, SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0); EXPECT_EQ(placement, CoreTracker::MixPlacement::RUNNING); } @@ -466,7 +466,7 @@ TEST(CoreTrackerTest, MixClassifyAllowsPendingForUsedRunningCoresOnly) { tracker.change_core_state(cluster_offset + 1); tracker.set_pending_occupied(cluster_offset + 2); // Unused AIV1 must not block this 1c1v task - auto placement = tracker.classify_mix_cluster(cluster_offset, PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0); + auto placement = tracker.classify_mix_cluster(cluster_offset, SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0); EXPECT_EQ(placement, CoreTracker::MixPlacement::PENDING); } @@ -480,7 +480,7 @@ TEST(CoreTrackerTest, MixClassifyAdmitsMixedUsedCoresAsPending) { // Mixed used-core state (AIC idle, AIV0 running) is admitted as PENDING; the // idle AIC takes its running slot and the busy AIV0 takes its pending slot. - auto placement = tracker.classify_mix_cluster(cluster_offset, PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0); + auto placement = tracker.classify_mix_cluster(cluster_offset, SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0); EXPECT_EQ(placement, CoreTracker::MixPlacement::PENDING); } @@ -494,7 +494,7 @@ TEST(CoreTrackerTest, MixClassifyRejectsOccupiedPendingSlotInUsedMask) { tracker.change_core_state(cluster_offset + 1); tracker.set_pending_occupied(cluster_offset + 1); - auto placement = tracker.classify_mix_cluster(cluster_offset, PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0); + auto placement = tracker.classify_mix_cluster(cluster_offset, SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0); EXPECT_EQ(placement, CoreTracker::MixPlacement::REJECT); } @@ -507,7 +507,7 @@ TEST(CoreTrackerTest, MixRunningClusterHelpersUseActiveMask) { tracker.change_core_state(2); tracker.change_core_state(5); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0; EXPECT_EQ(tracker.get_idle_core_offset_states(ResourceShape::MIX).count(), 0); EXPECT_EQ(tracker.count_mix_running_clusters(used_mask), 2); @@ -519,7 +519,7 @@ TEST(CoreTrackerTest, MixRunningClusterHelpersRejectOccupiedUsedPendingSlot) { tracker.init(1); tracker.set_cluster(0, 0, 1, 2); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0; tracker.set_pending_occupied(1); EXPECT_EQ(tracker.count_mix_running_clusters(used_mask), 0); @@ -534,7 +534,7 @@ TEST(CoreTrackerTest, CountAvailableBlocksAivIncludesFreePendingSlots) { tracker.change_core_state(1); tracker.change_core_state(4); - constexpr uint8_t aiv_mask = PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t aiv_mask = SUBTASK_MASK_AIV0; constexpr int32_t exact_fit = 4; EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, false), 2); EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, true), exact_fit); @@ -551,7 +551,7 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixCountsLogicalClusters) { tracker.change_core_state(4); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0 | PTO2_SUBTASK_MASK_AIV1; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0 | SUBTASK_MASK_AIV1; EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, false), 1); EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, true), 2); } @@ -561,7 +561,7 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixRejectsOccupiedUsedPendingSlot) { tracker.init(1); tracker.set_cluster(0, 0, 1, 2); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0; tracker.change_core_state(1); tracker.set_pending_occupied(1); @@ -574,7 +574,7 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixIgnoresUnavailableUnusedCore) { tracker.init(1); tracker.set_cluster(0, 0, 1, 2); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0; tracker.change_core_state(2); tracker.set_pending_occupied(2); diff --git a/tests/ut/cpp/a2a3/test_task_state.cpp b/tests/ut/cpp/a2a3/test_task_state.cpp index 90de1e4151..25048df050 100644 --- a/tests/ut/cpp/a2a3/test_task_state.cpp +++ b/tests/ut/cpp/a2a3/test_task_state.cpp @@ -70,7 +70,7 @@ class TaskStateTest : public ::testing::Test { slot.fanout_lock.store(0); slot.fanout_head = nullptr; slot.ring_id = 0; - slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index 5f7ac38254..fad7e20587 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -86,7 +86,7 @@ class WiringTest : public ::testing::Test { slot.fanout_lock.store(0); slot.fanout_head = nullptr; slot.ring_id = ring_id; - slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; @@ -800,7 +800,7 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); init_slot(downstream, CHIP_TASK_PENDING, 1, 1); - sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); + sync_consumer.active_mask = ActiveMask(SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.task_attrs.set_early_resolve(true); sync_consumer.logical_block_num = 2; @@ -837,7 +837,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); init_slot(downstream, CHIP_TASK_PENDING, 1, 1); - sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); + sync_consumer.active_mask = ActiveMask(SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.task_attrs.set_early_resolve(true); sync_consumer.logical_block_num = 2; @@ -866,7 +866,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { TEST_F(WiringTest, ArmedEarlySyncDrainOwnsFinalReadyRoute) { alignas(64) ChipTaskSlotState sync_consumer; init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); - sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); + sync_consumer.active_mask = ActiveMask(SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); @@ -935,7 +935,7 @@ TEST_F(WiringTest, ArmedEarlySyncDrainKeepsEveryStagerGatedAfterReady) { TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) { alignas(64) ChipTaskSlotState sync_consumer; init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); - sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); + sync_consumer.active_mask = ActiveMask(SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); @@ -958,7 +958,7 @@ TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) TEST_F(WiringTest, CancelledEarlySyncDrainRoutesProducerRelease) { alignas(64) ChipTaskSlotState sync_consumer; init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); - sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); + sync_consumer.active_mask = ActiveMask(SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); @@ -976,7 +976,7 @@ TEST_F(WiringTest, CancelledEarlySyncDrainRoutesProducerRelease) { TEST_F(WiringTest, ProducerReleaseTransfersReadyRouteToCancellingDrain) { alignas(64) ChipTaskSlotState sync_consumer; init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); - sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); + sync_consumer.active_mask = ActiveMask(SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); @@ -1099,12 +1099,12 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { // Consumer1: needs 1 more fanin to become ready init_slot(consumer1, CHIP_TASK_PENDING, 2, 1); consumer1.fanin_refcount.store(1); // 1 of 2 satisfied - consumer1.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + consumer1.active_mask = ActiveMask(SUBTASK_MASK_AIC); // Consumer2: this release will make it ready init_slot(consumer2, CHIP_TASK_PENDING, 2, 1); consumer2.fanin_refcount.store(1); // 1 of 2 satisfied - consumer2.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + consumer2.active_mask = ActiveMask(SUBTASK_MASK_AIC); // Build fanout chain: producer -> consumer2 -> consumer1 DepListEntry dep_entries[2]; diff --git a/tests/ut/cpp/a5/test_graph_activation.cpp b/tests/ut/cpp/a5/test_graph_activation.cpp index 885c7420dc..670e98a5e5 100644 --- a/tests/ut/cpp/a5/test_graph_activation.cpp +++ b/tests/ut/cpp/a5/test_graph_activation.cpp @@ -62,7 +62,7 @@ class GraphActivationTest : public ::testing::Test { memset(&node, 0, sizeof(GraphNodeStorage)); node.slot.task_state.store(state); node.slot.graph_node_index = node_index; - node.slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + node.slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); node.slot.task_kind = TaskKind::KERNEL; node.slot.total_required_subtasks = 1; node.slot.logical_block_num = 1; diff --git a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp index 9afc901a64..e0557bbe94 100644 --- a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp @@ -173,7 +173,7 @@ TEST_F(HbgSubmitPoisonTest, EveryDeviceReadFieldIsWrittenOverPoison) { EXPECT_EQ(root_pl.dump_metadata.scalar_dtypes[0], static_cast(DataType::FLOAT32)); EXPECT_NE(root_desc.packed_buffer_base, POISON_PTR); EXPECT_NE(root_desc.packed_buffer_base, nullptr); - EXPECT_EQ(root_desc.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)], 0); + EXPECT_EQ(root_desc.kernel_id[static_cast(SubtaskSlot::AIV0)], 0); // The consumer's fanin is written: two duplicate deps dedupe to one. const TaskPayload &cons_pl = ring.task_payloads[ring.get_slot_by_task_id(consumer.task_id().local())]; diff --git a/tests/ut/cpp/a5/test_scheduler_state.cpp b/tests/ut/cpp/a5/test_scheduler_state.cpp index 90730b524b..b9fac69df4 100644 --- a/tests/ut/cpp/a5/test_scheduler_state.cpp +++ b/tests/ut/cpp/a5/test_scheduler_state.cpp @@ -92,7 +92,7 @@ class SchedulerStateTest : public ::testing::Test { slot.fanout_lock.store(0); slot.fanout_head = nullptr; slot.ring_id = ring_id; - slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; @@ -119,7 +119,7 @@ class SchedulerStateTest : public ::testing::Test { slot.fanout_lock.store(0, std::memory_order_relaxed); slot.fanout_head = nullptr; slot.ring_id = ring_id; - slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); slot.completed_subtasks.store(0, std::memory_order_relaxed); slot.total_required_subtasks = 1; slot.logical_block_num = 1; @@ -455,7 +455,7 @@ TEST(CoreTrackerTest, MixPartiallyRunningClusterAdmittedAsPerCorePlacement) { // Per-core dispatch then puts the idle AIC/AIV1 on their running slots (marked // running so the completion poller tracks them) and the busy AIV0 on its pending // slot, executing after the in-flight AIV-only task. - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0 | PTO2_SUBTASK_MASK_AIV1; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0 | SUBTASK_MASK_AIV1; EXPECT_EQ(tracker.classify_mix_cluster(cluster_offset, used_mask), CoreTracker::MixPlacement::PENDING); // Not all used cores are idle, so the IDLE phase skips this cluster; it is @@ -518,7 +518,7 @@ TEST(CoreTrackerTest, MixClassifyIgnoresUnusedBusyCoreForRunningPlacement) { constexpr int32_t cluster_offset = 0; tracker.change_core_state(cluster_offset + 2); // AIV1 running, unused by this 1c1v task - auto placement = tracker.classify_mix_cluster(cluster_offset, PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0); + auto placement = tracker.classify_mix_cluster(cluster_offset, SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0); EXPECT_EQ(placement, CoreTracker::MixPlacement::RUNNING); } @@ -532,7 +532,7 @@ TEST(CoreTrackerTest, MixClassifyAllowsPendingForUsedRunningCoresOnly) { tracker.change_core_state(cluster_offset + 1); tracker.set_pending_occupied(cluster_offset + 2); // Unused AIV1 must not block this 1c1v task - auto placement = tracker.classify_mix_cluster(cluster_offset, PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0); + auto placement = tracker.classify_mix_cluster(cluster_offset, SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0); EXPECT_EQ(placement, CoreTracker::MixPlacement::PENDING); } @@ -546,7 +546,7 @@ TEST(CoreTrackerTest, MixClassifyAdmitsMixedUsedCoresAsPending) { // Mixed used-core state (AIC idle, AIV0 running) is admitted as PENDING; the // idle AIC takes its running slot and the busy AIV0 takes its pending slot. - auto placement = tracker.classify_mix_cluster(cluster_offset, PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0); + auto placement = tracker.classify_mix_cluster(cluster_offset, SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0); EXPECT_EQ(placement, CoreTracker::MixPlacement::PENDING); } @@ -560,7 +560,7 @@ TEST(CoreTrackerTest, MixClassifyRejectsOccupiedPendingSlotInUsedMask) { tracker.change_core_state(cluster_offset + 1); tracker.set_pending_occupied(cluster_offset + 1); - auto placement = tracker.classify_mix_cluster(cluster_offset, PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0); + auto placement = tracker.classify_mix_cluster(cluster_offset, SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0); EXPECT_EQ(placement, CoreTracker::MixPlacement::REJECT); } @@ -573,7 +573,7 @@ TEST(CoreTrackerTest, MixRunningClusterHelpersUseActiveMask) { tracker.change_core_state(2); tracker.change_core_state(5); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0; EXPECT_EQ(tracker.get_idle_core_offset_states(ResourceShape::MIX).count(), 0); EXPECT_EQ(tracker.count_mix_running_clusters(used_mask), 2); @@ -585,7 +585,7 @@ TEST(CoreTrackerTest, MixRunningClusterHelpersRejectOccupiedUsedPendingSlot) { tracker.init(1); tracker.set_cluster(0, 0, 1, 2); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0; tracker.set_pending_occupied(1); EXPECT_EQ(tracker.count_mix_running_clusters(used_mask), 0); @@ -600,7 +600,7 @@ TEST(CoreTrackerTest, CountAvailableBlocksAivIncludesFreePendingSlots) { tracker.change_core_state(1); tracker.change_core_state(4); - constexpr uint8_t aiv_mask = PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t aiv_mask = SUBTASK_MASK_AIV0; constexpr int32_t exact_fit = 4; EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, false), 2); EXPECT_EQ(tracker.count_available_blocks(ResourceShape::AIV, aiv_mask, true), exact_fit); @@ -617,7 +617,7 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixCountsLogicalClusters) { tracker.change_core_state(4); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0 | PTO2_SUBTASK_MASK_AIV1; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0 | SUBTASK_MASK_AIV1; EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, false), 1); EXPECT_EQ(tracker.count_available_blocks(ResourceShape::MIX, used_mask, true), 2); } @@ -627,7 +627,7 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixRejectsOccupiedUsedPendingSlot) { tracker.init(1); tracker.set_cluster(0, 0, 1, 2); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0; tracker.change_core_state(1); tracker.set_pending_occupied(1); @@ -640,7 +640,7 @@ TEST(CoreTrackerTest, CountAvailableBlocksMixIgnoresUnavailableUnusedCore) { tracker.init(1); tracker.set_cluster(0, 0, 1, 2); - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0; tracker.change_core_state(2); tracker.set_pending_occupied(2); diff --git a/tests/ut/cpp/a5/test_task_state.cpp b/tests/ut/cpp/a5/test_task_state.cpp index 90de1e4151..25048df050 100644 --- a/tests/ut/cpp/a5/test_task_state.cpp +++ b/tests/ut/cpp/a5/test_task_state.cpp @@ -70,7 +70,7 @@ class TaskStateTest : public ::testing::Test { slot.fanout_lock.store(0); slot.fanout_head = nullptr; slot.ring_id = 0; - slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index 1a57ae67ef..001a24b541 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -83,7 +83,7 @@ class WiringTest : public ::testing::Test { slot.fanout_lock.store(0); slot.fanout_head = nullptr; slot.ring_id = ring_id; - slot.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + slot.active_mask = ActiveMask(SUBTASK_MASK_AIC); slot.completed_subtasks.store(0); slot.total_required_subtasks = 1; slot.logical_block_num = 1; @@ -311,7 +311,7 @@ TEST_F(WiringTest, SyncStartStagingFinalizeRetriesProducerFirstRendezvous) { init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); init_slot(downstream, CHIP_TASK_PENDING, 1, 1); - sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); + sync_consumer.active_mask = ActiveMask(SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.task_attrs.set_early_resolve(true); sync_consumer.logical_block_num = 2; @@ -346,7 +346,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); init_slot(downstream, CHIP_TASK_PENDING, 1, 1); - sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); + sync_consumer.active_mask = ActiveMask(SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.task_attrs.set_early_resolve(true); sync_consumer.logical_block_num = 2; @@ -375,7 +375,7 @@ TEST_F(WiringTest, SyncStartProducerReleaseCompletesStagerFirstRendezvous) { TEST_F(WiringTest, EarlySyncFinishBetweenReleasePhasesRetainsOwnerCompleteState) { alignas(64) ChipTaskSlotState sync_consumer; init_slot(sync_consumer, CHIP_TASK_PENDING, 1, 1); - sync_consumer.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIV0); + sync_consumer.active_mask = ActiveMask(SUBTASK_MASK_AIV0); sync_consumer.task_attrs.set_sync_start(); sync_consumer.logical_block_num = 2; sync_consumer.payload->early_dispatch_state.store(EARLY_DISPATCH_STAGING, std::memory_order_relaxed); @@ -413,12 +413,12 @@ TEST_F(WiringTest, OnMixedTaskCompleteNotifiesConsumers) { // Consumer1: needs 1 more fanin to become ready init_slot(consumer1, CHIP_TASK_PENDING, 2, 1); consumer1.fanin_refcount.store(1); // 1 of 2 satisfied - consumer1.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + consumer1.active_mask = ActiveMask(SUBTASK_MASK_AIC); // Consumer2: this release will make it ready init_slot(consumer2, CHIP_TASK_PENDING, 2, 1); consumer2.fanin_refcount.store(1); // 1 of 2 satisfied - consumer2.active_mask = ActiveMask(PTO2_SUBTASK_MASK_AIC); + consumer2.active_mask = ActiveMask(SUBTASK_MASK_AIC); // Build fanout chain: producer -> consumer2 -> consumer1 DepListEntry dep_entries[2]; diff --git a/tests/ut/cpp/common/test_hbg_core_tracker.cpp b/tests/ut/cpp/common/test_hbg_core_tracker.cpp index 9556f0ed2c..d498e35bcd 100644 --- a/tests/ut/cpp/common/test_hbg_core_tracker.cpp +++ b/tests/ut/cpp/common/test_hbg_core_tracker.cpp @@ -21,7 +21,7 @@ TEST(HostBuildGraphCoreTrackerTest, MixPlacementSelectionPreservesClustersAboveB tracker.set_cluster(cluster, offset, offset + 1, offset + 2); } - constexpr uint8_t used_mask = PTO2_SUBTASK_MASK_AIC | PTO2_SUBTASK_MASK_AIV0 | PTO2_SUBTASK_MASK_AIV1; + constexpr uint8_t used_mask = SUBTASK_MASK_AIC | SUBTASK_MASK_AIV0 | SUBTASK_MASK_AIV1; int32_t high_offset = (CoreTracker::MAX_CLUSTERS - 1) * 3; auto running = tracker.get_mix_cluster_offset_states(used_mask, CoreTracker::MixPlacement::RUNNING); EXPECT_EQ(running.count(), CoreTracker::MAX_CLUSTERS); From f9ac2e665b5eb4d0ec0f241a4d0bcad3c361cf36 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:48:22 -0700 Subject: [PATCH 15/22] Refactor: name the chip runtime's scope capacities by their context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PTO2ScopeMode` -> `ScopeMode` is a plain strip. The two capacities are not: `CHIP_MAX_SCOPE_DEPTH` and `CHIP_SCOPE_TASKS_CAP` keep a prefix because `src/common/hierarchical/types.h:117` already declares `static constexpr int32_t MAX_SCOPE_DEPTH = 64` for the host orchestrator — and because the chip runtime's version is an object-like macro, the bare name would not have collided cleanly. It would have textually rewritten the host's declaration into `static constexpr int32_t 64 = 64;` in any translation unit that saw the chip header first. Rule 13 gives the bare name to the L3+ side regardless. The two scope-stats collector bounds do take bare names — `SCOPE_STATS_MAX_RING_DEPTH`, `SCOPE_STATS_MAX_SCOPE_DEPTH` — since they are already qualified by what they bound and nothing else spells them. 176 occurrences across 62 files. These are what the `SIMPLER_SCOPE` commit left behind: it renamed the macro and its guard, not the mode enum the macro sets or the caps that bound a scope. Co-Authored-By: Claude Opus 5 (1M context) --- docs/dfx/scope-stats.md | 2 +- docs/manual-scope.md | 6 ++--- .../orchestration/decode_fwd_graph.cpp | 6 ++--- .../orchestration/decode_fwd_layers.cpp | 4 +-- .../kernels/orchestration/decode_fwd.cpp | 10 +++---- .../paged_attention_manual_scope/README.md | 2 +- .../orchestration/paged_attention_orch.cpp | 4 +-- .../README.md | 2 +- .../orchestration/paged_attention_orch.cpp | 2 +- .../orchestration/decode_fwd_layers.cpp | 4 +-- .../orchestration/decode_fwd_layers.cpp | 4 +-- .../orchestration/paged_attention_orch.cpp | 2 +- .../orchestration/paged_attention_orch.cpp | 2 +- .../orchestration/decode_fwd_layers.cpp | 4 +-- .../orchestration/orchestration_api.h | 6 ++--- .../host_build_graph/runtime/orchestrator.h | 4 +-- .../orchestrator_core/orchestrator.cpp | 26 +++++++++---------- .../orchestrator_core/runtime_core.cpp | 4 +-- .../host_build_graph/runtime/runtime_core.h | 2 +- .../host_build_graph/runtime/runtime_types.h | 2 +- .../runtime/shared/runtime_init.cpp | 2 +- .../runtime/host_build_graph/runtime/types.h | 2 +- .../orchestration/orchestration_api.h | 6 ++--- .../runtime/orchestrator.cpp | 10 +++---- .../runtime/orchestrator.h | 6 ++--- .../runtime/runtime_core.cpp | 4 +-- .../runtime/runtime_core.h | 2 +- .../runtime/runtime_types.h | 4 +-- .../runtime/scheduler/scheduler_cold_path.cpp | 2 +- .../runtime/shared/runtime_init.cpp | 10 +++---- .../tensormap_and_ringbuffer/runtime/types.h | 2 +- .../orchestration/orchestration_api.h | 6 ++--- .../host_build_graph/runtime/orchestrator.h | 4 +-- .../orchestrator_core/orchestrator.cpp | 26 +++++++++---------- .../orchestrator_core/runtime_core.cpp | 4 +-- .../host_build_graph/runtime/runtime_core.h | 2 +- .../host_build_graph/runtime/runtime_types.h | 2 +- .../runtime/shared/runtime_init.cpp | 2 +- .../runtime/host_build_graph/runtime/types.h | 2 +- .../orchestration/orchestration_api.h | 6 ++--- .../runtime/orchestrator.cpp | 10 +++---- .../runtime/orchestrator.h | 6 ++--- .../runtime/runtime_core.cpp | 4 +-- .../runtime/runtime_core.h | 2 +- .../runtime/runtime_types.h | 4 +-- .../runtime/scheduler/scheduler_cold_path.cpp | 2 +- .../runtime/shared/runtime_init.cpp | 10 +++---- .../tensormap_and_ringbuffer/runtime/types.h | 2 +- src/common/hierarchical/types.h | 2 +- .../platform/include/common/scope_stats.h | 10 +++---- .../aicpu/scope_stats_collector_aicpu.cpp | 16 ++++++------ .../shared/host/scope_stats_collector.cpp | 2 +- .../sync_start_early_local_owner_orch.cpp | 2 +- .../spmd_sync_start_early_dispatch_orch.cpp | 2 +- .../sync_start_early_local_owner_orch.cpp | 2 +- .../spmd_sync_start_early_dispatch_orch.cpp | 2 +- tests/ut/cpp/a2a3/test_a2a3_fatal.cpp | 12 ++++----- tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp | 6 ++--- tests/ut/cpp/a5/test_a5_fatal.cpp | 12 ++++----- tests/ut/cpp/a5/test_orchestrator_fanin.cpp | 6 ++--- .../common/test_hbg_graph_async_submit.cpp | 2 +- .../common/test_hbg_graph_submit_failure.cpp | 4 +-- 62 files changed, 161 insertions(+), 161 deletions(-) diff --git a/docs/dfx/scope-stats.md b/docs/dfx/scope-stats.md index c1b477fc8f..8c3e36b161 100644 --- a/docs/dfx/scope-stats.md +++ b/docs/dfx/scope-stats.md @@ -295,7 +295,7 @@ the host to chase a device pointer into the orchestration `.so`'s string table. `on_fatal` sets `header.fatal_latched`, surfacing as `"fatal": true`; the host still emits whatever records made it. -`ring_id` outside `[0, PTO2_SCOPE_STATS_MAX_RING_DEPTH)` is silently +`ring_id` outside `[0, SCOPE_STATS_MAX_RING_DEPTH)` is silently dropped. Caps are copied verbatim into the buffer header so the host can render `used/cap` without a second device→host query. diff --git a/docs/manual-scope.md b/docs/manual-scope.md index ba9f4a2997..165ec42af2 100644 --- a/docs/manual-scope.md +++ b/docs/manual-scope.md @@ -13,7 +13,7 @@ Keep manual scope small and explicit: The v0 design keeps these rules: -1. `SIMPLER_SCOPE(PTO2ScopeMode::MANUAL)` opts a scope into manual mode. +1. `SIMPLER_SCOPE(ScopeMode::MANUAL)` opts a scope into manual mode. 2. `MANUAL` nested inside active `MANUAL` is allowed. 3. `AUTO` nested inside active `MANUAL` is rejected. 4. Manual deps are attached before submit through `CoreTaskArgs.set_dependencies(...)`. @@ -26,7 +26,7 @@ The v0 design keeps these rules: ### Scope ```cpp -SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { +SIMPLER_SCOPE(ScopeMode::MANUAL) { ... } ``` @@ -132,7 +132,7 @@ For a submitted task: ## Example Pattern ```cpp -SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { +SIMPLER_SCOPE(ScopeMode::MANUAL) { auto alloc = alloc_tensors(tmp_ci); CoreTaskArgs qk; diff --git a/examples/a2a3/host_build_graph/deepseek_v4_flash_decode/kernels/orchestration/decode_fwd_graph.cpp b/examples/a2a3/host_build_graph/deepseek_v4_flash_decode/kernels/orchestration/decode_fwd_graph.cpp index d7687d7d8c..fabc78ea1c 100644 --- a/examples/a2a3/host_build_graph/deepseek_v4_flash_decode/kernels/orchestration/decode_fwd_graph.cpp +++ b/examples/a2a3/host_build_graph/deepseek_v4_flash_decode/kernels/orchestration/decode_fwd_graph.cpp @@ -1093,7 +1093,7 @@ void csa_attn_block(const GraphTaskArgs &args) { for (int64_t __init_i = 0; __init_i < 8; ++__init_i) proj_b_tids_inline2079_inline10609[__init_i] = TaskId::invalid(); ChipTensor o_r_pad_inline2225_inline10832__rv_v2 = o_r_pad_inline2225_inline10832; - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { for (int64_t g_inline2162_inline10271 = 0; g_inline2162_inline10271 < 8; g_inline2162_inline10271 += 1) { int64_t row_base_o_inline2078_inline10745 = (g_inline2162_inline10271 * 8); int64_t out_col_g_inline2181_inline10439 = (g_inline2162_inline10271 * 1024); @@ -2804,7 +2804,7 @@ void hca_attn_block(const GraphTaskArgs &args) { for (int64_t __init_i = 0; __init_i < 8; ++__init_i) proj_b_tids_inline1519_inline11335[__init_i] = TaskId::invalid(); ChipTensor o_r_pad_inline1536_inline11403__rv_v2 = o_r_pad_inline1536_inline11403; - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { for (int64_t g_inline1480_inline11801 = 0; g_inline1480_inline11801 < 8; g_inline1480_inline11801 += 1) { int64_t row_base_o_inline1525_inline11698 = (g_inline1480_inline11801 * 8); int64_t out_col_g_inline1609_inline11551 = (g_inline1480_inline11801 * 1024); @@ -4315,7 +4315,7 @@ void swa_attn_block(const GraphTaskArgs &args) { TaskOutputTensors task_25_outs = rt_submit_aiv_task(26, params_t25); TaskId merge_tid_inline1108_inline8902 = task_25_outs.task_id(); ChipTensor o_r_pad_inline974_inline8862__rv_v2 = o_r_pad_inline974_inline8862; - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { for (int64_t g_inline978_inline8746 = 0; g_inline978_inline8746 < 8; g_inline978_inline8746 += 1) { int64_t row_base_o_inline1126_inline8745 = (g_inline978_inline8746 * 8); int64_t out_col_g_inline1092_inline8900 = (g_inline978_inline8746 * 1024); diff --git a/examples/a2a3/host_build_graph/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp b/examples/a2a3/host_build_graph/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp index f20155e09c..132df292a8 100644 --- a/examples/a2a3/host_build_graph/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp +++ b/examples/a2a3/host_build_graph/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp @@ -149,7 +149,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip TaskId prev_normed_tid[1]; for (int64_t __init_i = 0; __init_i < 1; ++__init_i) prev_normed_tid[__init_i] = TaskId::invalid(); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { TaskId _submit_deps_buf[1]; for (int64_t __init_i = 0; __init_i < 1; ++__init_i) _submit_deps_buf[__init_i] = TaskId::invalid(); @@ -335,7 +335,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip TaskId down_tids_inline156[85]; for (int64_t __init_i = 0; __init_i < 85; ++__init_i) down_tids_inline156[__init_i] = TaskId::invalid(); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { // Phase-fence barrier 1: dependency-only dummy task CoreTaskArgs params_phase_fence_barrier_1; TaskOutputTensors phase_fence_barrier_1_outs = diff --git a/examples/a2a3/tensormap_and_ringbuffer/deepseek_v4_flash_decode/kernels/orchestration/decode_fwd.cpp b/examples/a2a3/tensormap_and_ringbuffer/deepseek_v4_flash_decode/kernels/orchestration/decode_fwd.cpp index 898a32b030..6bd83abea2 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/deepseek_v4_flash_decode/kernels/orchestration/decode_fwd.cpp +++ b/examples/a2a3/tensormap_and_ringbuffer/deepseek_v4_flash_decode/kernels/orchestration/decode_fwd.cpp @@ -1575,7 +1575,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip TaskOutputTensors task_25_outs = rt_submit_aiv_task(26, params_t25); TaskId merge_tid_inline1108_inline8902 = task_25_outs.task_id(); ChipTensor o_r_pad_inline974_inline8862__rv_v2 = o_r_pad_inline974_inline8862; - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { for (int64_t g_inline978_inline8746 = 0; g_inline978_inline8746 < 8; g_inline978_inline8746 += 1) { int64_t row_base_o_inline1126_inline8745 = (g_inline978_inline8746 * 8); int64_t out_col_g_inline1092_inline8900 = (g_inline978_inline8746 * 1024); @@ -2995,7 +2995,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip TaskOutputTensors task_82_outs = rt_submit_aiv_task(85, params_t82); TaskId merge_tid_inline1108_inline9664 = task_82_outs.task_id(); ChipTensor o_r_pad_inline974_inline9624__rv_v2 = o_r_pad_inline974_inline9624; - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { for (int64_t g_inline978_inline9508 = 0; g_inline978_inline9508 < 8; g_inline978_inline9508 += 1) { int64_t row_base_o_inline1126_inline9507 = (g_inline978_inline9508 * 8); int64_t out_col_g_inline1092_inline9662 = (g_inline978_inline9508 * 1024); @@ -5590,7 +5590,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip for (int64_t __init_i = 0; __init_i < 8; ++__init_i) proj_b_tids_inline2079_inline10609[__init_i] = TaskId::invalid(); ChipTensor o_r_pad_inline2225_inline10832__rv_v2 = o_r_pad_inline2225_inline10832; - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { for (int64_t g_inline2162_inline10271 = 0; g_inline2162_inline10271 < 8; g_inline2162_inline10271 += 1) { int64_t row_base_o_inline2078_inline10745 = (g_inline2162_inline10271 * 8); @@ -7732,7 +7732,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip for (int64_t __init_i = 0; __init_i < 8; ++__init_i) proj_b_tids_inline1519_inline11335[__init_i] = TaskId::invalid(); ChipTensor o_r_pad_inline1536_inline11403__rv_v2 = o_r_pad_inline1536_inline11403; - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { for (int64_t g_inline1480_inline11801 = 0; g_inline1480_inline11801 < 8; g_inline1480_inline11801 += 1) { int64_t row_base_o_inline1525_inline11698 = (g_inline1480_inline11801 * 8); @@ -10306,7 +10306,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip for (int64_t __init_i = 0; __init_i < 8; ++__init_i) proj_b_tids_inline2079_inline12533[__init_i] = TaskId::invalid(); ChipTensor o_r_pad_inline2225_inline12756__rv_v2 = o_r_pad_inline2225_inline12756; - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { for (int64_t g_inline2162_inline12195 = 0; g_inline2162_inline12195 < 8; g_inline2162_inline12195 += 1) { int64_t row_base_o_inline2078_inline12669 = (g_inline2162_inline12195 * 8); int64_t out_col_g_inline2181_inline12363 = (g_inline2162_inline12195 * 1024); diff --git a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/README.md b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/README.md index 865fde06e6..b41562e7e2 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/README.md +++ b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/README.md @@ -15,7 +15,7 @@ diff -r ../paged_attention/kernels ./kernels # only the orch file differs ## What changes -`SIMPLER_SCOPE()` + `SIMPLER_SCOPE_GUARD()` becomes `SIMPLER_SCOPE(PTO2ScopeMode::MANUAL)`, +`SIMPLER_SCOPE()` + `SIMPLER_SCOPE_GUARD()` becomes `SIMPLER_SCOPE(ScopeMode::MANUAL)`, and each submit names its predecessors. See [`docs/manual-scope.md`](../../../../docs/manual-scope.md) for the mode itself. diff --git a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/kernels/orchestration/paged_attention_orch.cpp b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/kernels/orchestration/paged_attention_orch.cpp index 5ee3945a21..cb544566c3 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/kernels/orchestration/paged_attention_orch.cpp +++ b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/kernels/orchestration/paged_attention_orch.cpp @@ -13,7 +13,7 @@ * * Matches the small-case paged_attention orchestration shape while replacing * the automatic same-scope dependency wiring with explicit task-to-task deps - * inside SIMPLER_SCOPE(PTO2ScopeMode::MANUAL). + * inside SIMPLER_SCOPE(ScopeMode::MANUAL). */ #include @@ -149,7 +149,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip uint64_t cur_seq = static_cast(get_tensor_data(context_lens, 1, cl_idx)); uint64_t bn_this_batch = (cur_seq + block_size - 1) / block_size; for (uint64_t q_idx = 0; q_idx < q_loop; q_idx++) { - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { CYCLE_COUNT_LAP(prof_scope); uint64_t cur_offset = b_idx * q_head_num + q_idx * q_tile; diff --git a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/README.md b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/README.md index 3c4df9ffcd..6292926295 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/README.md +++ b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/README.md @@ -37,7 +37,7 @@ rewritten rather than adjusted. **64**. Dependencies are declared explicitly inside -`SIMPLER_SCOPE(PTO2ScopeMode::MANUAL)`, using the primitive +`SIMPLER_SCOPE(ScopeMode::MANUAL)`, using the primitive `set_dependencies(deps, n)` form throughout — [`../paged_attention_manual_scope/`](../paged_attention_manual_scope/) is the directory to read for that API on its own, against unchanged kernels. diff --git a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/kernels/orchestration/paged_attention_orch.cpp b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/kernels/orchestration/paged_attention_orch.cpp index f1cf6c9f12..78521f50ad 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/kernels/orchestration/paged_attention_orch.cpp +++ b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/kernels/orchestration/paged_attention_orch.cpp @@ -157,7 +157,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip for (uint64_t q_idx = 0; q_idx < q_loop; q_idx++) { CYCLE_COUNT_LAP(prof_scope_and_loop); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { uint64_t cur_offset = b_idx * q_head_num + q_idx * q_tile; uint32_t qi_shapes[2] = {static_cast(q_tile), static_cast(head_dim)}; diff --git a/examples/a2a3/tensormap_and_ringbuffer/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp b/examples/a2a3/tensormap_and_ringbuffer/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp index 501e6bd88a..73bdbfd8e9 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp +++ b/examples/a2a3/tensormap_and_ringbuffer/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp @@ -111,7 +111,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip TaskId prev_normed_tid[1]; for (int64_t __init_i = 0; __init_i < 1; ++__init_i) prev_normed_tid[__init_i] = TaskId::invalid(); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { TaskId _submit_deps_buf[1]; for (int64_t __init_i = 0; __init_i < 1; ++__init_i) _submit_deps_buf[__init_i] = TaskId::invalid(); @@ -228,7 +228,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip const ChipTensor &mlp_norm_in_inline71 = alloc_2.get_ref(5); const ChipTensor &inv_rms_tile_inline126 = alloc_2.get_ref(6); const ChipTensor &mlp_tile_inline149 = alloc_2.get_ref(7); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { // Phase-fence barrier 1: dependency-only dummy task CoreTaskArgs params_phase_fence_barrier_1; TaskOutputTensors phase_fence_barrier_1_outs = rt_submit_dummy_task(params_phase_fence_barrier_1); diff --git a/examples/a5/host_build_graph/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp b/examples/a5/host_build_graph/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp index f20155e09c..132df292a8 100644 --- a/examples/a5/host_build_graph/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp +++ b/examples/a5/host_build_graph/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp @@ -149,7 +149,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip TaskId prev_normed_tid[1]; for (int64_t __init_i = 0; __init_i < 1; ++__init_i) prev_normed_tid[__init_i] = TaskId::invalid(); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { TaskId _submit_deps_buf[1]; for (int64_t __init_i = 0; __init_i < 1; ++__init_i) _submit_deps_buf[__init_i] = TaskId::invalid(); @@ -335,7 +335,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip TaskId down_tids_inline156[85]; for (int64_t __init_i = 0; __init_i < 85; ++__init_i) down_tids_inline156[__init_i] = TaskId::invalid(); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { // Phase-fence barrier 1: dependency-only dummy task CoreTaskArgs params_phase_fence_barrier_1; TaskOutputTensors phase_fence_barrier_1_outs = diff --git a/examples/a5/tensormap_and_ringbuffer/paged_attention_manual_scope/kernels/orchestration/paged_attention_orch.cpp b/examples/a5/tensormap_and_ringbuffer/paged_attention_manual_scope/kernels/orchestration/paged_attention_orch.cpp index af67d826d1..4be52f3e8f 100644 --- a/examples/a5/tensormap_and_ringbuffer/paged_attention_manual_scope/kernels/orchestration/paged_attention_orch.cpp +++ b/examples/a5/tensormap_and_ringbuffer/paged_attention_manual_scope/kernels/orchestration/paged_attention_orch.cpp @@ -148,7 +148,7 @@ __attribute__((visibility("default"))) void build_paged_attention_graph(const Ch uint64_t cur_seq = static_cast(get_tensor_data(context_lens, 1, cl_idx)); uint64_t bn_this_batch = (cur_seq + block_size - 1) / block_size; for (uint64_t q_idx = 0; q_idx < q_loop; q_idx++) { - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { CYCLE_COUNT_LAP(prof_scope); uint64_t cur_offset = b_idx * q_head_num + q_idx * q_tile; diff --git a/examples/a5/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/kernels/orchestration/paged_attention_orch.cpp b/examples/a5/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/kernels/orchestration/paged_attention_orch.cpp index d575ee1031..d9737e5170 100644 --- a/examples/a5/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/kernels/orchestration/paged_attention_orch.cpp +++ b/examples/a5/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/kernels/orchestration/paged_attention_orch.cpp @@ -154,7 +154,7 @@ __attribute__((visibility("default"))) void build_paged_attention_graph(const Ch for (uint64_t q_idx = 0; q_idx < q_loop; q_idx++) { CYCLE_COUNT_LAP(prof_scope_and_loop); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { uint64_t cur_offset = b_idx * q_head_num + q_idx * q_tile; uint32_t qi_shapes[2] = {static_cast(q_tile), static_cast(head_dim)}; diff --git a/examples/a5/tensormap_and_ringbuffer/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp b/examples/a5/tensormap_and_ringbuffer/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp index 1d7e56210d..e262f56238 100644 --- a/examples/a5/tensormap_and_ringbuffer/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp +++ b/examples/a5/tensormap_and_ringbuffer/qwen3_14b_decode/kernels/orchestration/decode_fwd_layers.cpp @@ -111,7 +111,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip TaskId prev_normed_tid[1]; for (int64_t __init_i = 0; __init_i < 1; ++__init_i) prev_normed_tid[__init_i] = TaskId::invalid(); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { TaskId _submit_deps_buf[1]; for (int64_t __init_i = 0; __init_i < 1; ++__init_i) _submit_deps_buf[__init_i] = TaskId::invalid(); @@ -228,7 +228,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip const ChipTensor &mlp_norm_in_inline71 = alloc_2.get_ref(5); const ChipTensor &inv_rms_tile_inline126 = alloc_2.get_ref(6); const ChipTensor &mlp_tile_inline149 = alloc_2.get_ref(7); - SIMPLER_SCOPE(PTO2ScopeMode::MANUAL) { + SIMPLER_SCOPE(ScopeMode::MANUAL) { // Phase-fence barrier 1: dependency-only dummy task CoreTaskArgs params_phase_fence_barrier_1; TaskOutputTensors phase_fence_barrier_1_outs = rt_submit_dummy_task(params_phase_fence_barrier_1); diff --git a/src/a2a3/runtime/host_build_graph/orchestration/orchestration_api.h b/src/a2a3/runtime/host_build_graph/orchestration/orchestration_api.h index 4fb9394ead..903c7aa32e 100644 --- a/src/a2a3/runtime/host_build_graph/orchestration/orchestration_api.h +++ b/src/a2a3/runtime/host_build_graph/orchestration/orchestration_api.h @@ -132,7 +132,7 @@ typedef struct RuntimeOps { */ struct RuntimeContext { const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode; + ScopeMode pending_scope_mode; }; class GraphOwnedArgs { @@ -536,7 +536,7 @@ static inline void rt_graph_commit() { if (!rt->ops->is_fatal(rt) && rt->ops->graph_commit != nullptr) rt->ops->graph_commit(rt); } -static inline void rt_scope_begin(PTO2ScopeMode mode = PTO2ScopeMode::AUTO) { +static inline void rt_scope_begin(ScopeMode mode = ScopeMode::AUTO) { RuntimeContext *rt = current_runtime(); if (rt->ops->is_fatal(rt)) { return; @@ -683,7 +683,7 @@ static inline void set_tensor_data(const ChipTensor &tensor, uint32_t ndims, con */ class ScopeGuard { public: - explicit ScopeGuard(PTO2ScopeMode mode = PTO2ScopeMode::AUTO) : + explicit ScopeGuard(ScopeMode mode = ScopeMode::AUTO) : rt_(current_runtime()) { if (!rt_->ops->is_fatal(rt_)) { rt_->pending_scope_mode = mode; diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h index d4b433e7c2..4da4622db1 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator.h @@ -75,7 +75,7 @@ struct OrchestratorState { // one to be open; it bounds no task or buffer lifetime, so there is no // per-scope task list to walk at scope end. int32_t scope_stack_top{-1}; // Current top of stack (-1 = no scope open) - int32_t manual_begin_depth{PTO2_MAX_SCOPE_DEPTH}; + int32_t manual_begin_depth{CHIP_MAX_SCOPE_DEPTH}; // === SCHEDULER REFERENCE === // Note: In simulated mode, orchestrator and scheduler share address space @@ -143,7 +143,7 @@ struct OrchestratorState { void set_scheduler(SchedulerState *scheduler); void report_fatal(int32_t error_code, const char *func, const char *fmt, ...); - void begin_scope(PTO2ScopeMode mode = PTO2ScopeMode::AUTO); + void begin_scope(ScopeMode mode = ScopeMode::AUTO); void end_scope(); TaskOutputTensors submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args); TaskOutputTensors submit_dummy_task(const CoreTaskArgs &args); diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index f1d47ae637..c39851f37f 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -449,7 +449,7 @@ struct GraphRecording { // the manual-scope flag still has to follow the body: a manual scope // suppresses inference on the ring, so it must suppress it here too. int32_t scope_stack_top{-1}; - int32_t manual_begin_depth{PTO2_MAX_SCOPE_DEPTH}; + int32_t manual_begin_depth{CHIP_MAX_SCOPE_DEPTH}; bool in_manual_scope() const { return scope_stack_top >= manual_begin_depth; } @@ -718,7 +718,7 @@ bool graph_recording_reset(GraphRecording &recording, const GraphInflightRecordi recording.next_virtual_offset = 0; recording.unsupported = false; recording.scope_stack_top = -1; - recording.manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + recording.manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; // clear() keeps each array's capacity, which is the point: after the first recording // on this thread the arrays are already as large as this workload's bodies need. // nodes is deliberately not cleared: see GraphRecording::node_count. @@ -1339,7 +1339,7 @@ static bool prepare_task( // Scope Management // ============================================================================= -void OrchestratorState::begin_scope(PTO2ScopeMode mode) { +void OrchestratorState::begin_scope(ScopeMode mode) { auto *orch = this; if (orch->fatal) { return; @@ -1357,20 +1357,20 @@ void OrchestratorState::begin_scope(PTO2ScopeMode mode) { // body that ordinary submission refuses. The push still happens so // end_scope stays balanced -- the recording is doomed either way, since // graph_commit turns an unsupported recording into SIMPLER_ERROR_INVALID_ARGS. - if (recording->scope_stack_top >= PTO2_MAX_SCOPE_DEPTH - 1 || - (mode == PTO2ScopeMode::AUTO && recording->in_manual_scope())) { + if (recording->scope_stack_top >= CHIP_MAX_SCOPE_DEPTH - 1 || + (mode == ScopeMode::AUTO && recording->in_manual_scope())) { recording->unsupported = true; } - if (recording->scope_stack_top < PTO2_MAX_SCOPE_DEPTH - 1) { + if (recording->scope_stack_top < CHIP_MAX_SCOPE_DEPTH - 1) { ++recording->scope_stack_top; - if (mode == PTO2ScopeMode::MANUAL && !recording->in_manual_scope()) { + if (mode == ScopeMode::MANUAL && !recording->in_manual_scope()) { recording->manual_begin_depth = recording->scope_stack_top; } } return; } - assert(orch->scope_stack_top < PTO2_MAX_SCOPE_DEPTH - 1 && "Scope stack overflow"); - if (mode == PTO2ScopeMode::AUTO && orch->in_manual_scope()) { + assert(orch->scope_stack_top < CHIP_MAX_SCOPE_DEPTH - 1 && "Scope stack overflow"); + if (mode == ScopeMode::AUTO && orch->in_manual_scope()) { report_fatal( SIMPLER_ERROR_INVALID_ARGS, __FUNCTION__, "auto scope nested inside manual scope is not supported" ); @@ -1379,7 +1379,7 @@ void OrchestratorState::begin_scope(PTO2ScopeMode mode) { bool already_in_manual_scope = orch->in_manual_scope(); ++orch->scope_stack_top; - if (mode == PTO2ScopeMode::MANUAL && !already_in_manual_scope) { + if (mode == ScopeMode::MANUAL && !already_in_manual_scope) { orch->manual_begin_depth = orch->scope_stack_top; } } @@ -1394,7 +1394,7 @@ void OrchestratorState::end_scope() { if (GraphRecording *recording = active_graph_recording(orch); recording != nullptr) { if (recording->scope_stack_top >= 0) { if (recording->manual_begin_depth == recording->scope_stack_top) { - recording->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + recording->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; } --recording->scope_stack_top; } @@ -1403,7 +1403,7 @@ void OrchestratorState::end_scope() { assert(orch->scope_stack_top >= 0 && "Scope stack underflow"); if (orch->scope_stack_top == orch->manual_begin_depth) { - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; } --orch->scope_stack_top; } @@ -2781,7 +2781,7 @@ void OrchestratorState::mark_done() { } orch->sm_header->orchestrator_done.store(1, std::memory_order_release); orch->scope_stack_top = -1; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; #if !SIMPLER_ORCH_PROFILING && SIMPLER_DFX g_orch_submit_idx = 0; #endif diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 8653e6e327..553c64f8fa 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -124,8 +124,8 @@ static void graph_commit_impl(RuntimeContext *rt) { } void rt_scope_begin(RuntimeContext *rt) { - PTO2ScopeMode mode = rt->pending_scope_mode; - rt->pending_scope_mode = PTO2ScopeMode::AUTO; + ScopeMode mode = rt->pending_scope_mode; + rt->pending_scope_mode = ScopeMode::AUTO; rt->orchestrator->begin_scope(mode); } diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h index 25a7863cd9..ef0b28666c 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h @@ -167,7 +167,7 @@ struct RuntimeArenaLayout { struct RuntimeContext { // Ops table (first field — used by orchestration .so via function pointers) const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode; + ScopeMode pending_scope_mode; // Components SharedMemoryHandle *sm_handle; diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index d7f76ac405..0072b3834b 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -117,7 +117,7 @@ inline constexpr uint64_t GRAPH_RECORD_VIRTUAL_BASE = 1ULL << 63; inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HEAP_VIRTUAL_BASE; // Scope management -#define PTO2_MAX_SCOPE_DEPTH 64 // Maximum nesting depth +#define CHIP_MAX_SCOPE_DEPTH 64 // Maximum nesting depth // Per-shape ready-queue capacity (power of two). This is a ring buffer that // bounds peak CONCURRENT occupancy (enqueue_pos - dequeue_pos), not total task diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 0d6ce3571a..11dda9bb49 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -238,7 +238,7 @@ bool OrchestratorState::init( } orch->scope_stack_top = -1; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; return true; } diff --git a/src/a2a3/runtime/host_build_graph/runtime/types.h b/src/a2a3/runtime/host_build_graph/runtime/types.h index e7ca055336..e6106e1bd4 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/types.h @@ -67,7 +67,7 @@ enum class CompletionType : int32_t { // Task Output ChipTensors (return value from submit) // ============================================================================= -enum class PTO2ScopeMode : uint8_t { +enum class ScopeMode : uint8_t { AUTO = 0, MANUAL = 1, }; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h b/src/a2a3/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h index 1225fff11b..c89a2e2fb8 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h @@ -108,7 +108,7 @@ typedef struct RuntimeOps { */ struct RuntimeContext { const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode; + ScopeMode pending_scope_mode; }; // ============================================================================= @@ -208,7 +208,7 @@ static inline TaskOutputTensors rt_submit_dummy_task(const CoreTaskArgs &args) { return rt->ops->submit_dummy_task(rt, args); } -static inline void rt_scope_begin(PTO2ScopeMode mode = PTO2ScopeMode::AUTO) { +static inline void rt_scope_begin(ScopeMode mode = ScopeMode::AUTO) { RuntimeContext *rt = current_runtime(); if (rt->ops->is_fatal(rt)) { return; @@ -335,7 +335,7 @@ static inline void set_tensor_data(const ChipTensor &tensor, uint32_t ndims, con class ScopeGuard { public: explicit ScopeGuard( - PTO2ScopeMode mode = PTO2ScopeMode::AUTO, const char *file = __builtin_FILE(), int line = __builtin_LINE() + ScopeMode mode = ScopeMode::AUTO, const char *file = __builtin_FILE(), int line = __builtin_LINE() ) : rt_(current_runtime()) { if (!rt_->ops->is_fatal(rt_)) { diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 51a2c8f7da..77a0181546 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -684,13 +684,13 @@ static void scope_tasks_push(OrchestratorState *orch, ChipTaskSlotState *task_sl orch->scope_tasks[orch->scope_tasks_size++] = task_slot_state; } -void OrchestratorState::begin_scope(PTO2ScopeMode mode) { +void OrchestratorState::begin_scope(ScopeMode mode) { auto *orch = this; if (orch->fatal) { return; } assert(orch->scope_stack_top < static_cast(orch->scope_stack_capacity - 1) && "Scope stack overflow"); - if (mode == PTO2ScopeMode::AUTO && orch->in_manual_scope()) { + if (mode == ScopeMode::AUTO && orch->in_manual_scope()) { report_fatal( SIMPLER_ERROR_INVALID_ARGS, __FUNCTION__, "auto scope nested inside manual scope is not supported" ); @@ -700,7 +700,7 @@ void OrchestratorState::begin_scope(PTO2ScopeMode mode) { bool already_in_manual_scope = orch->in_manual_scope(); ++orch->scope_stack_top; orch->scope_begins[orch->scope_stack_top] = orch->scope_tasks_size; - if (mode == PTO2ScopeMode::MANUAL && !already_in_manual_scope) { + if (mode == ScopeMode::MANUAL && !already_in_manual_scope) { orch->manual_begin_depth = orch->scope_stack_top; } #if SIMPLER_DFX @@ -759,7 +759,7 @@ void OrchestratorState::end_scope() { int32_t begin = orch->scope_begins[orch->scope_stack_top--]; int32_t count = orch->scope_tasks_size - begin; if (ending_manual_scope) { - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; } if (orch->scheduler && count > 0) { @@ -1383,7 +1383,7 @@ void OrchestratorState::mark_done() { orch->sm_header->orchestrator_done.store(1, std::memory_order_release); orch->scope_tasks_size = 0; orch->scope_stack_top = -1; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; #if !SIMPLER_ORCH_PROFILING && SIMPLER_DFX g_orch_submit_idx = 0; #endif diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 7782ce32db..397442f2be 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -83,8 +83,8 @@ struct OrchestratorState { int32_t scope_tasks_capacity; // Allocated capacity of scope_tasks int32_t *scope_begins; // scope_begins[i] = start index of scope i in scope_tasks int32_t scope_stack_top; // Current top of stack (-1 = no scope open) - uint64_t scope_stack_capacity; // Max nesting depth (PTO2_MAX_SCOPE_DEPTH) - int32_t manual_begin_depth{PTO2_MAX_SCOPE_DEPTH}; + uint64_t scope_stack_capacity; // Max nesting depth (CHIP_MAX_SCOPE_DEPTH) + int32_t manual_begin_depth{CHIP_MAX_SCOPE_DEPTH}; // === SCHEDULER STATE ACCESS === // Same runtime-arena scheduler object; Orch-side wiring mutates dep_pool @@ -177,7 +177,7 @@ struct OrchestratorState { void mark_dep_pool_position(ChipTaskSlotState &slot_state); void wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin); void report_fatal(int32_t error_code, const char *func, const char *fmt, ...); - void begin_scope(PTO2ScopeMode mode = PTO2ScopeMode::AUTO); + void begin_scope(ScopeMode mode = ScopeMode::AUTO); void end_scope(); TaskOutputTensors submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args); TaskOutputTensors submit_dummy_task(const CoreTaskArgs &args); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index e3485bec80..174e2879d7 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -63,8 +63,8 @@ static TaskOutputTensors submit_dummy_task_impl(RuntimeContext *rt, const CoreTa } void rt_scope_begin(RuntimeContext *rt) { - PTO2ScopeMode mode = rt->pending_scope_mode; - rt->pending_scope_mode = PTO2ScopeMode::AUTO; + ScopeMode mode = rt->pending_scope_mode; + rt->pending_scope_mode = ScopeMode::AUTO; rt->orchestrator.begin_scope(mode); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index 7d2c7990aa..2f643bd1b9 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -153,7 +153,7 @@ struct RuntimeArenaLayout { struct RuntimeContext { // Ops table (first field — used by orchestration .so via function pointers) const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode; + ScopeMode pending_scope_mode; // Components SharedMemoryHandle *sm_handle; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 7059f5349b..bbe11d343b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -75,13 +75,13 @@ #define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) // Scope management -#define PTO2_MAX_SCOPE_DEPTH 64 // Maximum nesting depth +#define CHIP_MAX_SCOPE_DEPTH 64 // Maximum nesting depth // Hard cap for the scope_tasks buffer. Equals the total in-flight ring slot // budget (PTO2_TASK_WINDOW_SIZE × CHIP_MAX_RING_DEPTH): once every ring slot // is in flight, no more tasks can ever be pushed regardless of buffer size. // scope_tasks_push fatals on overflow rather than growing the arena-owned // buffer (which would be UB on the arena's malloc'd backing). -#define PTO2_SCOPE_TASKS_CAP (PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) +#define CHIP_SCOPE_TASKS_CAP (PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) // Ready queue #define CHIP_READY_QUEUE_SIZE 65536 // Per-shape queue size diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index d0a4ff73fa..b096f97eb3 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -1183,7 +1183,7 @@ int32_t SchedulerContext::pre_handshake_init( int64_t pto2_count = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { int32_t ring_tasks = header->rings[r].fc.current_task_index.load(std::memory_order_acquire); - if (ring_tasks > 0 && ring_tasks <= PTO2_SCOPE_TASKS_CAP) pto2_count += ring_tasks; + if (ring_tasks > 0 && ring_tasks <= CHIP_SCOPE_TASKS_CAP) pto2_count += ring_tasks; } total_tasks_ = static_cast(pto2_count); } else { diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index f35f9588ad..bb872ab8f7 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -305,7 +305,7 @@ OrchestratorLayout OrchestratorState::reserve_layout( OrchestratorLayout layout{}; // scope_tasks holds every task in the open scope across all rings, so its cap // is the real in-flight budget = sum of the (runtime) per-ring windows. Using - // the compile-time PTO2_SCOPE_TASKS_CAP instead under-sized the buffer when + // the compile-time CHIP_SCOPE_TASKS_CAP instead under-sized the buffer when // ring_task_window was enlarged past the default (premature SCOPE_TASKS_OVERFLOW) // and over-allocated it when shrunk. See issue #1188. // @@ -319,7 +319,7 @@ OrchestratorLayout OrchestratorState::reserve_layout( } always_assert(scope_tasks_cap <= std::numeric_limits::max()); layout.scope_tasks_cap = static_cast(scope_tasks_cap); - layout.scope_stack_capacity = PTO2_MAX_SCOPE_DEPTH; + layout.scope_stack_capacity = CHIP_MAX_SCOPE_DEPTH; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { layout.dep_pool_capacities[r] = dep_pool_capacities[r]; } @@ -409,7 +409,7 @@ bool OrchestratorState::init_data_from_layout( orch->scope_tasks_capacity = layout.scope_tasks_cap; orch->scope_stack_top = -1; orch->scope_stack_capacity = layout.scope_stack_capacity; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; return true; } @@ -463,7 +463,7 @@ bool OrchestratorState::reset_for_reuse( orch->scope_tasks_capacity = layout.scope_tasks_cap; orch->scope_stack_top = -1; orch->scope_stack_capacity = layout.scope_stack_capacity; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; orch->total_cluster_count = 0; orch->total_aiv_count = 0; #if SIMPLER_DFX @@ -603,7 +603,7 @@ bool runtime_reset_for_reuse(DeviceArena &arena, const RuntimeArenaLayout &layou return false; } - rt->pending_scope_mode = PTO2ScopeMode::AUTO; + rt->pending_scope_mode = ScopeMode::AUTO; rt->total_cycles = 0; rt->gm_heap_owned = false; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h index b2b9149036..8c4785b1e8 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h @@ -65,7 +65,7 @@ enum class CompletionType : int32_t { // Task Output ChipTensors (return value from submit) // ============================================================================= -enum class PTO2ScopeMode : uint8_t { +enum class ScopeMode : uint8_t { AUTO = 0, MANUAL = 1, }; diff --git a/src/a5/runtime/host_build_graph/orchestration/orchestration_api.h b/src/a5/runtime/host_build_graph/orchestration/orchestration_api.h index 4fb9394ead..903c7aa32e 100644 --- a/src/a5/runtime/host_build_graph/orchestration/orchestration_api.h +++ b/src/a5/runtime/host_build_graph/orchestration/orchestration_api.h @@ -132,7 +132,7 @@ typedef struct RuntimeOps { */ struct RuntimeContext { const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode; + ScopeMode pending_scope_mode; }; class GraphOwnedArgs { @@ -536,7 +536,7 @@ static inline void rt_graph_commit() { if (!rt->ops->is_fatal(rt) && rt->ops->graph_commit != nullptr) rt->ops->graph_commit(rt); } -static inline void rt_scope_begin(PTO2ScopeMode mode = PTO2ScopeMode::AUTO) { +static inline void rt_scope_begin(ScopeMode mode = ScopeMode::AUTO) { RuntimeContext *rt = current_runtime(); if (rt->ops->is_fatal(rt)) { return; @@ -683,7 +683,7 @@ static inline void set_tensor_data(const ChipTensor &tensor, uint32_t ndims, con */ class ScopeGuard { public: - explicit ScopeGuard(PTO2ScopeMode mode = PTO2ScopeMode::AUTO) : + explicit ScopeGuard(ScopeMode mode = ScopeMode::AUTO) : rt_(current_runtime()) { if (!rt_->ops->is_fatal(rt_)) { rt_->pending_scope_mode = mode; diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator.h b/src/a5/runtime/host_build_graph/runtime/orchestrator.h index 7a9d17cda4..27dfbebd45 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator.h +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator.h @@ -75,7 +75,7 @@ struct OrchestratorState { // one to be open; it bounds no task or buffer lifetime, so there is no // per-scope task list to walk at scope end. int32_t scope_stack_top{-1}; // Current top of stack (-1 = no scope open) - int32_t manual_begin_depth{PTO2_MAX_SCOPE_DEPTH}; + int32_t manual_begin_depth{CHIP_MAX_SCOPE_DEPTH}; // === SCHEDULER REFERENCE === // Note: In simulated mode, orchestrator and scheduler share address space @@ -142,7 +142,7 @@ struct OrchestratorState { void set_scheduler(SchedulerState *scheduler); void report_fatal(int32_t error_code, const char *func, const char *fmt, ...); - void begin_scope(PTO2ScopeMode mode = PTO2ScopeMode::AUTO); + void begin_scope(ScopeMode mode = ScopeMode::AUTO); void end_scope(); TaskOutputTensors submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args); TaskOutputTensors submit_dummy_task(const CoreTaskArgs &args); diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index f1d47ae637..c39851f37f 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -449,7 +449,7 @@ struct GraphRecording { // the manual-scope flag still has to follow the body: a manual scope // suppresses inference on the ring, so it must suppress it here too. int32_t scope_stack_top{-1}; - int32_t manual_begin_depth{PTO2_MAX_SCOPE_DEPTH}; + int32_t manual_begin_depth{CHIP_MAX_SCOPE_DEPTH}; bool in_manual_scope() const { return scope_stack_top >= manual_begin_depth; } @@ -718,7 +718,7 @@ bool graph_recording_reset(GraphRecording &recording, const GraphInflightRecordi recording.next_virtual_offset = 0; recording.unsupported = false; recording.scope_stack_top = -1; - recording.manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + recording.manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; // clear() keeps each array's capacity, which is the point: after the first recording // on this thread the arrays are already as large as this workload's bodies need. // nodes is deliberately not cleared: see GraphRecording::node_count. @@ -1339,7 +1339,7 @@ static bool prepare_task( // Scope Management // ============================================================================= -void OrchestratorState::begin_scope(PTO2ScopeMode mode) { +void OrchestratorState::begin_scope(ScopeMode mode) { auto *orch = this; if (orch->fatal) { return; @@ -1357,20 +1357,20 @@ void OrchestratorState::begin_scope(PTO2ScopeMode mode) { // body that ordinary submission refuses. The push still happens so // end_scope stays balanced -- the recording is doomed either way, since // graph_commit turns an unsupported recording into SIMPLER_ERROR_INVALID_ARGS. - if (recording->scope_stack_top >= PTO2_MAX_SCOPE_DEPTH - 1 || - (mode == PTO2ScopeMode::AUTO && recording->in_manual_scope())) { + if (recording->scope_stack_top >= CHIP_MAX_SCOPE_DEPTH - 1 || + (mode == ScopeMode::AUTO && recording->in_manual_scope())) { recording->unsupported = true; } - if (recording->scope_stack_top < PTO2_MAX_SCOPE_DEPTH - 1) { + if (recording->scope_stack_top < CHIP_MAX_SCOPE_DEPTH - 1) { ++recording->scope_stack_top; - if (mode == PTO2ScopeMode::MANUAL && !recording->in_manual_scope()) { + if (mode == ScopeMode::MANUAL && !recording->in_manual_scope()) { recording->manual_begin_depth = recording->scope_stack_top; } } return; } - assert(orch->scope_stack_top < PTO2_MAX_SCOPE_DEPTH - 1 && "Scope stack overflow"); - if (mode == PTO2ScopeMode::AUTO && orch->in_manual_scope()) { + assert(orch->scope_stack_top < CHIP_MAX_SCOPE_DEPTH - 1 && "Scope stack overflow"); + if (mode == ScopeMode::AUTO && orch->in_manual_scope()) { report_fatal( SIMPLER_ERROR_INVALID_ARGS, __FUNCTION__, "auto scope nested inside manual scope is not supported" ); @@ -1379,7 +1379,7 @@ void OrchestratorState::begin_scope(PTO2ScopeMode mode) { bool already_in_manual_scope = orch->in_manual_scope(); ++orch->scope_stack_top; - if (mode == PTO2ScopeMode::MANUAL && !already_in_manual_scope) { + if (mode == ScopeMode::MANUAL && !already_in_manual_scope) { orch->manual_begin_depth = orch->scope_stack_top; } } @@ -1394,7 +1394,7 @@ void OrchestratorState::end_scope() { if (GraphRecording *recording = active_graph_recording(orch); recording != nullptr) { if (recording->scope_stack_top >= 0) { if (recording->manual_begin_depth == recording->scope_stack_top) { - recording->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + recording->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; } --recording->scope_stack_top; } @@ -1403,7 +1403,7 @@ void OrchestratorState::end_scope() { assert(orch->scope_stack_top >= 0 && "Scope stack underflow"); if (orch->scope_stack_top == orch->manual_begin_depth) { - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; } --orch->scope_stack_top; } @@ -2781,7 +2781,7 @@ void OrchestratorState::mark_done() { } orch->sm_header->orchestrator_done.store(1, std::memory_order_release); orch->scope_stack_top = -1; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; #if !SIMPLER_ORCH_PROFILING && SIMPLER_DFX g_orch_submit_idx = 0; #endif diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 8653e6e327..553c64f8fa 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -124,8 +124,8 @@ static void graph_commit_impl(RuntimeContext *rt) { } void rt_scope_begin(RuntimeContext *rt) { - PTO2ScopeMode mode = rt->pending_scope_mode; - rt->pending_scope_mode = PTO2ScopeMode::AUTO; + ScopeMode mode = rt->pending_scope_mode; + rt->pending_scope_mode = ScopeMode::AUTO; rt->orchestrator->begin_scope(mode); } diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_core.h b/src/a5/runtime/host_build_graph/runtime/runtime_core.h index 22a739a68c..9fba17166b 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_core.h @@ -168,7 +168,7 @@ struct RuntimeArenaLayout { struct RuntimeContext { // Ops table (first field — used by orchestration .so via function pointers) const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode; + ScopeMode pending_scope_mode; // Components SharedMemoryHandle *sm_handle; diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 8e71c2553b..5d023ab729 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -118,7 +118,7 @@ inline constexpr uint64_t GRAPH_RECORD_VIRTUAL_BASE = 1ULL << 63; inline constexpr uint64_t HEAP_VIRTUAL_CAPACITY = GRAPH_RECORD_VIRTUAL_BASE - HEAP_VIRTUAL_BASE; // Scope management -#define PTO2_MAX_SCOPE_DEPTH 64 // Maximum nesting depth +#define CHIP_MAX_SCOPE_DEPTH 64 // Maximum nesting depth // Per-shape ready-queue capacity (power of two). This is a ring buffer that // bounds peak CONCURRENT occupancy (enqueue_pos - dequeue_pos), not total task diff --git a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 0d6ce3571a..11dda9bb49 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -238,7 +238,7 @@ bool OrchestratorState::init( } orch->scope_stack_top = -1; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; return true; } diff --git a/src/a5/runtime/host_build_graph/runtime/types.h b/src/a5/runtime/host_build_graph/runtime/types.h index e7ca055336..e6106e1bd4 100644 --- a/src/a5/runtime/host_build_graph/runtime/types.h +++ b/src/a5/runtime/host_build_graph/runtime/types.h @@ -67,7 +67,7 @@ enum class CompletionType : int32_t { // Task Output ChipTensors (return value from submit) // ============================================================================= -enum class PTO2ScopeMode : uint8_t { +enum class ScopeMode : uint8_t { AUTO = 0, MANUAL = 1, }; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h b/src/a5/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h index 1225fff11b..c89a2e2fb8 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h @@ -108,7 +108,7 @@ typedef struct RuntimeOps { */ struct RuntimeContext { const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode; + ScopeMode pending_scope_mode; }; // ============================================================================= @@ -208,7 +208,7 @@ static inline TaskOutputTensors rt_submit_dummy_task(const CoreTaskArgs &args) { return rt->ops->submit_dummy_task(rt, args); } -static inline void rt_scope_begin(PTO2ScopeMode mode = PTO2ScopeMode::AUTO) { +static inline void rt_scope_begin(ScopeMode mode = ScopeMode::AUTO) { RuntimeContext *rt = current_runtime(); if (rt->ops->is_fatal(rt)) { return; @@ -335,7 +335,7 @@ static inline void set_tensor_data(const ChipTensor &tensor, uint32_t ndims, con class ScopeGuard { public: explicit ScopeGuard( - PTO2ScopeMode mode = PTO2ScopeMode::AUTO, const char *file = __builtin_FILE(), int line = __builtin_LINE() + ScopeMode mode = ScopeMode::AUTO, const char *file = __builtin_FILE(), int line = __builtin_LINE() ) : rt_(current_runtime()) { if (!rt_->ops->is_fatal(rt_)) { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index cebe9f4b84..9d05e79008 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -672,13 +672,13 @@ static void scope_tasks_push(OrchestratorState *orch, ChipTaskSlotState *task_sl orch->scope_tasks[orch->scope_tasks_size++] = task_slot_state; } -void OrchestratorState::begin_scope(PTO2ScopeMode mode) { +void OrchestratorState::begin_scope(ScopeMode mode) { auto *orch = this; if (orch->fatal) { return; } assert(orch->scope_stack_top < static_cast(orch->scope_stack_capacity - 1) && "Scope stack overflow"); - if (mode == PTO2ScopeMode::AUTO && orch->in_manual_scope()) { + if (mode == ScopeMode::AUTO && orch->in_manual_scope()) { report_fatal( SIMPLER_ERROR_INVALID_ARGS, __FUNCTION__, "auto scope nested inside manual scope is not supported" ); @@ -688,7 +688,7 @@ void OrchestratorState::begin_scope(PTO2ScopeMode mode) { bool already_in_manual_scope = orch->in_manual_scope(); ++orch->scope_stack_top; orch->scope_begins[orch->scope_stack_top] = orch->scope_tasks_size; - if (mode == PTO2ScopeMode::MANUAL && !already_in_manual_scope) { + if (mode == ScopeMode::MANUAL && !already_in_manual_scope) { orch->manual_begin_depth = orch->scope_stack_top; } #if SIMPLER_DFX @@ -747,7 +747,7 @@ void OrchestratorState::end_scope() { int32_t begin = orch->scope_begins[orch->scope_stack_top--]; int32_t count = orch->scope_tasks_size - begin; if (ending_manual_scope) { - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; } if (orch->scheduler && count > 0) { @@ -1385,7 +1385,7 @@ void OrchestratorState::mark_done() { orch->sm_header->orchestrator_done.store(1, std::memory_order_release); orch->scope_tasks_size = 0; orch->scope_stack_top = -1; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; #if !SIMPLER_ORCH_PROFILING && SIMPLER_DFX g_orch_submit_idx = 0; #endif diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h index 7782ce32db..397442f2be 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.h @@ -83,8 +83,8 @@ struct OrchestratorState { int32_t scope_tasks_capacity; // Allocated capacity of scope_tasks int32_t *scope_begins; // scope_begins[i] = start index of scope i in scope_tasks int32_t scope_stack_top; // Current top of stack (-1 = no scope open) - uint64_t scope_stack_capacity; // Max nesting depth (PTO2_MAX_SCOPE_DEPTH) - int32_t manual_begin_depth{PTO2_MAX_SCOPE_DEPTH}; + uint64_t scope_stack_capacity; // Max nesting depth (CHIP_MAX_SCOPE_DEPTH) + int32_t manual_begin_depth{CHIP_MAX_SCOPE_DEPTH}; // === SCHEDULER STATE ACCESS === // Same runtime-arena scheduler object; Orch-side wiring mutates dep_pool @@ -177,7 +177,7 @@ struct OrchestratorState { void mark_dep_pool_position(ChipTaskSlotState &slot_state); void wire_fanin_task(ChipTaskSlotState &slot_state, int32_t wfanin); void report_fatal(int32_t error_code, const char *func, const char *fmt, ...); - void begin_scope(PTO2ScopeMode mode = PTO2ScopeMode::AUTO); + void begin_scope(ScopeMode mode = ScopeMode::AUTO); void end_scope(); TaskOutputTensors submit_task(const MixedKernels &mixed_kernels, const CoreTaskArgs &args); TaskOutputTensors submit_dummy_task(const CoreTaskArgs &args); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index e3485bec80..174e2879d7 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -63,8 +63,8 @@ static TaskOutputTensors submit_dummy_task_impl(RuntimeContext *rt, const CoreTa } void rt_scope_begin(RuntimeContext *rt) { - PTO2ScopeMode mode = rt->pending_scope_mode; - rt->pending_scope_mode = PTO2ScopeMode::AUTO; + ScopeMode mode = rt->pending_scope_mode; + rt->pending_scope_mode = ScopeMode::AUTO; rt->orchestrator.begin_scope(mode); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index 7d2c7990aa..2f643bd1b9 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -153,7 +153,7 @@ struct RuntimeArenaLayout { struct RuntimeContext { // Ops table (first field — used by orchestration .so via function pointers) const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode; + ScopeMode pending_scope_mode; // Components SharedMemoryHandle *sm_handle; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index c662966e50..13ba25cc28 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -75,13 +75,13 @@ #define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) // Scope management -#define PTO2_MAX_SCOPE_DEPTH 64 // Maximum nesting depth +#define CHIP_MAX_SCOPE_DEPTH 64 // Maximum nesting depth // Hard cap for the scope_tasks buffer. Equals the total in-flight ring slot // budget (PTO2_TASK_WINDOW_SIZE × CHIP_MAX_RING_DEPTH): once every ring slot // is in flight, no more tasks can ever be pushed regardless of buffer size. // scope_tasks_push fatals on overflow rather than growing the arena-owned // buffer (which would be UB on the arena's malloc'd backing). -#define PTO2_SCOPE_TASKS_CAP (PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) +#define CHIP_SCOPE_TASKS_CAP (PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) // Ready queue #define CHIP_READY_QUEUE_SIZE 65536 // Per-shape queue size diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 364b72adb9..6388461101 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -1164,7 +1164,7 @@ int32_t SchedulerContext::pre_handshake_init( int64_t task_count = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { int32_t ring_tasks = header->rings[r].fc.current_task_index.load(std::memory_order_acquire); - if (ring_tasks > 0 && ring_tasks <= PTO2_SCOPE_TASKS_CAP) task_count += ring_tasks; + if (ring_tasks > 0 && ring_tasks <= CHIP_SCOPE_TASKS_CAP) task_count += ring_tasks; } total_tasks_ = static_cast(task_count); } else { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 0ebb948cfe..0ce29309ca 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -322,7 +322,7 @@ OrchestratorLayout OrchestratorState::reserve_layout( OrchestratorLayout layout{}; // scope_tasks holds every task in the open scope across all rings, so its cap // is the real in-flight budget = sum of the (runtime) per-ring windows. Using - // the compile-time PTO2_SCOPE_TASKS_CAP instead under-sized the buffer when + // the compile-time CHIP_SCOPE_TASKS_CAP instead under-sized the buffer when // ring_task_window was enlarged past the default (premature SCOPE_TASKS_OVERFLOW) // and over-allocated it when shrunk. See issue #1188. // @@ -336,7 +336,7 @@ OrchestratorLayout OrchestratorState::reserve_layout( } always_assert(scope_tasks_cap <= std::numeric_limits::max()); layout.scope_tasks_cap = static_cast(scope_tasks_cap); - layout.scope_stack_capacity = PTO2_MAX_SCOPE_DEPTH; + layout.scope_stack_capacity = CHIP_MAX_SCOPE_DEPTH; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { layout.dep_pool_capacities[r] = dep_pool_capacities[r]; } @@ -426,7 +426,7 @@ bool OrchestratorState::init_data_from_layout( orch->scope_tasks_capacity = layout.scope_tasks_cap; orch->scope_stack_top = -1; orch->scope_stack_capacity = layout.scope_stack_capacity; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; return true; } @@ -485,7 +485,7 @@ bool OrchestratorState::reset_for_reuse( orch->scope_tasks_capacity = layout.scope_tasks_cap; orch->scope_stack_top = -1; orch->scope_stack_capacity = layout.scope_stack_capacity; - orch->manual_begin_depth = PTO2_MAX_SCOPE_DEPTH; + orch->manual_begin_depth = CHIP_MAX_SCOPE_DEPTH; orch->total_cluster_count = 0; orch->total_aiv_count = 0; #if SIMPLER_DFX @@ -664,7 +664,7 @@ bool runtime_reset_for_reuse(DeviceArena &arena, const RuntimeArenaLayout &layou rt->scheduler.set_publication_batching_enabled(false); - rt->pending_scope_mode = PTO2ScopeMode::AUTO; + rt->pending_scope_mode = ScopeMode::AUTO; rt->total_cycles = 0; rt->gm_heap_owned = false; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h index 2fb26df9d7..7faae66a16 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h @@ -66,7 +66,7 @@ enum class CompletionType : int32_t { // Task Output ChipTensors (return value from submit) // ============================================================================= -enum class PTO2ScopeMode : uint8_t { +enum class ScopeMode : uint8_t { AUTO = 0, MANUAL = 1, }; diff --git a/src/common/hierarchical/types.h b/src/common/hierarchical/types.h index 50b272bac6..d25c5ee3e4 100644 --- a/src/common/hierarchical/types.h +++ b/src/common/hierarchical/types.h @@ -113,7 +113,7 @@ struct TensorKeyHash { // Constants // ============================================================================= -// User-visible scope-nesting cap. Matches L2 PTO2_MAX_SCOPE_DEPTH. +// User-visible scope-nesting cap. Matches L2 CHIP_MAX_SCOPE_DEPTH. static constexpr int32_t MAX_SCOPE_DEPTH = 64; // Number of independent HeapRing layers inside Ring. Scope depth maps diff --git a/src/common/platform/include/common/scope_stats.h b/src/common/platform/include/common/scope_stats.h index 2cccd6bf98..9512ef77b8 100644 --- a/src/common/platform/include/common/scope_stats.h +++ b/src/common/platform/include/common/scope_stats.h @@ -50,8 +50,8 @@ #include "common/dfx_backpressure_device.h" #include "common/platform_config.h" -#define PTO2_SCOPE_STATS_MAX_RING_DEPTH 4 -#define PTO2_SCOPE_STATS_MAX_SCOPE_DEPTH 64 +#define SCOPE_STATS_MAX_RING_DEPTH 4 +#define SCOPE_STATS_MAX_SCOPE_DEPTH 64 #ifdef __cplusplus extern "C" { @@ -160,9 +160,9 @@ struct ScopeStatsDataHeader { // Per-ring static capacities — written once by AICPU at orchestrator init // (scope_stats_set_ring_capacity / scope_stats_set_tensormap_capacity). // Host needs them to render the "used/cap" ratio without a separate query. - int32_t task_window_cap[PTO2_SCOPE_STATS_MAX_RING_DEPTH]; - int32_t dep_pool_cap[PTO2_SCOPE_STATS_MAX_RING_DEPTH]; - uint64_t heap_cap[PTO2_SCOPE_STATS_MAX_RING_DEPTH]; + int32_t task_window_cap[SCOPE_STATS_MAX_RING_DEPTH]; + int32_t dep_pool_cap[SCOPE_STATS_MAX_RING_DEPTH]; + uint64_t heap_cap[SCOPE_STATS_MAX_RING_DEPTH]; int32_t tensormap_cap; volatile uint32_t fatal_latched; // AICPU sets to 1 on first fatal. } __attribute__((aligned(64))); diff --git a/src/common/platform/shared/aicpu/scope_stats_collector_aicpu.cpp b/src/common/platform/shared/aicpu/scope_stats_collector_aicpu.cpp index 6e92b2cb9a..c1ffa95f07 100644 --- a/src/common/platform/shared/aicpu/scope_stats_collector_aicpu.cpp +++ b/src/common/platform/shared/aicpu/scope_stats_collector_aicpu.cpp @@ -43,7 +43,7 @@ static int s_orch_thread_idx = -1; // set via scope_stats_aicpu_set_orch_thread // Fed by the runtime via scope_stats_note_heap_wrap at each wrap; used to unroll // the boundary-sampled wrapping offsets into monotonic values (see // unroll_heap_offset). Reset in set_platform_scope_stats_base. -static uint64_t s_heap_wraps[PTO2_SCOPE_STATS_MAX_RING_DEPTH][2] = {}; +static uint64_t s_heap_wraps[SCOPE_STATS_MAX_RING_DEPTH][2] = {}; static constexpr uint64_t kScopeStatsQueueBackpressureWaitCycles = PLATFORM_DFX_BACKPRESSURE_TIMEOUT_CYCLES; @@ -52,11 +52,11 @@ namespace { const char *s_pending_site_file = nullptr; int32_t s_pending_site_line = 0; -const char *s_scope_site_file[PTO2_SCOPE_STATS_MAX_SCOPE_DEPTH] = {}; -int32_t s_scope_site_line[PTO2_SCOPE_STATS_MAX_SCOPE_DEPTH] = {}; +const char *s_scope_site_file[SCOPE_STATS_MAX_SCOPE_DEPTH] = {}; +int32_t s_scope_site_line[SCOPE_STATS_MAX_SCOPE_DEPTH] = {}; // Ring id of each open scope, recorded at scope_stats_begin. Lets a heap-wrap // report (which carries no ring id) be attributed to the current scope's ring. -int s_scope_ring[PTO2_SCOPE_STATS_MAX_SCOPE_DEPTH] = {}; +int s_scope_ring[SCOPE_STATS_MAX_SCOPE_DEPTH] = {}; inline const char *basename_of(const char *path) { if (!path) return "(unknown)"; @@ -160,7 +160,7 @@ void switch_buffer() { ScopeStatsEngine::switch_buffer(scope_stats_context(), s_ // scope_alloc, high-water) is an exact subtraction regardless of wrap count. inline uint64_t unroll_heap_offset(uint64_t offset, int ring_id, int side) { if (s_scope_stats_header == nullptr) return offset; - if (ring_id < 0 || ring_id >= PTO2_SCOPE_STATS_MAX_RING_DEPTH) return offset; + if (ring_id < 0 || ring_id >= SCOPE_STATS_MAX_RING_DEPTH) return offset; return offset + s_heap_wraps[ring_id][side] * s_scope_stats_header->heap_cap[ring_id]; } @@ -285,7 +285,7 @@ extern "C" void scope_stats_begin( int32_t dep_pool_end, int32_t tensormap_used ) { if (!scope_stats_enabled) return; - if (scope_stats_depth + 1 >= PTO2_SCOPE_STATS_MAX_SCOPE_DEPTH) return; + if (scope_stats_depth + 1 >= SCOPE_STATS_MAX_SCOPE_DEPTH) return; int32_t d = ++scope_stats_depth; s_scope_site_file[d] = s_pending_site_file; s_scope_site_line[d] = s_pending_site_line; @@ -331,7 +331,7 @@ extern "C" void scope_stats_on_fatal() { extern "C" void scope_stats_set_ring_capacity(int ring_id, int32_t window_cap, uint64_t heap_cap, int32_t dep_pool_cap) { if (!s_scope_stats_header) return; - if (ring_id < 0 || ring_id >= PTO2_SCOPE_STATS_MAX_RING_DEPTH) return; + if (ring_id < 0 || ring_id >= SCOPE_STATS_MAX_RING_DEPTH) return; s_scope_stats_header->task_window_cap[ring_id] = window_cap; s_scope_stats_header->heap_cap[ring_id] = heap_cap; s_scope_stats_header->dep_pool_cap[ring_id] = dep_pool_cap; @@ -347,6 +347,6 @@ extern "C" void scope_stats_note_heap_wrap(int side) { if (scope_stats_depth < 0) return; // wrap outside any scope — unattributable if (side != SCOPE_STATS_HEAP_SIDE_ALLOC && side != SCOPE_STATS_HEAP_SIDE_RECLAIM) return; int ring_id = s_scope_ring[scope_stats_depth]; - if (ring_id < 0 || ring_id >= PTO2_SCOPE_STATS_MAX_RING_DEPTH) return; + if (ring_id < 0 || ring_id >= SCOPE_STATS_MAX_RING_DEPTH) return; s_heap_wraps[ring_id][side] += 1; } diff --git a/src/common/platform/shared/host/scope_stats_collector.cpp b/src/common/platform/shared/host/scope_stats_collector.cpp index ef992fe16c..a3eefd30cf 100644 --- a/src/common/platform/shared/host/scope_stats_collector.cpp +++ b/src/common/platform/shared/host/scope_stats_collector.cpp @@ -277,7 +277,7 @@ int ScopeStatsCollector::write_jsonl(const std::string &output_dir) { std::string task_window_max; std::string heap_max; std::string dep_pool_max; - for (int r = 0; r < PTO2_SCOPE_STATS_MAX_RING_DEPTH; r++) { + for (int r = 0; r < SCOPE_STATS_MAX_RING_DEPTH; r++) { char buf[32]; std::snprintf(buf, sizeof(buf), "%s%d", r == 0 ? "" : ", ", hdr->task_window_cap[r]); task_window_max += buf; diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/kernels/orchestration/sync_start_early_local_owner_orch.cpp b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/kernels/orchestration/sync_start_early_local_owner_orch.cpp index 254c5efb8e..17203995a6 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/kernels/orchestration/sync_start_early_local_owner_orch.cpp +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/kernels/orchestration/sync_start_early_local_owner_orch.cpp @@ -140,7 +140,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip return; } - rt_scope_begin(PTO2ScopeMode::MANUAL); + rt_scope_begin(ScopeMode::MANUAL); TaskId producer = use_pending ? submit_aiv_blocker(output, blocker_count) : submit_producer(output); if (use_pending && (!wait_for_blockers_started(output, blocker_count) || !wait_for_scheduler_loop_fence())) return; submit_consumer(output, producer, consumer_blocks); diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_early_dispatch/kernels/orchestration/spmd_sync_start_early_dispatch_orch.cpp b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_early_dispatch/kernels/orchestration/spmd_sync_start_early_dispatch_orch.cpp index bfc889c72b..d671881dc0 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_early_dispatch/kernels/orchestration/spmd_sync_start_early_dispatch_orch.cpp +++ b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_early_dispatch/kernels/orchestration/spmd_sync_start_early_dispatch_orch.cpp @@ -88,7 +88,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip // the device on purpose; it carries no sync_start, so no cap applies. const int32_t sync_blocks = rt_available_cluster_count(); - rt_scope_begin(PTO2ScopeMode::MANUAL); + rt_scope_begin(ScopeMode::MANUAL); TaskId prod = submit_producer(ext_output, PRODUCER_BLOCKS, 0); submit_sync_consumer(ext_output, static_cast(sync_blocks), PRODUCER_BLOCKS, prod); rt_scope_end(); diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/kernels/orchestration/sync_start_early_local_owner_orch.cpp b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/kernels/orchestration/sync_start_early_local_owner_orch.cpp index faf5eedbdb..b11f37a007 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/kernels/orchestration/sync_start_early_local_owner_orch.cpp +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/kernels/orchestration/sync_start_early_local_owner_orch.cpp @@ -132,7 +132,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip return; } - rt_scope_begin(PTO2ScopeMode::MANUAL); + rt_scope_begin(ScopeMode::MANUAL); TaskId producer = use_pending ? submit_aiv_blocker(output, blocker_count) : submit_producer(output); if (use_pending && (!wait_for_blockers_started(output, blocker_count) || !wait_for_scheduler_loop_fence())) return; submit_consumer(output, producer, consumer_blocks); diff --git a/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_early_dispatch/kernels/orchestration/spmd_sync_start_early_dispatch_orch.cpp b/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_early_dispatch/kernels/orchestration/spmd_sync_start_early_dispatch_orch.cpp index 355b477373..125843d089 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_early_dispatch/kernels/orchestration/spmd_sync_start_early_dispatch_orch.cpp +++ b/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_early_dispatch/kernels/orchestration/spmd_sync_start_early_dispatch_orch.cpp @@ -83,7 +83,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip // the device on purpose; it carries no sync_start, so no cap applies. const int32_t sync_blocks = rt_available_cluster_count(); - rt_scope_begin(PTO2ScopeMode::MANUAL); + rt_scope_begin(ScopeMode::MANUAL); TaskId prod = submit_producer(ext_output, PRODUCER_BLOCKS, 0, early_on); submit_sync_consumer(ext_output, static_cast(sync_blocks), PRODUCER_BLOCKS, prod); rt_scope_end(); diff --git a/tests/ut/cpp/a2a3/test_a2a3_fatal.cpp b/tests/ut/cpp/a2a3/test_a2a3_fatal.cpp index 23525996a5..3d7ddd392a 100644 --- a/tests/ut/cpp/a2a3/test_a2a3_fatal.cpp +++ b/tests/ut/cpp/a2a3/test_a2a3_fatal.cpp @@ -34,13 +34,13 @@ extern "C" void framework_bind_runtime(RuntimeContext *rt) { g_bound_runtime = r struct FakeRuntime { const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode = PTO2ScopeMode::AUTO; + ScopeMode pending_scope_mode = ScopeMode::AUTO; bool fatal = false; int submit_calls = 0; int alloc_calls = 0; int scope_begin_calls = 0; int scope_end_calls = 0; - PTO2ScopeMode last_scope_mode = PTO2ScopeMode::AUTO; + ScopeMode last_scope_mode = ScopeMode::AUTO; int get_calls = 0; int set_calls = 0; int report_fatal_calls = 0; @@ -193,22 +193,22 @@ TEST(A2A3Fatal, ScopeGuardForwardsModeUntilLexicalScopeExit) { runtime.ops = &kFakeOps; RuntimeBindingGuard bind(reinterpret_cast(&runtime)); - runtime.pending_scope_mode = PTO2ScopeMode::MANUAL; + runtime.pending_scope_mode = ScopeMode::MANUAL; { SIMPLER_SCOPE_GUARD(); EXPECT_EQ(runtime.scope_begin_calls, 1); EXPECT_EQ(runtime.scope_end_calls, 0); - EXPECT_EQ(runtime.last_scope_mode, PTO2ScopeMode::AUTO); + EXPECT_EQ(runtime.last_scope_mode, ScopeMode::AUTO); } EXPECT_EQ(runtime.scope_end_calls, 1); { - SIMPLER_SCOPE_GUARD(PTO2ScopeMode::MANUAL); + SIMPLER_SCOPE_GUARD(ScopeMode::MANUAL); EXPECT_EQ(runtime.scope_begin_calls, 2); EXPECT_EQ(runtime.scope_end_calls, 1); - EXPECT_EQ(runtime.last_scope_mode, PTO2ScopeMode::MANUAL); + EXPECT_EQ(runtime.last_scope_mode, ScopeMode::MANUAL); } EXPECT_EQ(runtime.scope_end_calls, 2); diff --git a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp index b4cb662b86..c3223620ce 100644 --- a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp @@ -339,7 +339,7 @@ TEST_F(OrchestratorFaninTest, ClosedChildHeadUsesTimeoutWithOpenParentOnSharedRi } // Regression for issue #1188: scope_tasks_cap must equal the real in-flight budget -// (sum of the runtime per-ring windows), not the compile-time PTO2_SCOPE_TASKS_CAP. +// (sum of the runtime per-ring windows), not the compile-time CHIP_SCOPE_TASKS_CAP. // reserve_layout only computes offsets, so no commit()/backing is needed here. TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { auto cap_for = [](const int32_t windows[CHIP_MAX_RING_DEPTH]) { @@ -355,7 +355,7 @@ TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) windows[r] = PTO2_TASK_WINDOW_SIZE; EXPECT_EQ(cap_for(windows), PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH); - EXPECT_EQ(cap_for(windows), PTO2_SCOPE_TASKS_CAP); + EXPECT_EQ(cap_for(windows), CHIP_SCOPE_TASKS_CAP); // Shrunk window: cap shrinks to the real budget (no over-allocation). for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) @@ -368,5 +368,5 @@ TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) windows[r] = big; EXPECT_EQ(cap_for(windows), big * CHIP_MAX_RING_DEPTH); - EXPECT_GT(cap_for(windows), PTO2_SCOPE_TASKS_CAP); + EXPECT_GT(cap_for(windows), CHIP_SCOPE_TASKS_CAP); } diff --git a/tests/ut/cpp/a5/test_a5_fatal.cpp b/tests/ut/cpp/a5/test_a5_fatal.cpp index 94acb17c40..bcb3bca931 100644 --- a/tests/ut/cpp/a5/test_a5_fatal.cpp +++ b/tests/ut/cpp/a5/test_a5_fatal.cpp @@ -28,13 +28,13 @@ extern "C" void framework_bind_runtime(RuntimeContext *rt) { g_bound_runtime = r struct FakeRuntime { const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode = PTO2ScopeMode::AUTO; + ScopeMode pending_scope_mode = ScopeMode::AUTO; bool fatal = false; int submit_calls = 0; int alloc_calls = 0; int scope_begin_calls = 0; int scope_end_calls = 0; - PTO2ScopeMode last_scope_mode = PTO2ScopeMode::AUTO; + ScopeMode last_scope_mode = ScopeMode::AUTO; int get_calls = 0; int set_calls = 0; int report_fatal_calls = 0; @@ -187,22 +187,22 @@ TEST(A5Fatal, ScopeGuardForwardsModeUntilLexicalScopeExit) { runtime.ops = &kFakeOps; RuntimeBindingGuard bind(reinterpret_cast(&runtime)); - runtime.pending_scope_mode = PTO2ScopeMode::MANUAL; + runtime.pending_scope_mode = ScopeMode::MANUAL; { SIMPLER_SCOPE_GUARD(); EXPECT_EQ(runtime.scope_begin_calls, 1); EXPECT_EQ(runtime.scope_end_calls, 0); - EXPECT_EQ(runtime.last_scope_mode, PTO2ScopeMode::AUTO); + EXPECT_EQ(runtime.last_scope_mode, ScopeMode::AUTO); } EXPECT_EQ(runtime.scope_end_calls, 1); { - SIMPLER_SCOPE_GUARD(PTO2ScopeMode::MANUAL); + SIMPLER_SCOPE_GUARD(ScopeMode::MANUAL); EXPECT_EQ(runtime.scope_begin_calls, 2); EXPECT_EQ(runtime.scope_end_calls, 1); - EXPECT_EQ(runtime.last_scope_mode, PTO2ScopeMode::MANUAL); + EXPECT_EQ(runtime.last_scope_mode, ScopeMode::MANUAL); } EXPECT_EQ(runtime.scope_end_calls, 2); diff --git a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp index 479b3d81a9..c6bd7810cc 100644 --- a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp @@ -353,7 +353,7 @@ TEST_F(OrchestratorFaninTest, ClosedChildHeadUsesTimeoutWithOpenParentOnSharedRi } // Regression for issue #1188: scope_tasks_cap must equal the real in-flight budget -// (sum of the runtime per-ring windows), not the compile-time PTO2_SCOPE_TASKS_CAP. +// (sum of the runtime per-ring windows), not the compile-time CHIP_SCOPE_TASKS_CAP. // reserve_layout only computes offsets, so no commit()/backing is needed here. TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { auto cap_for = [](const int32_t windows[CHIP_MAX_RING_DEPTH]) { @@ -369,7 +369,7 @@ TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) windows[r] = PTO2_TASK_WINDOW_SIZE; EXPECT_EQ(cap_for(windows), PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH); - EXPECT_EQ(cap_for(windows), PTO2_SCOPE_TASKS_CAP); + EXPECT_EQ(cap_for(windows), CHIP_SCOPE_TASKS_CAP); // Shrunk window: cap shrinks to the real budget (no over-allocation). for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) @@ -382,5 +382,5 @@ TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) windows[r] = big; EXPECT_EQ(cap_for(windows), big * CHIP_MAX_RING_DEPTH); - EXPECT_GT(cap_for(windows), PTO2_SCOPE_TASKS_CAP); + EXPECT_GT(cap_for(windows), CHIP_SCOPE_TASKS_CAP); } diff --git a/tests/ut/cpp/common/test_hbg_graph_async_submit.cpp b/tests/ut/cpp/common/test_hbg_graph_async_submit.cpp index 6aada184a1..9831a027af 100644 --- a/tests/ut/cpp/common/test_hbg_graph_async_submit.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_async_submit.cpp @@ -35,7 +35,7 @@ extern "C" void framework_bind_runtime(RuntimeContext *rt) { g_bound_runtime = r struct FakeRuntime { const RuntimeOps *ops; - PTO2ScopeMode pending_scope_mode{PTO2ScopeMode::AUTO}; + ScopeMode pending_scope_mode{ScopeMode::AUTO}; std::mutex mutex; std::condition_variable cv; bool record_entered{false}; diff --git a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp index 696597fd49..ebb51c991f 100644 --- a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp @@ -319,8 +319,8 @@ TEST_F(HbgGraphSubmitFailureTest, AutoScopeNestedInManualScopeRefusesTheRecordin ASSERT_TRUE(graph.recording); ASSERT_TRUE(orch.graph_prepare(graph.recording_handle, boundary_args)); - orch.begin_scope(PTO2ScopeMode::MANUAL); - orch.begin_scope(PTO2ScopeMode::AUTO); + orch.begin_scope(ScopeMode::MANUAL); + orch.begin_scope(ScopeMode::AUTO); CoreTaskArgs node_args; node_args.add_input(boundary); From 89db474872c75ac7e30a295ed90fc4b1b441ae1a Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:49:17 -0700 Subject: [PATCH 16/22] Refactor: name the chip runtime's alignment macros by their context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PTO2_ALIGN_UP` becomes `CHIP_ALIGN_UP` rather than `ALIGN_UP`, because CANN already defines `ALIGN_UP` — in `acl/ops/acl_dvpp.h` and `acl/dvpp/hi_media_common.h`, both reachable from translation units that include our headers. No grep of this repository can see that, which is why the check has to run against the external headers we compile against and not only against the tree. `PTO2_ALIGN_SIZE` takes the same prefix even though `ALIGN_SIZE` is free. The two are one pair — the granularity and the macro that rounds to it — and splitting their spelling would read as two unrelated constants. `PTO2_PACKED_OUTPUT_ALIGN` is a plain strip to `PACKED_OUTPUT_ALIGN`: it belongs to the packed-output layout rather than to the arena's alignment, and nothing claims the name. 343 occurrences across 70 files. Co-Authored-By: Claude Opus 5 (1M context) --- docs/orchestrator.md | 2 +- .../host_build_graph/host/runtime_maker.cpp | 20 ++++++++--------- .../runtime/aicore_completion_mailbox.h | 14 ++++++------ .../runtime/aicore_completion_mailbox_types.h | 4 ++-- .../runtime/async_kernel_api.h | 8 +++---- .../host_build_graph/runtime/async_wait.h | 2 +- .../backend/sdma/sdma_completion_scheduler.h | 4 ++-- .../host_build_graph/runtime/constants.h | 6 ++--- .../runtime/dispatch_payload.h | 6 ++--- .../orchestrator_core/orchestrator.cpp | 17 +++++++------- .../host_build_graph/runtime/ring_buffer.h | 4 ++-- .../host_build_graph/runtime/runtime_core.h | 2 +- .../host_build_graph/runtime/runtime_types.h | 2 +- .../runtime/shared/runtime_init.cpp | 6 ++--- .../runtime/shared/shared_memory.cpp | 12 +++++----- .../host_build_graph/runtime/shared_memory.h | 22 +++++++++---------- .../docs/RUNTIME_LOGIC.md | 2 +- .../runtime/aicore_completion_mailbox.h | 14 ++++++------ .../runtime/aicore_completion_mailbox_types.h | 4 ++-- .../runtime/async_kernel_api.h | 8 +++---- .../runtime/async_wait.h | 2 +- .../backend/sdma/sdma_completion_scheduler.h | 4 ++-- .../runtime/constants.h | 6 ++--- .../runtime/dispatch_payload.h | 6 ++--- .../runtime/orchestrator.cpp | 3 +-- .../runtime/ring_buffer.h | 2 +- .../runtime/runtime_types.h | 2 +- .../runtime/shared/runtime_init.cpp | 22 +++++++++---------- .../runtime/shared/shared_memory.cpp | 12 +++++----- .../runtime/shared_memory.h | 18 +++++++-------- .../host_build_graph/host/runtime_maker.cpp | 20 ++++++++--------- .../runtime/aicore_completion_mailbox.h | 14 ++++++------ .../runtime/aicore_completion_mailbox_types.h | 4 ++-- .../runtime/async_kernel_api.h | 8 +++---- .../host_build_graph/runtime/async_wait.h | 2 +- .../host_build_graph/runtime/constants.h | 6 ++--- .../runtime/dispatch_payload.h | 6 ++--- .../orchestrator_core/orchestrator.cpp | 17 +++++++------- .../host_build_graph/runtime/ring_buffer.h | 4 ++-- .../host_build_graph/runtime/runtime_core.h | 2 +- .../host_build_graph/runtime/runtime_types.h | 2 +- .../runtime/shared/runtime_init.cpp | 6 ++--- .../runtime/shared/shared_memory.cpp | 12 +++++----- .../host_build_graph/runtime/shared_memory.h | 22 +++++++++---------- .../docs/RUNTIME_LOGIC.md | 2 +- .../runtime/aicore_completion_mailbox.h | 14 ++++++------ .../runtime/aicore_completion_mailbox_types.h | 4 ++-- .../runtime/async_kernel_api.h | 8 +++---- .../backend/urma/urma_completion_scheduler.h | 6 ++--- .../runtime/constants.h | 6 ++--- .../runtime/dispatch_payload.h | 6 ++--- .../runtime/orchestrator.cpp | 3 +-- .../runtime/ring_buffer.h | 2 +- .../runtime/runtime_types.h | 2 +- .../runtime/shared/runtime_init.cpp | 22 +++++++++---------- .../runtime/shared/shared_memory.cpp | 12 +++++----- .../runtime/shared_memory.h | 18 +++++++-------- src/common/hierarchical/ring.h | 2 +- .../host_build_graph/graph_execution.cpp | 4 ++-- src/common/utils/device_arena.h | 2 +- .../a2a3/test_aicore_completion_mailbox.cpp | 2 +- tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp | 2 +- tests/ut/cpp/a2a3/test_shared_memory.cpp | 14 ++++++------ tests/ut/cpp/a2a3/test_task_allocator.cpp | 4 ++-- .../cpp/a5/test_aicore_completion_mailbox.cpp | 2 +- tests/ut/cpp/a5/test_shared_memory.cpp | 14 ++++++------ tests/ut/cpp/a5/test_task_allocator.cpp | 4 ++-- tests/ut/cpp/common/test_hbg_graph_cache.cpp | 6 ++--- .../common/test_hbg_graph_submit_failure.cpp | 4 ++-- .../ut/cpp/common/test_hbg_sm_compaction.cpp | 8 +++---- 70 files changed, 265 insertions(+), 269 deletions(-) diff --git a/docs/orchestrator.md b/docs/orchestrator.md index 1b135874f7..5e7fd60e7c 100644 --- a/docs/orchestrator.md +++ b/docs/orchestrator.md @@ -407,7 +407,7 @@ exception so users can enlarge `heap_ring_size` on the `Worker` instead of deadlocking. **Alignment**: every heap allocation is rounded up to `HEAP_ALIGN = 1024 B` -(matches L2's `PTO2_PACKED_OUTPUT_ALIGN`, Strict-3). +(matches L2's `PACKED_OUTPUT_ALIGN`, Strict-3). **FIFO reclamation per ring**: each `alloc()` appends the slot's `heap_end_offset` onto the selected ring's `slot_heap_end[]` vector, and diff --git a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp index 42bc0c1928..e48e4431f4 100644 --- a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp @@ -556,12 +556,12 @@ int32_t run_host_orchestration( // orch::prepare_task and shipped bounded to total_tasks below. const sm_layout::ChipRingSegmentOffsets sm_segs = sm_layout::ring_segment_offsets(eff_task_window_size); // Over-allocated and rounded up: every segment offset is a multiple of - // PTO2_ALIGN_SIZE and ChipTaskSlotState is alignas(64), which a plain + // CHIP_ALIGN_SIZE and ChipTaskSlotState is alignas(64), which a plain // new uint8_t[] does not guarantee. - std::unique_ptr host_sm_buf(new uint8_t[sm_size + PTO2_ALIGN_SIZE]); + std::unique_ptr host_sm_buf(new uint8_t[sm_size + CHIP_ALIGN_SIZE]); void *host_sm = reinterpret_cast( - (reinterpret_cast(host_sm_buf.get()) + PTO2_ALIGN_SIZE - 1) & - ~static_cast(PTO2_ALIGN_SIZE - 1) + (reinterpret_cast(host_sm_buf.get()) + CHIP_ALIGN_SIZE - 1) & + ~static_cast(CHIP_ALIGN_SIZE - 1) ); std::memset(host_sm, 0, sm_segs.descriptors); @@ -773,8 +773,8 @@ int32_t run_host_orchestration( // same boundary it starts on, so an access at the tail of the last packed buffer // stays inside the region even when its width exceeds the bytes that buffer // asked for. - const uint64_t heap_bytes = PTO2_ALIGN_UP( - std::max(orchestrator.task_allocator.heap_used_bytes(), PTO2_ALIGN_SIZE), + const uint64_t heap_bytes = CHIP_ALIGN_UP( + std::max(orchestrator.task_allocator.heap_used_bytes(), CHIP_ALIGN_SIZE), DeviceArena::kDefaultBaseAlign ); if (api->setup_static_arena(heap_bytes, /*gm_sm_size=*/0, device_arena_bytes) != 0) { @@ -837,14 +837,14 @@ int32_t run_host_orchestration( // One host source for one copy: the copied zone and shared-memory image at // exactly the offsets they occupy on the device. // Over-allocated and rounded up because every segment offset is - // PTO2_ALIGN_SIZE-aligned and ChipTaskSlotState is alignas(64), which a byte + // CHIP_ALIGN_SIZE-aligned and ChipTaskSlotState is alignas(64), which a byte // vector's data() is not. const uint64_t copied_bytes = layout.off_copied_end - layout.off_copied_begin; const uint64_t upload_bytes = copied_bytes + image_bytes; - std::vector storage(upload_bytes + PTO2_ALIGN_SIZE, std::byte{0}); + std::vector storage(upload_bytes + CHIP_ALIGN_SIZE, std::byte{0}); char *upload_base = reinterpret_cast( - (reinterpret_cast(storage.data()) + PTO2_ALIGN_SIZE - 1) & - ~static_cast(PTO2_ALIGN_SIZE - 1) + (reinterpret_cast(storage.data()) + CHIP_ALIGN_SIZE - 1) & + ~static_cast(CHIP_ALIGN_SIZE - 1) ); // The copied zone carries no host address: the orchestrator is host-only and diff --git a/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox.h b/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox.h index 9ed3480e38..27fd8a8853 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox.h +++ b/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox.h @@ -63,7 +63,7 @@ struct AICoreCompletionMailboxMessage { uint32_t _pad[3]; }; -static_assert(sizeof(AICoreCompletionMailboxMessage) == PTO2_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); +static_assert(sizeof(AICoreCompletionMailboxMessage) == CHIP_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); static_assert( sizeof(std::atomic) == sizeof(uint64_t), "std::atomic must be layout-compatible with uint64_t for the message slot layout to hold" @@ -89,11 +89,11 @@ struct AICoreCompletionMsgView { struct AICoreCompletionMailbox { // head and tail live on their own cache lines so producer CAS contention // on head can't false-share with the consumer's tail updates. - alignas(PTO2_ALIGN_SIZE) std::atomic head; - uint8_t _head_pad[PTO2_ALIGN_SIZE - sizeof(uint64_t)]; - alignas(PTO2_ALIGN_SIZE) std::atomic tail; - uint8_t _tail_pad[PTO2_ALIGN_SIZE - sizeof(uint64_t)]; - alignas(PTO2_ALIGN_SIZE) AICoreCompletionMailboxMessage entries[AICORE_COMPLETION_MAILBOX_CAPACITY]; + alignas(CHIP_ALIGN_SIZE) std::atomic head; + uint8_t _head_pad[CHIP_ALIGN_SIZE - sizeof(uint64_t)]; + alignas(CHIP_ALIGN_SIZE) std::atomic tail; + uint8_t _tail_pad[CHIP_ALIGN_SIZE - sizeof(uint64_t)]; + alignas(CHIP_ALIGN_SIZE) AICoreCompletionMailboxMessage entries[AICORE_COMPLETION_MAILBOX_CAPACITY]; // Cheap, lock-free pending hint. Callers may invoke this outside the // consumer lock; a stale answer only over/under-triggers a drain attempt. @@ -218,5 +218,5 @@ struct AICoreCompletionMailbox { }; static_assert( - sizeof(AICoreCompletionMailbox) % PTO2_ALIGN_SIZE == 0, "AICoreCompletionMailbox size must be cache-line aligned" + sizeof(AICoreCompletionMailbox) % CHIP_ALIGN_SIZE == 0, "AICoreCompletionMailbox size must be cache-line aligned" ); diff --git a/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h b/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h index e6f5a22b7b..9332e4ac70 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h @@ -54,14 +54,14 @@ struct DeferredCompletionEntry { static_assert(sizeof(DeferredCompletionEntry) == 32, "DeferredCompletionEntry layout drift"); -struct alignas(PTO2_ALIGN_SIZE) DeferredCompletionSlab { +struct alignas(CHIP_ALIGN_SIZE) DeferredCompletionSlab { volatile uint32_t count; volatile int32_t error_code; DeferredCompletionEntry entries[MAX_COMPLETIONS_PER_TASK]; }; static_assert( - sizeof(DeferredCompletionSlab) % PTO2_ALIGN_SIZE == 0, + sizeof(DeferredCompletionSlab) % CHIP_ALIGN_SIZE == 0, "DeferredCompletionSlab size must preserve array element cache-line boundaries" ); diff --git a/src/a2a3/runtime/host_build_graph/runtime/async_kernel_api.h b/src/a2a3/runtime/host_build_graph/runtime/async_kernel_api.h index f415a24fc9..884a45561e 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/async_kernel_api.h +++ b/src/a2a3/runtime/host_build_graph/runtime/async_kernel_api.h @@ -38,7 +38,7 @@ namespace pto2::detail { inline __aicore__ void defer_load_slab(AsyncCtx &ctx) { if (ctx.completion_count == nullptr) return; #if defined(__CCE_KT_TEST__) || defined(__CCE_AICORE__) || defined(__DAV_C220__) - uintptr_t line = reinterpret_cast(ctx.completion_count) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + uintptr_t line = reinterpret_cast(ctx.completion_count) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); dcci((__gm__ int32_t *)line, SINGLE_CACHE_LINE); #else __asm__ __volatile__("" ::: "memory"); @@ -54,10 +54,10 @@ inline __aicore__ void defer_error(AsyncCtx &ctx, int32_t error_code) { inline __aicore__ void defer_flush_range(volatile __gm__ void *addr, uint32_t size_bytes) { if (addr == nullptr || size_bytes == 0) return; #if defined(__CCE_KT_TEST__) || defined(__CCE_AICORE__) || defined(__DAV_C220__) - uintptr_t start = reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + uintptr_t start = reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); uintptr_t end = - (reinterpret_cast(addr) + size_bytes + PTO2_ALIGN_SIZE - 1u) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); - for (uintptr_t p = start; p < end; p += PTO2_ALIGN_SIZE) { + (reinterpret_cast(addr) + size_bytes + CHIP_ALIGN_SIZE - 1u) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); + for (uintptr_t p = start; p < end; p += CHIP_ALIGN_SIZE) { dcci((__gm__ int32_t *)p, SINGLE_CACHE_LINE, CACHELINE_OUT); } #else diff --git a/src/a2a3/runtime/host_build_graph/runtime/async_wait.h b/src/a2a3/runtime/host_build_graph/runtime/async_wait.h index db86ba05d1..3c7b10186d 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/async_wait.h +++ b/src/a2a3/runtime/host_build_graph/runtime/async_wait.h @@ -33,7 +33,7 @@ inline constexpr int32_t MAX_ASYNC_WAITS = 64; // application layer: translating drained messages into wait-list state. inline uintptr_t mailbox_cache_line(const volatile void *addr) { - return reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + return reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); } struct CompletionCondition; diff --git a/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h b/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h index 7410c97278..332bbb91d8 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h +++ b/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h @@ -20,7 +20,7 @@ #include "runtime_status.h" inline uintptr_t sdma_completion_cache_line(const volatile void *addr) { - return reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + return reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); } // PTO-ISA stores the latest completed post ID as a monotonic uint64_t in one @@ -32,7 +32,7 @@ inline CompletionPollResult poll_sdma_post_done_record(uint64_t record_addr, uin return {CompletionPollState::FAILED, SIMPLER_ERROR_ASYNC_COMPLETION_INVALID}; } volatile uint64_t *record = reinterpret_cast(static_cast(record_addr)); - cache_invalidate_range(reinterpret_cast(sdma_completion_cache_line(record)), PTO2_ALIGN_SIZE); + cache_invalidate_range(reinterpret_cast(sdma_completion_cache_line(record)), CHIP_ALIGN_SIZE); uint64_t completed_post_id = __atomic_load_n(record, __ATOMIC_ACQUIRE); return { completed_post_id >= expected_post_id ? CompletionPollState::READY : CompletionPollState::PENDING, diff --git a/src/a2a3/runtime/host_build_graph/runtime/constants.h b/src/a2a3/runtime/host_build_graph/runtime/constants.h index 07251cc396..3d22a80ea7 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/constants.h +++ b/src/a2a3/runtime/host_build_graph/runtime/constants.h @@ -12,8 +12,8 @@ #ifndef SRC_A2A3_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_PTO_CONSTANTS_H_ #define SRC_A2A3_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_PTO_CONSTANTS_H_ -#define PTO2_ALIGN_SIZE 64 // Cache line alignment -#define PTO2_PACKED_OUTPUT_ALIGN 1024 // Each output in packed buffer aligned to 1024B; gap is padding -#define PTO2_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#define CHIP_ALIGN_SIZE 64 // Cache line alignment +#define PACKED_OUTPUT_ALIGN 1024 // Each output in packed buffer aligned to 1024B; gap is padding +#define CHIP_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) #endif // SRC_A2A3_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_PTO_CONSTANTS_H_ diff --git a/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h b/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h index 13169f89bc..021a4d7cb4 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h +++ b/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h @@ -46,8 +46,8 @@ #define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) #endif -#ifndef PTO2_ALIGN_UP -#define PTO2_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#ifndef CHIP_ALIGN_UP +#define CHIP_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) #endif // Verify hardcoded indices in intrinsic.h match the computed values. @@ -124,7 +124,7 @@ struct alignas(64) DispatchPayload { static_assert(sizeof(args[0]) == 8); static_assert( - PTO2_ALIGN_UP((MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]), 64) == + CHIP_ALIGN_UP((MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]), 64) == (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]) ); }; diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index c39851f37f..2f382301bd 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -971,7 +971,7 @@ bool graph_fill_definition(const GraphRecording &recording, GraphDefinition defi return false; } node_offsets[i] = required_heap; - const uint64_t output_bytes = PTO2_ALIGN_UP(source.total_output_size, PTO2_ALIGN_SIZE); + const uint64_t output_bytes = CHIP_ALIGN_UP(source.total_output_size, CHIP_ALIGN_SIZE); if (required_heap > UINT64_MAX - output_bytes) return false; required_heap += output_bytes; @@ -1231,8 +1231,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { continue; } layout.offsets[i] = layout.total_output_size; - layout.buffer_sizes[i] = - PTO2_ALIGN_UP(args.tensor(i).create_info().buffer_size_bytes(), PTO2_PACKED_OUTPUT_ALIGN); + layout.buffer_sizes[i] = CHIP_ALIGN_UP(args.tensor(i).create_info().buffer_size_bytes(), PACKED_OUTPUT_ALIGN); layout.total_output_size += layout.buffer_sizes[i]; } return layout; @@ -1276,7 +1275,7 @@ static bool prepare_task( // ChipTensor being two cache lines. The fanin cursor advances at publish, not // here — see the comment where it does. const uint64_t window = orch->sm_header->ring.task_window_size; - const int32_t scalar_span = PTO2_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); + const int32_t scalar_span = CHIP_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); debug_assert(static_cast(orch->tensor_pool_cursor) + args.tensor_count() <= window * MAX_TENSOR_ARGS); debug_assert(static_cast(orch->scalar_pool_cursor) + scalar_span <= window * MAX_SCALAR_ARGS); debug_assert(static_cast(orch->fanin_pool_cursor) + CHIP_MAX_FANIN <= window * CHIP_MAX_FANIN); @@ -1614,7 +1613,7 @@ static TaskOutputTensors submit_task_common( // equality holds only while nothing between the bind and here bound another fanin // region, which is what makes the deferred advance safe. debug_assert(orch->fanin_pool_cursor == static_cast(payload.fanin_data() - orch->fanin_pool)); - orch->fanin_pool_cursor += PTO2_ALIGN_UP(fanin_builder.count, ARG_POOL_ALIGN / (int32_t)sizeof(int32_t)); + orch->fanin_pool_cursor += CHIP_ALIGN_UP(fanin_builder.count, ARG_POOL_ALIGN / (int32_t)sizeof(int32_t)); (void)sched; @@ -1773,7 +1772,7 @@ bool graph_submit_outer( const uint64_t task_window = orch->sm_header->ring.task_window_size; const int32_t tensor_slots = static_cast(graph_boundary_tensor_pool_slots(static_cast(args.tensor_count()))); - const int32_t scalar_span = PTO2_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); + const int32_t scalar_span = CHIP_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); if (static_cast(orch->tensor_pool_cursor) + tensor_slots > task_window * MAX_TENSOR_ARGS || static_cast(orch->scalar_pool_cursor) + scalar_span > task_window * MAX_SCALAR_ARGS) { LOG_WARN("%s", "[GraphExecution] boundary exceeds the argument pools; using ordinary path"); @@ -1841,7 +1840,7 @@ bool graph_submit_outer( if (args.scalar_count() != 0) { std::memcpy( payload.scalar_data(), args.scalar_data(), - PTO2_ALIGN_UP(static_cast(args.scalar_count()) * sizeof(uint64_t), ARG_POOL_ALIGN) + CHIP_ALIGN_UP(static_cast(args.scalar_count()) * sizeof(uint64_t), ARG_POOL_ALIGN) ); } @@ -1884,7 +1883,7 @@ bool graph_submit_outer( // equality holds only while nothing between the bind and here bound another fanin // region, which is what makes the deferred advance safe. debug_assert(orch->fanin_pool_cursor == static_cast(payload.fanin_data() - orch->fanin_pool)); - orch->fanin_pool_cursor += PTO2_ALIGN_UP(fanin_builder.count, ARG_POOL_ALIGN / (int32_t)sizeof(int32_t)); + orch->fanin_pool_cursor += CHIP_ALIGN_UP(fanin_builder.count, ARG_POOL_ALIGN / (int32_t)sizeof(int32_t)); pending.outer_slot = &slot; state->pending_uploads.push_back(pending); @@ -1983,7 +1982,7 @@ TaskOutputTensors graph_record_submit_node( const PTO2OutputLayout layout = calculate_output_layout(args); const uint64_t aligned_output = - layout.total_output_size > 0 ? PTO2_ALIGN_UP(static_cast(layout.total_output_size), PTO2_ALIGN_SIZE) : + layout.total_output_size > 0 ? CHIP_ALIGN_UP(static_cast(layout.total_output_size), CHIP_ALIGN_SIZE) : 0; if (recording.next_virtual_offset > GRAPH_RECORD_VIRTUAL_BASE - aligned_output) { recording.unsupported = true; diff --git a/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h b/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h index 3bbe149f71..9d2dc1eac6 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h +++ b/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h @@ -102,7 +102,7 @@ class TaskAllocator { */ TaskAllocResult alloc(int32_t output_size) { uint64_t aligned_size = - output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; + output_size > 0 ? CHIP_ALIGN_UP(static_cast(output_size), CHIP_ALIGN_SIZE) : 0; if (error_code_ptr_ != nullptr && error_code_ptr_->load(std::memory_order_acquire) != SIMPLER_ERROR_NONE) { return {-1, -1, nullptr, nullptr}; @@ -128,7 +128,7 @@ class TaskAllocator { return false; } const uint64_t aligned_size = - output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; + output_size > 0 ? CHIP_ALIGN_UP(static_cast(output_size), CHIP_ALIGN_SIZE) : 0; void *base = try_bump_heap(aligned_size); if (base == nullptr) return false; *packed_base = base; diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h index ef0b28666c..62e258d9f0 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h @@ -139,7 +139,7 @@ struct RuntimeArenaLayout { // therefore not known when this layout is built. bind grows the device region // to cover it once orchestration ends. // - // The copied zone is padded to a PTO2_ALIGN_SIZE boundary, so that tail begins + // The copied zone is padded to a CHIP_ALIGN_SIZE boundary, so that tail begins // exactly at off_copied_end: the copied zone and the shared-memory image are // adjacent on the device and travel as one copy. // diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index 0072b3834b..39a8766a79 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -457,7 +457,7 @@ struct TaskPayload { // cache lines (ARG_POOL_ALIGN), so the rounded copy stays inside this // task's own region. Eliminates branches; extra bytes within the same CL have // zero additional cost. - memcpy(scalar_data(), args.scalars(), PTO2_ALIGN_UP(args.scalar_count() * sizeof(uint64_t), 64)); + memcpy(scalar_data(), args.scalars(), CHIP_ALIGN_UP(args.scalar_count() * sizeof(uint64_t), 64)); // The ring's payload storage is reused raw memory that no constructor runs // over, so an unset predicate reads back as whatever the slot last held — diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 11dda9bb49..e9f7689253 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -36,7 +36,7 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Align the slots[] base to a full cache line so MPMC CAS traffic on the // first slot cannot false-share with whatever region sits in front of us. - return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), PTO2_ALIGN_SIZE); + return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), CHIP_ALIGN_SIZE); } // Initialize the queue header only. slots[] carries the sequence ramp push @@ -267,9 +267,9 @@ RuntimeArenaLayout runtime_reserve_layout(DeviceArena &arena, uint64_t task_wind layout.sched = SchedulerState::reserve_layout(arena); layout.off_copied_begin = arena.total_size(); - // Padded to a PTO2_ALIGN_SIZE boundary: the shared-memory image starts at + // Padded to a CHIP_ALIGN_SIZE boundary: the shared-memory image starts at // off_copied_end on the device and its segment offsets are aligned from there. - layout.off_runtime = arena.reserve(PTO2_ALIGN_UP(sizeof(RuntimeContext), PTO2_ALIGN_SIZE), PTO2_ALIGN_SIZE); + layout.off_runtime = arena.reserve(CHIP_ALIGN_UP(sizeof(RuntimeContext), CHIP_ALIGN_SIZE), CHIP_ALIGN_SIZE); layout.off_copied_end = arena.total_size(); layout.arena_size = arena.total_size(); diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 7cc4fd7308..8ad041e91d 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -113,7 +113,7 @@ bool SharedMemoryHandle::attach_populated( SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); - const size_t off_buffer = arena.reserve(static_cast(buffer_size), PTO2_ALIGN_SIZE); + const size_t off_buffer = arena.reserve(static_cast(buffer_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; auto *handle = static_cast(arena.region_ptr(off_handle)); @@ -149,13 +149,13 @@ void SharedMemoryHandle::init_header(uint64_t task_window_size) { header->orchestrator_done.store(0, std::memory_order_relaxed); // Ring layout info - uint64_t offset = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t offset = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); header->ring.task_window_size = task_window_size; header->ring.task_window_mask = static_cast(task_window_size - 1); header->ring.task_descriptors_offset = offset; - offset += PTO2_ALIGN_UP(task_window_size * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_size * sizeof(TaskPayload), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_size * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_size * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_size * sizeof(TaskPayload), CHIP_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_size * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); header->total_size = sm_size; @@ -221,7 +221,7 @@ bool ChipRingFlowControl::validate(SharedMemoryHandle *handle) const { if (h->ring.task_descriptors_offset >= h->total_size) return false; // Check pointer alignment - if ((uintptr_t)h->ring.task_descriptors % PTO2_ALIGN_SIZE != 0) return false; + if ((uintptr_t)h->ring.task_descriptors % CHIP_ALIGN_SIZE != 0) return false; // Check flow control pointer sanity int32_t current = current_task_index.load(std::memory_order_acquire); diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h index 018e9e5056..f65fc338ef 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h @@ -165,7 +165,7 @@ static_assert( * * Contains per-ring flow control and global layout information. */ -struct alignas(PTO2_ALIGN_SIZE) SharedMemoryHeader { +struct alignas(CHIP_ALIGN_SIZE) SharedMemoryHeader { // === RING FLOW CONTROL + LAYOUT INFO (single ring, set once at init) === SharedMemoryRingHeader ring; @@ -299,7 +299,7 @@ inline std::atomic *ring_current_task_index_addr(void *sm_dev_base) noe // Byte offsets (from the SM base) of the ring's segments. The layout is: header, then // descriptors -> payloads -> slot_states -> completion_flags -> the three argument -// pools, every segment PTO2_ALIGN_UP-padded. RingImageExtents dimensions them: the +// pools, every segment CHIP_ALIGN_UP-padded. RingImageExtents dimensions them: the // mirror for the worst case the API allows, the image for what this bind holds, which // is what makes the live prefixes contiguous and the upload one copy. // @@ -343,22 +343,22 @@ inline RingImageExtents mirror_extents(uint64_t task_window_size) noexcept { // or reordering a segment is a one-line edit here; every consumer follows // automatically, so the layout walk can never silently disagree across call sites. inline ChipRingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) noexcept { - uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t off = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); ChipRingSegmentOffsets o{}; o.descriptors = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.slots * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); o.payloads = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(TaskPayload), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.slots * sizeof(TaskPayload), CHIP_ALIGN_SIZE); o.slot_states = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.slots * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); o.completion_flags = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(std::atomic), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.slots * sizeof(std::atomic), CHIP_ALIGN_SIZE); o.fanin_pool = off; - off += PTO2_ALIGN_UP(e.fanin_elems * sizeof(int32_t), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.fanin_elems * sizeof(int32_t), CHIP_ALIGN_SIZE); o.tensor_pool = off; - off += PTO2_ALIGN_UP(e.tensor_elems * sizeof(ChipTensor), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.tensor_elems * sizeof(ChipTensor), CHIP_ALIGN_SIZE); o.scalar_pool = off; - off += PTO2_ALIGN_UP(e.scalar_elems * sizeof(uint64_t), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.scalar_elems * sizeof(uint64_t), CHIP_ALIGN_SIZE); o.end = off; return o; } @@ -436,7 +436,7 @@ inline uint64_t rebased_heap_addr(uint64_t addr, const HeapRebase &rebase) noexc // orchestrator wrote into an image dimensioned for what this bind holds, where the // prefixes are contiguous and can travel as one copy. // -// `out_base` must be PTO2_ALIGN_SIZE-aligned and hold +// `out_base` must be CHIP_ALIGN_SIZE-aligned and hold // `ring_segment_offsets(image_extents(used)).end` bytes. Returns that byte count. // // Three things the restack has to fix up, all because the image is not the mirror: diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index e571436f67..7f7f374465 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -177,7 +177,7 @@ total = ALIGN(Header) + ALIGN(window_size * sizeof(ChipTaskSlotState)) ] ``` -Alignment is 64 bytes (`PTO2_ALIGN_SIZE`). +Alignment is 64 bytes (`CHIP_ALIGN_SIZE`). --- diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h index c849e33f73..2da496b49c 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h @@ -64,7 +64,7 @@ struct AICoreCompletionMailboxMessage { uint32_t _pad[3]; }; -static_assert(sizeof(AICoreCompletionMailboxMessage) == PTO2_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); +static_assert(sizeof(AICoreCompletionMailboxMessage) == CHIP_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); static_assert( sizeof(std::atomic) == sizeof(uint64_t), "std::atomic must be layout-compatible with uint64_t for the message slot layout to hold" @@ -90,11 +90,11 @@ struct AICoreCompletionMsgView { struct AICoreCompletionMailbox { // head and tail live on their own cache lines so producer CAS contention // on head can't false-share with the consumer's tail updates. - alignas(PTO2_ALIGN_SIZE) std::atomic head; - uint8_t _head_pad[PTO2_ALIGN_SIZE - sizeof(uint64_t)]; - alignas(PTO2_ALIGN_SIZE) std::atomic tail; - uint8_t _tail_pad[PTO2_ALIGN_SIZE - sizeof(uint64_t)]; - alignas(PTO2_ALIGN_SIZE) AICoreCompletionMailboxMessage entries[AICORE_COMPLETION_MAILBOX_CAPACITY]; + alignas(CHIP_ALIGN_SIZE) std::atomic head; + uint8_t _head_pad[CHIP_ALIGN_SIZE - sizeof(uint64_t)]; + alignas(CHIP_ALIGN_SIZE) std::atomic tail; + uint8_t _tail_pad[CHIP_ALIGN_SIZE - sizeof(uint64_t)]; + alignas(CHIP_ALIGN_SIZE) AICoreCompletionMailboxMessage entries[AICORE_COMPLETION_MAILBOX_CAPACITY]; // Cheap, lock-free pending hint. Callers may invoke this outside the // consumer lock; a stale answer only over/under-triggers a drain attempt. @@ -195,7 +195,7 @@ struct AICoreCompletionMailbox { }; static_assert( - sizeof(AICoreCompletionMailbox) % PTO2_ALIGN_SIZE == 0, "AICoreCompletionMailbox size must be cache-line aligned" + sizeof(AICoreCompletionMailbox) % CHIP_ALIGN_SIZE == 0, "AICoreCompletionMailbox size must be cache-line aligned" ); #endif // SRC_A2A3_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_AICORE_COMPLETION_MAILBOX_H_ diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h index e6f5a22b7b..9332e4ac70 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h @@ -54,14 +54,14 @@ struct DeferredCompletionEntry { static_assert(sizeof(DeferredCompletionEntry) == 32, "DeferredCompletionEntry layout drift"); -struct alignas(PTO2_ALIGN_SIZE) DeferredCompletionSlab { +struct alignas(CHIP_ALIGN_SIZE) DeferredCompletionSlab { volatile uint32_t count; volatile int32_t error_code; DeferredCompletionEntry entries[MAX_COMPLETIONS_PER_TASK]; }; static_assert( - sizeof(DeferredCompletionSlab) % PTO2_ALIGN_SIZE == 0, + sizeof(DeferredCompletionSlab) % CHIP_ALIGN_SIZE == 0, "DeferredCompletionSlab size must preserve array element cache-line boundaries" ); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_kernel_api.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_kernel_api.h index f415a24fc9..884a45561e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_kernel_api.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_kernel_api.h @@ -38,7 +38,7 @@ namespace pto2::detail { inline __aicore__ void defer_load_slab(AsyncCtx &ctx) { if (ctx.completion_count == nullptr) return; #if defined(__CCE_KT_TEST__) || defined(__CCE_AICORE__) || defined(__DAV_C220__) - uintptr_t line = reinterpret_cast(ctx.completion_count) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + uintptr_t line = reinterpret_cast(ctx.completion_count) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); dcci((__gm__ int32_t *)line, SINGLE_CACHE_LINE); #else __asm__ __volatile__("" ::: "memory"); @@ -54,10 +54,10 @@ inline __aicore__ void defer_error(AsyncCtx &ctx, int32_t error_code) { inline __aicore__ void defer_flush_range(volatile __gm__ void *addr, uint32_t size_bytes) { if (addr == nullptr || size_bytes == 0) return; #if defined(__CCE_KT_TEST__) || defined(__CCE_AICORE__) || defined(__DAV_C220__) - uintptr_t start = reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + uintptr_t start = reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); uintptr_t end = - (reinterpret_cast(addr) + size_bytes + PTO2_ALIGN_SIZE - 1u) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); - for (uintptr_t p = start; p < end; p += PTO2_ALIGN_SIZE) { + (reinterpret_cast(addr) + size_bytes + CHIP_ALIGN_SIZE - 1u) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); + for (uintptr_t p = start; p < end; p += CHIP_ALIGN_SIZE) { dcci((__gm__ int32_t *)p, SINGLE_CACHE_LINE, CACHELINE_OUT); } #else diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_wait.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_wait.h index c7a36e6d38..6000abeb78 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_wait.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/async_wait.h @@ -34,7 +34,7 @@ inline constexpr int32_t MAX_ASYNC_WAITS = 64; // application layer: translating drained messages into wait-list state. inline uintptr_t mailbox_cache_line(const volatile void *addr) { - return reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + return reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); } struct CompletionCondition; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h index 7410c97278..332bbb91d8 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h @@ -20,7 +20,7 @@ #include "runtime_status.h" inline uintptr_t sdma_completion_cache_line(const volatile void *addr) { - return reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + return reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); } // PTO-ISA stores the latest completed post ID as a monotonic uint64_t in one @@ -32,7 +32,7 @@ inline CompletionPollResult poll_sdma_post_done_record(uint64_t record_addr, uin return {CompletionPollState::FAILED, SIMPLER_ERROR_ASYNC_COMPLETION_INVALID}; } volatile uint64_t *record = reinterpret_cast(static_cast(record_addr)); - cache_invalidate_range(reinterpret_cast(sdma_completion_cache_line(record)), PTO2_ALIGN_SIZE); + cache_invalidate_range(reinterpret_cast(sdma_completion_cache_line(record)), CHIP_ALIGN_SIZE); uint64_t completed_post_id = __atomic_load_n(record, __ATOMIC_ACQUIRE); return { completed_post_id >= expected_post_id ? CompletionPollState::READY : CompletionPollState::PENDING, diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/constants.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/constants.h index 07251cc396..3d22a80ea7 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/constants.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/constants.h @@ -12,8 +12,8 @@ #ifndef SRC_A2A3_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_PTO_CONSTANTS_H_ #define SRC_A2A3_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_PTO_CONSTANTS_H_ -#define PTO2_ALIGN_SIZE 64 // Cache line alignment -#define PTO2_PACKED_OUTPUT_ALIGN 1024 // Each output in packed buffer aligned to 1024B; gap is padding -#define PTO2_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#define CHIP_ALIGN_SIZE 64 // Cache line alignment +#define PACKED_OUTPUT_ALIGN 1024 // Each output in packed buffer aligned to 1024B; gap is padding +#define CHIP_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) #endif // SRC_A2A3_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_PTO_CONSTANTS_H_ diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h index 81db71c5bd..b9ad62f2b7 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h @@ -47,8 +47,8 @@ #define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) #endif -#ifndef PTO2_ALIGN_UP -#define PTO2_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#ifndef CHIP_ALIGN_UP +#define CHIP_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) #endif // Verify hardcoded indices in intrinsic.h match the computed values. @@ -122,7 +122,7 @@ struct alignas(64) DispatchPayload { static_assert(sizeof(args[0]) == 8); static_assert( - PTO2_ALIGN_UP((MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]), 64) == + CHIP_ALIGN_UP((MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]), 64) == (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]) ); }; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 77a0181546..1dab2c0053 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -552,8 +552,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { continue; } layout.offsets[i] = layout.total_output_size; - layout.buffer_sizes[i] = - PTO2_ALIGN_UP(args.tensor(i).create_info().buffer_size_bytes(), PTO2_PACKED_OUTPUT_ALIGN); + layout.buffer_sizes[i] = CHIP_ALIGN_UP(args.tensor(i).create_info().buffer_size_bytes(), PACKED_OUTPUT_ALIGN); layout.total_output_size += layout.buffer_sizes[i]; } return layout; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index e6f57b4aba..4d2db6b714 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -124,7 +124,7 @@ class TaskAllocator { */ TaskAllocResult alloc(int32_t output_size, ChipTaskSlotState *oldest_open_task = nullptr) { uint64_t aligned_size = - output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; + output_size > 0 ? CHIP_ALIGN_UP(static_cast(output_size), CHIP_ALIGN_SIZE) : 0; int spin_count = 0; int32_t prev_last_alive = last_alive_ptr_->load(std::memory_order_acquire); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index bbe11d343b..2e3c92457f 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -395,7 +395,7 @@ struct TaskPayload { } // Round up to cache line boundary. Both arrays are 128B so no overrun. // Eliminates branches; extra bytes within the same CL have zero additional cost. - memcpy(scalars, args.scalars(), PTO2_ALIGN_UP(args.scalar_count() * sizeof(uint64_t), 64)); + memcpy(scalars, args.scalars(), CHIP_ALIGN_UP(args.scalar_count() * sizeof(uint64_t), 64)); // reset_for_reuse deliberately skips this cold payload. These fields // are consumer-side state and must be initialized on every submit even diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index bb872ab8f7..28288a7304 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -52,7 +52,7 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Align the slots[] base to a full cache line so MPMC CAS traffic on the // first slot cannot false-share with whatever region sits in front of us // (e.g. orchestrator tensormap heads written by the orch thread). - return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), PTO2_ALIGN_SIZE); + return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), CHIP_ALIGN_SIZE); } bool ready_queue_init_data_from_layout(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity) { @@ -150,7 +150,7 @@ SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capaci // Force a cache-line base so Orch-side dep_pool writes do not invalidate // adjacent multi-threaded regions like ready_queue.slots. layout.off_dep_pool_entries[r] = - arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(DepListEntry), PTO2_ALIGN_SIZE); + arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(DepListEntry), CHIP_ALIGN_SIZE); } return layout; } @@ -326,13 +326,13 @@ OrchestratorLayout OrchestratorState::reserve_layout( for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { const size_t fanin_pool_bytes = - PTO2_ALIGN_UP(static_cast(dep_pool_capacities[r]) * sizeof(FaninSpillEntry), PTO2_ALIGN_SIZE); - layout.off_fanin_pool[r] = arena.reserve(fanin_pool_bytes, PTO2_ALIGN_SIZE); + CHIP_ALIGN_UP(static_cast(dep_pool_capacities[r]) * sizeof(FaninSpillEntry), CHIP_ALIGN_SIZE); + layout.off_fanin_pool[r] = arena.reserve(fanin_pool_bytes, CHIP_ALIGN_SIZE); always_assert(task_window_sizes[r] > 0 && (task_window_sizes[r] & (task_window_sizes[r] - 1)) == 0); const size_t seen_epoch_bytes = - PTO2_ALIGN_UP(static_cast(task_window_sizes[r]) * sizeof(uint32_t), PTO2_ALIGN_SIZE); - layout.off_fanin_seen_epoch[r] = arena.reserve(seen_epoch_bytes, PTO2_ALIGN_SIZE); + CHIP_ALIGN_UP(static_cast(task_window_sizes[r]) * sizeof(uint32_t), CHIP_ALIGN_SIZE); + layout.off_fanin_seen_epoch[r] = arena.reserve(seen_epoch_bytes, CHIP_ALIGN_SIZE); } layout.off_scope_tasks = arena.reserve(static_cast(layout.scope_tasks_cap) * sizeof(uintptr_t), alignof(ChipTaskSlotState *)); @@ -386,15 +386,15 @@ bool OrchestratorState::init_data_from_layout( ); heap_offset += heap_sizes[r]; - const size_t fanin_pool_bytes = PTO2_ALIGN_UP( - static_cast(layout.dep_pool_capacities[r]) * sizeof(FaninSpillEntry), PTO2_ALIGN_SIZE + const size_t fanin_pool_bytes = CHIP_ALIGN_UP( + static_cast(layout.dep_pool_capacities[r]) * sizeof(FaninSpillEntry), CHIP_ALIGN_SIZE ); auto *fanin_entries = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); memset(fanin_entries, 0, fanin_pool_bytes); orch->rings[r].fanin_pool.init(fanin_entries, layout.dep_pool_capacities[r], orch_err); - const size_t seen_epoch_bytes = PTO2_ALIGN_UP( - static_cast(layout.tensor_map.task_window_sizes[r]) * sizeof(uint32_t), PTO2_ALIGN_SIZE + const size_t seen_epoch_bytes = CHIP_ALIGN_UP( + static_cast(layout.tensor_map.task_window_sizes[r]) * sizeof(uint32_t), CHIP_ALIGN_SIZE ); auto *seen_epoch = static_cast(arena.region_ptr(layout.off_fanin_seen_epoch[r])); memset(seen_epoch, 0, seen_epoch_bytes); @@ -536,7 +536,7 @@ RuntimeArenaLayout runtime_reserve_layout( } layout.offsets.orch = OrchestratorState::reserve_layout(arena, task_window_sizes_i32, dep_pool_capacities); layout.offsets.sched = SchedulerState::reserve_layout(arena, dep_pool_capacities); - layout.offsets.off_runtime = arena.reserve(sizeof(RuntimeContext), PTO2_ALIGN_SIZE); + layout.offsets.off_runtime = arena.reserve(sizeof(RuntimeContext), CHIP_ALIGN_SIZE); layout.offsets.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); layout.offsets.arena_size = arena.total_size(); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index b1e17c0a43..d1d01d984b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -97,7 +97,7 @@ bool SharedMemoryHandle::init_per_ring( SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); - const size_t off_buffer = arena.reserve(static_cast(buffer_size), PTO2_ALIGN_SIZE); + const size_t off_buffer = arena.reserve(static_cast(buffer_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; auto *handle = static_cast(arena.region_ptr(off_handle)); @@ -143,15 +143,15 @@ void SharedMemoryHandle::init_header_per_ring( header->orchestrator_done.store(0, std::memory_order_relaxed); // Per-ring layout info - uint64_t offset = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t offset = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { header->rings[r].task_window_size = task_window_sizes[r]; header->rings[r].task_window_mask = static_cast(task_window_sizes[r] - 1); header->rings[r].heap_size = heap_sizes[r]; header->rings[r].task_descriptors_offset = offset; - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), CHIP_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); } header->total_size = sm_size; @@ -234,7 +234,7 @@ bool ChipRingFlowControl::validate(SharedMemoryHandle *handle, int32_t ring_id) if (h->rings[ring_id].task_descriptors_offset >= h->total_size) return false; // Check pointer alignment - if ((uintptr_t)h->rings[ring_id].task_descriptors % PTO2_ALIGN_SIZE != 0) return false; + if ((uintptr_t)h->rings[ring_id].task_descriptors % CHIP_ALIGN_SIZE != 0) return false; // Check flow control pointer sanity int32_t current = current_task_index.load(std::memory_order_acquire); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 8a8692acf7..4cb9d8387c 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -125,7 +125,7 @@ static_assert( * * Contains per-ring flow control and global layout information. */ -struct alignas(PTO2_ALIGN_SIZE) SharedMemoryHeader { +struct alignas(CHIP_ALIGN_SIZE) SharedMemoryHeader { // === PER-RING FLOW CONTROL + LAYOUT INFO (set once at init) === SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; @@ -268,7 +268,7 @@ inline std::atomic *ring_last_task_alive_addr(void *sm_dev_base, int ri // Byte offsets (from the SM base) of one ring's three segments. The per-ring // layout is: header, then for each ring descriptors -> payloads -> slot_states, -// every segment PTO2_ALIGN_UP-padded. +// every segment CHIP_ALIGN_UP-padded. struct ChipRingSegmentOffsets { uint64_t descriptors; uint64_t payloads; @@ -285,19 +285,19 @@ struct ChipRingSegmentOffsets { inline ChipRingSegmentOffsets ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int ring_id) noexcept { assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "sm_layout: ring_id out of range"); - uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t off = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); for (int r = 0; r < ring_id; r++) { - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), CHIP_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); } ChipRingSegmentOffsets o{}; o.descriptors = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); o.payloads = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskPayload), CHIP_ALIGN_SIZE); o.slot_states = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[ring_id] * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); o.end = off; return o; } diff --git a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp index c59547b2b7..54c3f455e3 100644 --- a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp @@ -562,12 +562,12 @@ int32_t run_host_orchestration( // orch::prepare_task and shipped bounded to total_tasks below. const sm_layout::ChipRingSegmentOffsets sm_segs = sm_layout::ring_segment_offsets(eff_task_window_size); // Over-allocated and rounded up: every segment offset is a multiple of - // PTO2_ALIGN_SIZE and ChipTaskSlotState is alignas(64), which a plain + // CHIP_ALIGN_SIZE and ChipTaskSlotState is alignas(64), which a plain // new uint8_t[] does not guarantee. - std::unique_ptr host_sm_buf(new uint8_t[sm_size + PTO2_ALIGN_SIZE]); + std::unique_ptr host_sm_buf(new uint8_t[sm_size + CHIP_ALIGN_SIZE]); void *host_sm = reinterpret_cast( - (reinterpret_cast(host_sm_buf.get()) + PTO2_ALIGN_SIZE - 1) & - ~static_cast(PTO2_ALIGN_SIZE - 1) + (reinterpret_cast(host_sm_buf.get()) + CHIP_ALIGN_SIZE - 1) & + ~static_cast(CHIP_ALIGN_SIZE - 1) ); std::memset(host_sm, 0, sm_segs.descriptors); @@ -789,8 +789,8 @@ int32_t run_host_orchestration( // same boundary it starts on, so an access at the tail of the last packed buffer // stays inside the region even when its width exceeds the bytes that buffer // asked for. - const uint64_t heap_bytes = PTO2_ALIGN_UP( - std::max(orchestrator.task_allocator.heap_used_bytes(), PTO2_ALIGN_SIZE), + const uint64_t heap_bytes = CHIP_ALIGN_UP( + std::max(orchestrator.task_allocator.heap_used_bytes(), CHIP_ALIGN_SIZE), DeviceArena::kDefaultBaseAlign ); if (api->setup_static_arena(heap_bytes, /*gm_sm_size=*/0, device_arena_bytes) != 0) { @@ -853,14 +853,14 @@ int32_t run_host_orchestration( // One host source for one copy: the copied zone and shared-memory image at // exactly the offsets they occupy on the device. // Over-allocated and rounded up because every segment offset is - // PTO2_ALIGN_SIZE-aligned and ChipTaskSlotState is alignas(64), which a byte + // CHIP_ALIGN_SIZE-aligned and ChipTaskSlotState is alignas(64), which a byte // vector's data() is not. const uint64_t copied_bytes = layout.off_copied_end - layout.off_copied_begin; const uint64_t upload_bytes = copied_bytes + image_bytes; - std::vector storage(upload_bytes + PTO2_ALIGN_SIZE, std::byte{0}); + std::vector storage(upload_bytes + CHIP_ALIGN_SIZE, std::byte{0}); char *upload_base = reinterpret_cast( - (reinterpret_cast(storage.data()) + PTO2_ALIGN_SIZE - 1) & - ~static_cast(PTO2_ALIGN_SIZE - 1) + (reinterpret_cast(storage.data()) + CHIP_ALIGN_SIZE - 1) & + ~static_cast(CHIP_ALIGN_SIZE - 1) ); // The copied zone carries no host address: the orchestrator is host-only and diff --git a/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox.h b/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox.h index 9ed3480e38..27fd8a8853 100644 --- a/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox.h +++ b/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox.h @@ -63,7 +63,7 @@ struct AICoreCompletionMailboxMessage { uint32_t _pad[3]; }; -static_assert(sizeof(AICoreCompletionMailboxMessage) == PTO2_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); +static_assert(sizeof(AICoreCompletionMailboxMessage) == CHIP_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); static_assert( sizeof(std::atomic) == sizeof(uint64_t), "std::atomic must be layout-compatible with uint64_t for the message slot layout to hold" @@ -89,11 +89,11 @@ struct AICoreCompletionMsgView { struct AICoreCompletionMailbox { // head and tail live on their own cache lines so producer CAS contention // on head can't false-share with the consumer's tail updates. - alignas(PTO2_ALIGN_SIZE) std::atomic head; - uint8_t _head_pad[PTO2_ALIGN_SIZE - sizeof(uint64_t)]; - alignas(PTO2_ALIGN_SIZE) std::atomic tail; - uint8_t _tail_pad[PTO2_ALIGN_SIZE - sizeof(uint64_t)]; - alignas(PTO2_ALIGN_SIZE) AICoreCompletionMailboxMessage entries[AICORE_COMPLETION_MAILBOX_CAPACITY]; + alignas(CHIP_ALIGN_SIZE) std::atomic head; + uint8_t _head_pad[CHIP_ALIGN_SIZE - sizeof(uint64_t)]; + alignas(CHIP_ALIGN_SIZE) std::atomic tail; + uint8_t _tail_pad[CHIP_ALIGN_SIZE - sizeof(uint64_t)]; + alignas(CHIP_ALIGN_SIZE) AICoreCompletionMailboxMessage entries[AICORE_COMPLETION_MAILBOX_CAPACITY]; // Cheap, lock-free pending hint. Callers may invoke this outside the // consumer lock; a stale answer only over/under-triggers a drain attempt. @@ -218,5 +218,5 @@ struct AICoreCompletionMailbox { }; static_assert( - sizeof(AICoreCompletionMailbox) % PTO2_ALIGN_SIZE == 0, "AICoreCompletionMailbox size must be cache-line aligned" + sizeof(AICoreCompletionMailbox) % CHIP_ALIGN_SIZE == 0, "AICoreCompletionMailbox size must be cache-line aligned" ); diff --git a/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h b/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h index 9aa666b57f..46ba742c1a 100644 --- a/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h +++ b/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h @@ -53,13 +53,13 @@ struct DeferredCompletionEntry { static_assert(sizeof(DeferredCompletionEntry) == 32, "DeferredCompletionEntry layout drift"); -struct alignas(PTO2_ALIGN_SIZE) DeferredCompletionSlab { +struct alignas(CHIP_ALIGN_SIZE) DeferredCompletionSlab { volatile uint32_t count; volatile int32_t error_code; DeferredCompletionEntry entries[MAX_COMPLETIONS_PER_TASK]; }; static_assert( - sizeof(DeferredCompletionSlab) % PTO2_ALIGN_SIZE == 0, + sizeof(DeferredCompletionSlab) % CHIP_ALIGN_SIZE == 0, "DeferredCompletionSlab size must preserve array element cache-line boundaries" ); diff --git a/src/a5/runtime/host_build_graph/runtime/async_kernel_api.h b/src/a5/runtime/host_build_graph/runtime/async_kernel_api.h index f415a24fc9..884a45561e 100644 --- a/src/a5/runtime/host_build_graph/runtime/async_kernel_api.h +++ b/src/a5/runtime/host_build_graph/runtime/async_kernel_api.h @@ -38,7 +38,7 @@ namespace pto2::detail { inline __aicore__ void defer_load_slab(AsyncCtx &ctx) { if (ctx.completion_count == nullptr) return; #if defined(__CCE_KT_TEST__) || defined(__CCE_AICORE__) || defined(__DAV_C220__) - uintptr_t line = reinterpret_cast(ctx.completion_count) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + uintptr_t line = reinterpret_cast(ctx.completion_count) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); dcci((__gm__ int32_t *)line, SINGLE_CACHE_LINE); #else __asm__ __volatile__("" ::: "memory"); @@ -54,10 +54,10 @@ inline __aicore__ void defer_error(AsyncCtx &ctx, int32_t error_code) { inline __aicore__ void defer_flush_range(volatile __gm__ void *addr, uint32_t size_bytes) { if (addr == nullptr || size_bytes == 0) return; #if defined(__CCE_KT_TEST__) || defined(__CCE_AICORE__) || defined(__DAV_C220__) - uintptr_t start = reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + uintptr_t start = reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); uintptr_t end = - (reinterpret_cast(addr) + size_bytes + PTO2_ALIGN_SIZE - 1u) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); - for (uintptr_t p = start; p < end; p += PTO2_ALIGN_SIZE) { + (reinterpret_cast(addr) + size_bytes + CHIP_ALIGN_SIZE - 1u) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); + for (uintptr_t p = start; p < end; p += CHIP_ALIGN_SIZE) { dcci((__gm__ int32_t *)p, SINGLE_CACHE_LINE, CACHELINE_OUT); } #else diff --git a/src/a5/runtime/host_build_graph/runtime/async_wait.h b/src/a5/runtime/host_build_graph/runtime/async_wait.h index db86ba05d1..3c7b10186d 100644 --- a/src/a5/runtime/host_build_graph/runtime/async_wait.h +++ b/src/a5/runtime/host_build_graph/runtime/async_wait.h @@ -33,7 +33,7 @@ inline constexpr int32_t MAX_ASYNC_WAITS = 64; // application layer: translating drained messages into wait-list state. inline uintptr_t mailbox_cache_line(const volatile void *addr) { - return reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + return reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); } struct CompletionCondition; diff --git a/src/a5/runtime/host_build_graph/runtime/constants.h b/src/a5/runtime/host_build_graph/runtime/constants.h index 5fac988770..3c684cc67e 100644 --- a/src/a5/runtime/host_build_graph/runtime/constants.h +++ b/src/a5/runtime/host_build_graph/runtime/constants.h @@ -11,6 +11,6 @@ #pragma once -#define PTO2_ALIGN_SIZE 64 // Cache line alignment -#define PTO2_PACKED_OUTPUT_ALIGN 1024 // Each output in packed buffer aligned to 1024B; gap is padding -#define PTO2_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#define CHIP_ALIGN_SIZE 64 // Cache line alignment +#define PACKED_OUTPUT_ALIGN 1024 // Each output in packed buffer aligned to 1024B; gap is padding +#define CHIP_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) diff --git a/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h b/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h index 13169f89bc..021a4d7cb4 100644 --- a/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h +++ b/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h @@ -46,8 +46,8 @@ #define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) #endif -#ifndef PTO2_ALIGN_UP -#define PTO2_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#ifndef CHIP_ALIGN_UP +#define CHIP_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) #endif // Verify hardcoded indices in intrinsic.h match the computed values. @@ -124,7 +124,7 @@ struct alignas(64) DispatchPayload { static_assert(sizeof(args[0]) == 8); static_assert( - PTO2_ALIGN_UP((MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]), 64) == + CHIP_ALIGN_UP((MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]), 64) == (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]) ); }; diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index c39851f37f..2f382301bd 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -971,7 +971,7 @@ bool graph_fill_definition(const GraphRecording &recording, GraphDefinition defi return false; } node_offsets[i] = required_heap; - const uint64_t output_bytes = PTO2_ALIGN_UP(source.total_output_size, PTO2_ALIGN_SIZE); + const uint64_t output_bytes = CHIP_ALIGN_UP(source.total_output_size, CHIP_ALIGN_SIZE); if (required_heap > UINT64_MAX - output_bytes) return false; required_heap += output_bytes; @@ -1231,8 +1231,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { continue; } layout.offsets[i] = layout.total_output_size; - layout.buffer_sizes[i] = - PTO2_ALIGN_UP(args.tensor(i).create_info().buffer_size_bytes(), PTO2_PACKED_OUTPUT_ALIGN); + layout.buffer_sizes[i] = CHIP_ALIGN_UP(args.tensor(i).create_info().buffer_size_bytes(), PACKED_OUTPUT_ALIGN); layout.total_output_size += layout.buffer_sizes[i]; } return layout; @@ -1276,7 +1275,7 @@ static bool prepare_task( // ChipTensor being two cache lines. The fanin cursor advances at publish, not // here — see the comment where it does. const uint64_t window = orch->sm_header->ring.task_window_size; - const int32_t scalar_span = PTO2_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); + const int32_t scalar_span = CHIP_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); debug_assert(static_cast(orch->tensor_pool_cursor) + args.tensor_count() <= window * MAX_TENSOR_ARGS); debug_assert(static_cast(orch->scalar_pool_cursor) + scalar_span <= window * MAX_SCALAR_ARGS); debug_assert(static_cast(orch->fanin_pool_cursor) + CHIP_MAX_FANIN <= window * CHIP_MAX_FANIN); @@ -1614,7 +1613,7 @@ static TaskOutputTensors submit_task_common( // equality holds only while nothing between the bind and here bound another fanin // region, which is what makes the deferred advance safe. debug_assert(orch->fanin_pool_cursor == static_cast(payload.fanin_data() - orch->fanin_pool)); - orch->fanin_pool_cursor += PTO2_ALIGN_UP(fanin_builder.count, ARG_POOL_ALIGN / (int32_t)sizeof(int32_t)); + orch->fanin_pool_cursor += CHIP_ALIGN_UP(fanin_builder.count, ARG_POOL_ALIGN / (int32_t)sizeof(int32_t)); (void)sched; @@ -1773,7 +1772,7 @@ bool graph_submit_outer( const uint64_t task_window = orch->sm_header->ring.task_window_size; const int32_t tensor_slots = static_cast(graph_boundary_tensor_pool_slots(static_cast(args.tensor_count()))); - const int32_t scalar_span = PTO2_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); + const int32_t scalar_span = CHIP_ALIGN_UP(args.scalar_count(), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)); if (static_cast(orch->tensor_pool_cursor) + tensor_slots > task_window * MAX_TENSOR_ARGS || static_cast(orch->scalar_pool_cursor) + scalar_span > task_window * MAX_SCALAR_ARGS) { LOG_WARN("%s", "[GraphExecution] boundary exceeds the argument pools; using ordinary path"); @@ -1841,7 +1840,7 @@ bool graph_submit_outer( if (args.scalar_count() != 0) { std::memcpy( payload.scalar_data(), args.scalar_data(), - PTO2_ALIGN_UP(static_cast(args.scalar_count()) * sizeof(uint64_t), ARG_POOL_ALIGN) + CHIP_ALIGN_UP(static_cast(args.scalar_count()) * sizeof(uint64_t), ARG_POOL_ALIGN) ); } @@ -1884,7 +1883,7 @@ bool graph_submit_outer( // equality holds only while nothing between the bind and here bound another fanin // region, which is what makes the deferred advance safe. debug_assert(orch->fanin_pool_cursor == static_cast(payload.fanin_data() - orch->fanin_pool)); - orch->fanin_pool_cursor += PTO2_ALIGN_UP(fanin_builder.count, ARG_POOL_ALIGN / (int32_t)sizeof(int32_t)); + orch->fanin_pool_cursor += CHIP_ALIGN_UP(fanin_builder.count, ARG_POOL_ALIGN / (int32_t)sizeof(int32_t)); pending.outer_slot = &slot; state->pending_uploads.push_back(pending); @@ -1983,7 +1982,7 @@ TaskOutputTensors graph_record_submit_node( const PTO2OutputLayout layout = calculate_output_layout(args); const uint64_t aligned_output = - layout.total_output_size > 0 ? PTO2_ALIGN_UP(static_cast(layout.total_output_size), PTO2_ALIGN_SIZE) : + layout.total_output_size > 0 ? CHIP_ALIGN_UP(static_cast(layout.total_output_size), CHIP_ALIGN_SIZE) : 0; if (recording.next_virtual_offset > GRAPH_RECORD_VIRTUAL_BASE - aligned_output) { recording.unsupported = true; diff --git a/src/a5/runtime/host_build_graph/runtime/ring_buffer.h b/src/a5/runtime/host_build_graph/runtime/ring_buffer.h index 3bbe149f71..9d2dc1eac6 100644 --- a/src/a5/runtime/host_build_graph/runtime/ring_buffer.h +++ b/src/a5/runtime/host_build_graph/runtime/ring_buffer.h @@ -102,7 +102,7 @@ class TaskAllocator { */ TaskAllocResult alloc(int32_t output_size) { uint64_t aligned_size = - output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; + output_size > 0 ? CHIP_ALIGN_UP(static_cast(output_size), CHIP_ALIGN_SIZE) : 0; if (error_code_ptr_ != nullptr && error_code_ptr_->load(std::memory_order_acquire) != SIMPLER_ERROR_NONE) { return {-1, -1, nullptr, nullptr}; @@ -128,7 +128,7 @@ class TaskAllocator { return false; } const uint64_t aligned_size = - output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; + output_size > 0 ? CHIP_ALIGN_UP(static_cast(output_size), CHIP_ALIGN_SIZE) : 0; void *base = try_bump_heap(aligned_size); if (base == nullptr) return false; *packed_base = base; diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_core.h b/src/a5/runtime/host_build_graph/runtime/runtime_core.h index 9fba17166b..6fac6296fd 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_core.h @@ -140,7 +140,7 @@ struct RuntimeArenaLayout { // therefore not known when this layout is built. bind grows the device region // to cover it once orchestration ends. // - // The copied zone is padded to a PTO2_ALIGN_SIZE boundary, so that tail begins + // The copied zone is padded to a CHIP_ALIGN_SIZE boundary, so that tail begins // exactly at off_copied_end: the copied zone and the shared-memory image are // adjacent on the device and travel as one copy. // diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 5d023ab729..fb4b3555bb 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -458,7 +458,7 @@ struct TaskPayload { // cache lines (ARG_POOL_ALIGN), so the rounded copy stays inside this // task's own region. Eliminates branches; extra bytes within the same CL have // zero additional cost. - memcpy(scalar_data(), args.scalars(), PTO2_ALIGN_UP(args.scalar_count() * sizeof(uint64_t), 64)); + memcpy(scalar_data(), args.scalars(), CHIP_ALIGN_UP(args.scalar_count() * sizeof(uint64_t), 64)); // The ring's payload storage is reused raw memory that no constructor runs // over, so an unset predicate reads back as whatever the slot last held — diff --git a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp index 11dda9bb49..e9f7689253 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -36,7 +36,7 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Align the slots[] base to a full cache line so MPMC CAS traffic on the // first slot cannot false-share with whatever region sits in front of us. - return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), PTO2_ALIGN_SIZE); + return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), CHIP_ALIGN_SIZE); } // Initialize the queue header only. slots[] carries the sequence ramp push @@ -267,9 +267,9 @@ RuntimeArenaLayout runtime_reserve_layout(DeviceArena &arena, uint64_t task_wind layout.sched = SchedulerState::reserve_layout(arena); layout.off_copied_begin = arena.total_size(); - // Padded to a PTO2_ALIGN_SIZE boundary: the shared-memory image starts at + // Padded to a CHIP_ALIGN_SIZE boundary: the shared-memory image starts at // off_copied_end on the device and its segment offsets are aligned from there. - layout.off_runtime = arena.reserve(PTO2_ALIGN_UP(sizeof(RuntimeContext), PTO2_ALIGN_SIZE), PTO2_ALIGN_SIZE); + layout.off_runtime = arena.reserve(CHIP_ALIGN_UP(sizeof(RuntimeContext), CHIP_ALIGN_SIZE), CHIP_ALIGN_SIZE); layout.off_copied_end = arena.total_size(); layout.arena_size = arena.total_size(); diff --git a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 7cc4fd7308..8ad041e91d 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -113,7 +113,7 @@ bool SharedMemoryHandle::attach_populated( SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); - const size_t off_buffer = arena.reserve(static_cast(buffer_size), PTO2_ALIGN_SIZE); + const size_t off_buffer = arena.reserve(static_cast(buffer_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; auto *handle = static_cast(arena.region_ptr(off_handle)); @@ -149,13 +149,13 @@ void SharedMemoryHandle::init_header(uint64_t task_window_size) { header->orchestrator_done.store(0, std::memory_order_relaxed); // Ring layout info - uint64_t offset = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t offset = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); header->ring.task_window_size = task_window_size; header->ring.task_window_mask = static_cast(task_window_size - 1); header->ring.task_descriptors_offset = offset; - offset += PTO2_ALIGN_UP(task_window_size * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_size * sizeof(TaskPayload), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_size * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_size * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_size * sizeof(TaskPayload), CHIP_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_size * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); header->total_size = sm_size; @@ -221,7 +221,7 @@ bool ChipRingFlowControl::validate(SharedMemoryHandle *handle) const { if (h->ring.task_descriptors_offset >= h->total_size) return false; // Check pointer alignment - if ((uintptr_t)h->ring.task_descriptors % PTO2_ALIGN_SIZE != 0) return false; + if ((uintptr_t)h->ring.task_descriptors % CHIP_ALIGN_SIZE != 0) return false; // Check flow control pointer sanity int32_t current = current_task_index.load(std::memory_order_acquire); diff --git a/src/a5/runtime/host_build_graph/runtime/shared_memory.h b/src/a5/runtime/host_build_graph/runtime/shared_memory.h index 018e9e5056..f65fc338ef 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a5/runtime/host_build_graph/runtime/shared_memory.h @@ -165,7 +165,7 @@ static_assert( * * Contains per-ring flow control and global layout information. */ -struct alignas(PTO2_ALIGN_SIZE) SharedMemoryHeader { +struct alignas(CHIP_ALIGN_SIZE) SharedMemoryHeader { // === RING FLOW CONTROL + LAYOUT INFO (single ring, set once at init) === SharedMemoryRingHeader ring; @@ -299,7 +299,7 @@ inline std::atomic *ring_current_task_index_addr(void *sm_dev_base) noe // Byte offsets (from the SM base) of the ring's segments. The layout is: header, then // descriptors -> payloads -> slot_states -> completion_flags -> the three argument -// pools, every segment PTO2_ALIGN_UP-padded. RingImageExtents dimensions them: the +// pools, every segment CHIP_ALIGN_UP-padded. RingImageExtents dimensions them: the // mirror for the worst case the API allows, the image for what this bind holds, which // is what makes the live prefixes contiguous and the upload one copy. // @@ -343,22 +343,22 @@ inline RingImageExtents mirror_extents(uint64_t task_window_size) noexcept { // or reordering a segment is a one-line edit here; every consumer follows // automatically, so the layout walk can never silently disagree across call sites. inline ChipRingSegmentOffsets ring_segment_offsets(const RingImageExtents &e) noexcept { - uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t off = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); ChipRingSegmentOffsets o{}; o.descriptors = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.slots * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); o.payloads = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(TaskPayload), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.slots * sizeof(TaskPayload), CHIP_ALIGN_SIZE); o.slot_states = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.slots * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); o.completion_flags = off; - off += PTO2_ALIGN_UP(e.slots * sizeof(std::atomic), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.slots * sizeof(std::atomic), CHIP_ALIGN_SIZE); o.fanin_pool = off; - off += PTO2_ALIGN_UP(e.fanin_elems * sizeof(int32_t), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.fanin_elems * sizeof(int32_t), CHIP_ALIGN_SIZE); o.tensor_pool = off; - off += PTO2_ALIGN_UP(e.tensor_elems * sizeof(ChipTensor), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.tensor_elems * sizeof(ChipTensor), CHIP_ALIGN_SIZE); o.scalar_pool = off; - off += PTO2_ALIGN_UP(e.scalar_elems * sizeof(uint64_t), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(e.scalar_elems * sizeof(uint64_t), CHIP_ALIGN_SIZE); o.end = off; return o; } @@ -436,7 +436,7 @@ inline uint64_t rebased_heap_addr(uint64_t addr, const HeapRebase &rebase) noexc // orchestrator wrote into an image dimensioned for what this bind holds, where the // prefixes are contiguous and can travel as one copy. // -// `out_base` must be PTO2_ALIGN_SIZE-aligned and hold +// `out_base` must be CHIP_ALIGN_SIZE-aligned and hold // `ring_segment_offsets(image_extents(used)).end` bytes. Returns that byte count. // // Three things the restack has to fix up, all because the image is not the mirror: diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 0ea43e79a7..82f8a1e4bb 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -178,7 +178,7 @@ total = ALIGN(Header) + ALIGN(window_size * sizeof(ChipTaskSlotState)) ] ``` -Alignment is 64 bytes (`PTO2_ALIGN_SIZE`). +Alignment is 64 bytes (`CHIP_ALIGN_SIZE`). --- diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h index c9e5698203..70cf1d9e1a 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h @@ -64,7 +64,7 @@ struct AICoreCompletionMailboxMessage { uint32_t _pad[3]; }; -static_assert(sizeof(AICoreCompletionMailboxMessage) == PTO2_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); +static_assert(sizeof(AICoreCompletionMailboxMessage) == CHIP_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); static_assert( sizeof(std::atomic) == sizeof(uint64_t), "std::atomic must be layout-compatible with uint64_t for the message slot layout to hold" @@ -90,11 +90,11 @@ struct AICoreCompletionMsgView { struct AICoreCompletionMailbox { // head and tail live on their own cache lines so producer CAS contention // on head can't false-share with the consumer's tail updates. - alignas(PTO2_ALIGN_SIZE) std::atomic head; - uint8_t _head_pad[PTO2_ALIGN_SIZE - sizeof(uint64_t)]; - alignas(PTO2_ALIGN_SIZE) std::atomic tail; - uint8_t _tail_pad[PTO2_ALIGN_SIZE - sizeof(uint64_t)]; - alignas(PTO2_ALIGN_SIZE) AICoreCompletionMailboxMessage entries[AICORE_COMPLETION_MAILBOX_CAPACITY]; + alignas(CHIP_ALIGN_SIZE) std::atomic head; + uint8_t _head_pad[CHIP_ALIGN_SIZE - sizeof(uint64_t)]; + alignas(CHIP_ALIGN_SIZE) std::atomic tail; + uint8_t _tail_pad[CHIP_ALIGN_SIZE - sizeof(uint64_t)]; + alignas(CHIP_ALIGN_SIZE) AICoreCompletionMailboxMessage entries[AICORE_COMPLETION_MAILBOX_CAPACITY]; // Cheap, lock-free pending hint. Callers may invoke this outside the // consumer lock; a stale answer only over/under-triggers a drain attempt. @@ -195,7 +195,7 @@ struct AICoreCompletionMailbox { }; static_assert( - sizeof(AICoreCompletionMailbox) % PTO2_ALIGN_SIZE == 0, "AICoreCompletionMailbox size must be cache-line aligned" + sizeof(AICoreCompletionMailbox) % CHIP_ALIGN_SIZE == 0, "AICoreCompletionMailbox size must be cache-line aligned" ); #endif // SRC_A5_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_AICORE_COMPLETION_MAILBOX_H_ diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h index d8906e8bf2..3bdbec18be 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h @@ -55,14 +55,14 @@ struct DeferredCompletionEntry { static_assert(sizeof(DeferredCompletionEntry) == 32, "DeferredCompletionEntry layout drift"); -struct alignas(PTO2_ALIGN_SIZE) DeferredCompletionSlab { +struct alignas(CHIP_ALIGN_SIZE) DeferredCompletionSlab { volatile uint32_t count; volatile int32_t error_code; DeferredCompletionEntry entries[MAX_COMPLETIONS_PER_TASK]; }; static_assert( - sizeof(DeferredCompletionSlab) % PTO2_ALIGN_SIZE == 0, + sizeof(DeferredCompletionSlab) % CHIP_ALIGN_SIZE == 0, "DeferredCompletionSlab size must preserve array element cache-line boundaries" ); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/async_kernel_api.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/async_kernel_api.h index f415a24fc9..884a45561e 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/async_kernel_api.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/async_kernel_api.h @@ -38,7 +38,7 @@ namespace pto2::detail { inline __aicore__ void defer_load_slab(AsyncCtx &ctx) { if (ctx.completion_count == nullptr) return; #if defined(__CCE_KT_TEST__) || defined(__CCE_AICORE__) || defined(__DAV_C220__) - uintptr_t line = reinterpret_cast(ctx.completion_count) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + uintptr_t line = reinterpret_cast(ctx.completion_count) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); dcci((__gm__ int32_t *)line, SINGLE_CACHE_LINE); #else __asm__ __volatile__("" ::: "memory"); @@ -54,10 +54,10 @@ inline __aicore__ void defer_error(AsyncCtx &ctx, int32_t error_code) { inline __aicore__ void defer_flush_range(volatile __gm__ void *addr, uint32_t size_bytes) { if (addr == nullptr || size_bytes == 0) return; #if defined(__CCE_KT_TEST__) || defined(__CCE_AICORE__) || defined(__DAV_C220__) - uintptr_t start = reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + uintptr_t start = reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); uintptr_t end = - (reinterpret_cast(addr) + size_bytes + PTO2_ALIGN_SIZE - 1u) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); - for (uintptr_t p = start; p < end; p += PTO2_ALIGN_SIZE) { + (reinterpret_cast(addr) + size_bytes + CHIP_ALIGN_SIZE - 1u) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); + for (uintptr_t p = start; p < end; p += CHIP_ALIGN_SIZE) { dcci((__gm__ int32_t *)p, SINGLE_CACHE_LINE, CACHELINE_OUT); } #else diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/urma/urma_completion_scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/urma/urma_completion_scheduler.h index e1f46b07fc..19ca263493 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/urma/urma_completion_scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/urma/urma_completion_scheduler.h @@ -83,13 +83,13 @@ inline void decode_urma_event_handle(uint64_t handle, uint32_t &remote_rank, uin } inline uintptr_t cache_line(const volatile void *addr) { - return reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + return reinterpret_cast(addr) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); } inline void invalidate_object(const volatile void *addr, std::size_t size) { const uintptr_t object_addr = reinterpret_cast(addr); const uintptr_t begin = cache_line(addr); - const uintptr_t end = (object_addr + size + PTO2_ALIGN_SIZE - 1u) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); + const uintptr_t end = (object_addr + size + CHIP_ALIGN_SIZE - 1u) & ~(uintptr_t(CHIP_ALIGN_SIZE) - 1u); cache_invalidate_range(reinterpret_cast(begin), end - begin); } @@ -114,7 +114,7 @@ inline void store_device_u32(uint64_t addr, uint32_t value) { inline uint32_t load_cqe_dw0(uint64_t cqe_addr) { auto *ptr = reinterpret_cast(static_cast(cqe_addr)); - cache_invalidate_range(reinterpret_cast(cache_line(ptr)), PTO2_ALIGN_SIZE); + cache_invalidate_range(reinterpret_cast(cache_line(ptr)), CHIP_ALIGN_SIZE); return __atomic_load_n(ptr, __ATOMIC_ACQUIRE); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/constants.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/constants.h index 0707f53f94..5aaba2925f 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/constants.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/constants.h @@ -12,8 +12,8 @@ #ifndef SRC_A5_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_PTO_CONSTANTS_H_ #define SRC_A5_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_PTO_CONSTANTS_H_ -#define PTO2_ALIGN_SIZE 64 // Cache line alignment -#define PTO2_PACKED_OUTPUT_ALIGN 1024 // Each output in packed buffer aligned to 1024B; gap is padding -#define PTO2_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#define CHIP_ALIGN_SIZE 64 // Cache line alignment +#define PACKED_OUTPUT_ALIGN 1024 // Each output in packed buffer aligned to 1024B; gap is padding +#define CHIP_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) #endif // SRC_A5_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_PTO_CONSTANTS_H_ diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h index a147665ca4..1434658bb5 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h @@ -47,8 +47,8 @@ #define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) #endif -#ifndef PTO2_ALIGN_UP -#define PTO2_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#ifndef CHIP_ALIGN_UP +#define CHIP_ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) #endif // Verify hardcoded indices in intrinsic.h match the computed values. @@ -122,7 +122,7 @@ struct alignas(64) DispatchPayload { static_assert(sizeof(args[0]) == 8); static_assert( - PTO2_ALIGN_UP((MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]), 64) == + CHIP_ALIGN_UP((MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]), 64) == (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS) * sizeof(args[0]) ); }; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 9d05e79008..030c4999c7 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -544,8 +544,7 @@ static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { continue; } layout.offsets[i] = layout.total_output_size; - layout.buffer_sizes[i] = - PTO2_ALIGN_UP(args.tensor(i).create_info().buffer_size_bytes(), PTO2_PACKED_OUTPUT_ALIGN); + layout.buffer_sizes[i] = CHIP_ALIGN_UP(args.tensor(i).create_info().buffer_size_bytes(), PACKED_OUTPUT_ALIGN); layout.total_output_size += layout.buffer_sizes[i]; } return layout; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 298f39a958..9bb7271c91 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -181,7 +181,7 @@ class TaskAllocator { */ TaskAllocResult alloc(int32_t output_size, ChipTaskSlotState *oldest_open_task = nullptr) { uint64_t aligned_size = - output_size > 0 ? PTO2_ALIGN_UP(static_cast(output_size), PTO2_ALIGN_SIZE) : 0; + output_size > 0 ? CHIP_ALIGN_UP(static_cast(output_size), CHIP_ALIGN_SIZE) : 0; int spin_count = 0; int32_t prev_last_alive = last_alive_ptr_->load(std::memory_order_acquire); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 13ba25cc28..6d8033c227 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -395,7 +395,7 @@ struct TaskPayload { } // Round up to cache line boundary. Both arrays are 128B so no overrun. // Eliminates branches; extra bytes within the same CL have zero additional cost. - memcpy(scalars, args.scalars(), PTO2_ALIGN_UP(args.scalar_count() * sizeof(uint64_t), 64)); + memcpy(scalars, args.scalars(), CHIP_ALIGN_UP(args.scalar_count() * sizeof(uint64_t), 64)); // reset_for_reuse deliberately skips this cold payload. These fields // are consumer-side state and must be initialized on every submit even diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp index 0ce29309ca..0ac724d72f 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/runtime_init.cpp @@ -52,7 +52,7 @@ size_t ready_queue_reserve_layout(DeviceArena &arena, uint64_t capacity) { // Align the slots[] base to a full cache line so MPMC CAS traffic on the // first slot cannot false-share with whatever region sits in front of us // (e.g. orchestrator tensormap heads written by the orch thread). - return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), PTO2_ALIGN_SIZE); + return arena.reserve(capacity * sizeof(ChipReadyQueueSlot), CHIP_ALIGN_SIZE); } bool ready_queue_init_data_from_layout(ChipReadyQueue *queue, DeviceArena &arena, size_t slots_off, uint64_t capacity) { @@ -157,7 +157,7 @@ SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capaci // Force a cache-line base so Orch-side dep_pool writes do not invalidate // adjacent multi-threaded regions like ready_queue.slots. layout.off_dep_pool_entries[r] = - arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(DepListEntry), PTO2_ALIGN_SIZE); + arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(DepListEntry), CHIP_ALIGN_SIZE); } return layout; } @@ -343,13 +343,13 @@ OrchestratorLayout OrchestratorState::reserve_layout( for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { const size_t fanin_pool_bytes = - PTO2_ALIGN_UP(static_cast(dep_pool_capacities[r]) * sizeof(FaninSpillEntry), PTO2_ALIGN_SIZE); - layout.off_fanin_pool[r] = arena.reserve(fanin_pool_bytes, PTO2_ALIGN_SIZE); + CHIP_ALIGN_UP(static_cast(dep_pool_capacities[r]) * sizeof(FaninSpillEntry), CHIP_ALIGN_SIZE); + layout.off_fanin_pool[r] = arena.reserve(fanin_pool_bytes, CHIP_ALIGN_SIZE); always_assert(task_window_sizes[r] > 0 && (task_window_sizes[r] & (task_window_sizes[r] - 1)) == 0); const size_t seen_epoch_bytes = - PTO2_ALIGN_UP(static_cast(task_window_sizes[r]) * sizeof(uint32_t), PTO2_ALIGN_SIZE); - layout.off_fanin_seen_epoch[r] = arena.reserve(seen_epoch_bytes, PTO2_ALIGN_SIZE); + CHIP_ALIGN_UP(static_cast(task_window_sizes[r]) * sizeof(uint32_t), CHIP_ALIGN_SIZE); + layout.off_fanin_seen_epoch[r] = arena.reserve(seen_epoch_bytes, CHIP_ALIGN_SIZE); } layout.off_scope_tasks = arena.reserve(static_cast(layout.scope_tasks_cap) * sizeof(uintptr_t), alignof(ChipTaskSlotState *)); @@ -403,15 +403,15 @@ bool OrchestratorState::init_data_from_layout( ); heap_offset += heap_sizes[r]; - const size_t fanin_pool_bytes = PTO2_ALIGN_UP( - static_cast(layout.dep_pool_capacities[r]) * sizeof(FaninSpillEntry), PTO2_ALIGN_SIZE + const size_t fanin_pool_bytes = CHIP_ALIGN_UP( + static_cast(layout.dep_pool_capacities[r]) * sizeof(FaninSpillEntry), CHIP_ALIGN_SIZE ); auto *fanin_entries = static_cast(arena.region_ptr(layout.off_fanin_pool[r])); memset(fanin_entries, 0, fanin_pool_bytes); orch->rings[r].fanin_pool.init(fanin_entries, layout.dep_pool_capacities[r], orch_err); - const size_t seen_epoch_bytes = PTO2_ALIGN_UP( - static_cast(layout.tensor_map.task_window_sizes[r]) * sizeof(uint32_t), PTO2_ALIGN_SIZE + const size_t seen_epoch_bytes = CHIP_ALIGN_UP( + static_cast(layout.tensor_map.task_window_sizes[r]) * sizeof(uint32_t), CHIP_ALIGN_SIZE ); auto *seen_epoch = static_cast(arena.region_ptr(layout.off_fanin_seen_epoch[r])); memset(seen_epoch, 0, seen_epoch_bytes); @@ -569,7 +569,7 @@ RuntimeArenaLayout runtime_reserve_layout( } layout.offsets.orch = OrchestratorState::reserve_layout(arena, task_window_sizes_i32, dep_pool_capacities); layout.offsets.sched = SchedulerState::reserve_layout(arena, dep_pool_capacities); - layout.offsets.off_runtime = arena.reserve(sizeof(RuntimeContext), PTO2_ALIGN_SIZE); + layout.offsets.off_runtime = arena.reserve(sizeof(RuntimeContext), CHIP_ALIGN_SIZE); layout.offsets.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); layout.offsets.arena_size = arena.total_size(); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index b1e17c0a43..d1d01d984b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -97,7 +97,7 @@ bool SharedMemoryHandle::init_per_ring( SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); - const size_t off_buffer = arena.reserve(static_cast(buffer_size), PTO2_ALIGN_SIZE); + const size_t off_buffer = arena.reserve(static_cast(buffer_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; auto *handle = static_cast(arena.region_ptr(off_handle)); @@ -143,15 +143,15 @@ void SharedMemoryHandle::init_header_per_ring( header->orchestrator_done.store(0, std::memory_order_relaxed); // Per-ring layout info - uint64_t offset = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t offset = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { header->rings[r].task_window_size = task_window_sizes[r]; header->rings[r].task_window_mask = static_cast(task_window_sizes[r] - 1); header->rings[r].heap_size = heap_sizes[r]; header->rings[r].task_descriptors_offset = offset; - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); - offset += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), CHIP_ALIGN_SIZE); + offset += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); } header->total_size = sm_size; @@ -234,7 +234,7 @@ bool ChipRingFlowControl::validate(SharedMemoryHandle *handle, int32_t ring_id) if (h->rings[ring_id].task_descriptors_offset >= h->total_size) return false; // Check pointer alignment - if ((uintptr_t)h->rings[ring_id].task_descriptors % PTO2_ALIGN_SIZE != 0) return false; + if ((uintptr_t)h->rings[ring_id].task_descriptors % CHIP_ALIGN_SIZE != 0) return false; // Check flow control pointer sanity int32_t current = current_task_index.load(std::memory_order_acquire); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 8a8692acf7..4cb9d8387c 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -125,7 +125,7 @@ static_assert( * * Contains per-ring flow control and global layout information. */ -struct alignas(PTO2_ALIGN_SIZE) SharedMemoryHeader { +struct alignas(CHIP_ALIGN_SIZE) SharedMemoryHeader { // === PER-RING FLOW CONTROL + LAYOUT INFO (set once at init) === SharedMemoryRingHeader rings[CHIP_MAX_RING_DEPTH]; @@ -268,7 +268,7 @@ inline std::atomic *ring_last_task_alive_addr(void *sm_dev_base, int ri // Byte offsets (from the SM base) of one ring's three segments. The per-ring // layout is: header, then for each ring descriptors -> payloads -> slot_states, -// every segment PTO2_ALIGN_UP-padded. +// every segment CHIP_ALIGN_UP-padded. struct ChipRingSegmentOffsets { uint64_t descriptors; uint64_t payloads; @@ -285,19 +285,19 @@ struct ChipRingSegmentOffsets { inline ChipRingSegmentOffsets ring_segment_offsets(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH], int ring_id) noexcept { assert(ring_id >= 0 && ring_id < CHIP_MAX_RING_DEPTH && "sm_layout: ring_id out of range"); - uint64_t off = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t off = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); for (int r = 0; r < ring_id; r++) { - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); - off += PTO2_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(TaskPayload), CHIP_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[r] * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); } ChipRingSegmentOffsets o{}; o.descriptors = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskDescriptor), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskDescriptor), CHIP_ALIGN_SIZE); o.payloads = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskPayload), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[ring_id] * sizeof(TaskPayload), CHIP_ALIGN_SIZE); o.slot_states = off; - off += PTO2_ALIGN_UP(task_window_sizes[ring_id] * sizeof(ChipTaskSlotState), PTO2_ALIGN_SIZE); + off += CHIP_ALIGN_UP(task_window_sizes[ring_id] * sizeof(ChipTaskSlotState), CHIP_ALIGN_SIZE); o.end = off; return o; } diff --git a/src/common/hierarchical/ring.h b/src/common/hierarchical/ring.h index a36fe3db60..8105fe854e 100644 --- a/src/common/hierarchical/ring.h +++ b/src/common/hierarchical/ring.h @@ -62,7 +62,7 @@ #include "types.h" -// User-facing output alignment (Strict-3; matches L2 PTO2_PACKED_OUTPUT_ALIGN). +// User-facing output alignment (Strict-3; matches L2 PACKED_OUTPUT_ALIGN). static constexpr uint64_t HEAP_ALIGN = 1024; // Default PER-RING heap size. Total VA reservation is this value times diff --git a/src/common/host_build_graph/graph_execution.cpp b/src/common/host_build_graph/graph_execution.cpp index 398c3a414f..edf4b0777d 100644 --- a/src/common/host_build_graph/graph_execution.cpp +++ b/src/common/host_build_graph/graph_execution.cpp @@ -113,7 +113,7 @@ bool bind_graph_topology(GraphExecution &execution) { const bool active = (node.active_mask & (1U << slot)) != 0; if (active != (node.kernel_id[slot] != INVALID_KERNEL_ID)) return false; } - const uint64_t output_bytes = PTO2_ALIGN_UP(static_cast(node.total_output_size), PTO2_ALIGN_SIZE); + const uint64_t output_bytes = CHIP_ALIGN_UP(static_cast(node.total_output_size), CHIP_ALIGN_SIZE); if (output_bytes > definition.required_heap - required_heap) return false; required_heap += output_bytes; } @@ -443,7 +443,7 @@ GraphMaterializeResult graph_execution_materialize_slice( task.task_id = TaskId::make(1, synthetic_local); const GraphNodeDefinition &source = nodes[i]; const uint64_t node_offset = node_offsets[i]; - const uint64_t output_bytes = PTO2_ALIGN_UP(static_cast(source.total_output_size), PTO2_ALIGN_SIZE); + const uint64_t output_bytes = CHIP_ALIGN_UP(static_cast(source.total_output_size), CHIP_ALIGN_SIZE); for (int k = 0; k < SUBTASK_SLOT_COUNT; ++k) task.kernel_id[k] = source.kernel_id[k]; task.packed_buffer_base = reinterpret_cast(outer_base + node_offset); diff --git a/src/common/utils/device_arena.h b/src/common/utils/device_arena.h index c2cc8436be..119c48a4b5 100644 --- a/src/common/utils/device_arena.h +++ b/src/common/utils/device_arena.h @@ -44,7 +44,7 @@ class DeviceArena { static constexpr size_t kMaxRegions = 128; // Default base alignment for the underlying allocation. 1024 bytes is a - // hardware requirement (HBM DMA granularity / PTO2_PACKED_OUTPUT_ALIGN), + // hardware requirement (HBM DMA granularity / PACKED_OUTPUT_ALIGN), // not a software-side optimization — do NOT lower it to the 64-byte cache // line that satisfies every sub-region alignment requirement. The cost is // a one-time forward-alignment pad of up to 1023 bytes per arena, which is diff --git a/tests/ut/cpp/a2a3/test_aicore_completion_mailbox.cpp b/tests/ut/cpp/a2a3/test_aicore_completion_mailbox.cpp index 1bb140dfe7..838999769d 100644 --- a/tests/ut/cpp/a2a3/test_aicore_completion_mailbox.cpp +++ b/tests/ut/cpp/a2a3/test_aicore_completion_mailbox.cpp @@ -182,7 +182,7 @@ TEST(AICoreCompletionMailbox, ConditionAttachesToExistingEntry) { } TEST(AICoreCompletionMailbox, SdmaPostDoneRecordUsesMonotonicPostId) { - alignas(PTO2_ALIGN_SIZE) uint64_t completed_post_id = (uint64_t{1} << 40) + 16; + alignas(CHIP_ALIGN_SIZE) uint64_t completed_post_id = (uint64_t{1} << 40) + 16; CompletionCondition cond{}; cond.completion_type = COMPLETION_TYPE_SDMA_EVENT_RECORD; cond.addr = reinterpret_cast(&completed_post_id); diff --git a/tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp b/tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp index 7b9fd2176d..f7e2fa2daa 100644 --- a/tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_task_allocator.cpp @@ -75,7 +75,7 @@ TEST_F(HbgTaskAllocatorTest, AllocNonZeroSize) { EXPECT_EQ(result.task_id, 0); EXPECT_EQ(result.slot, 0); EXPECT_NE(result.packed_base, nullptr); - uint64_t expected_aligned = PTO2_ALIGN_UP(100u, PTO2_ALIGN_SIZE); + uint64_t expected_aligned = CHIP_ALIGN_UP(100u, CHIP_ALIGN_SIZE); EXPECT_EQ(expected_aligned, 128u); EXPECT_EQ(allocator.heap_top(), expected_aligned); EXPECT_EQ( diff --git a/tests/ut/cpp/a2a3/test_shared_memory.cpp b/tests/ut/cpp/a2a3/test_shared_memory.cpp index 0456212146..f94a35f03d 100644 --- a/tests/ut/cpp/a2a3/test_shared_memory.cpp +++ b/tests/ut/cpp/a2a3/test_shared_memory.cpp @@ -32,7 +32,7 @@ namespace { SharedMemoryHandle *make_handle(DeviceArena &arena, uint64_t task_window_size, uint64_t heap_size) { const uint64_t sm_size = SharedMemoryHandle::calculate_size(task_window_size); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); - const size_t off_buffer = arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); + const size_t off_buffer = arena.reserve(static_cast(sm_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; auto *handle = static_cast(arena.region_ptr(off_handle)); @@ -153,13 +153,13 @@ TEST_F(SharedMemoryTest, PerRingIndependence) { TEST_F(SharedMemoryTest, PointerAlignment) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { auto addr = reinterpret_cast(handle->header->rings[r].task_descriptors); - EXPECT_EQ(addr % PTO2_ALIGN_SIZE, 0u) << "Ring " << r << " descriptors not aligned"; + EXPECT_EQ(addr % CHIP_ALIGN_SIZE, 0u) << "Ring " << r << " descriptors not aligned"; } } TEST_F(SharedMemoryTest, HeaderAlignment) { uintptr_t header_addr = (uintptr_t)handle->header; - EXPECT_EQ(header_addr % PTO2_ALIGN_SIZE, 0u) << "Header must be cache-line aligned"; + EXPECT_EQ(header_addr % CHIP_ALIGN_SIZE, 0u) << "Header must be cache-line aligned"; } // Descriptor and payload regions don't overlap within or across rings. @@ -198,7 +198,7 @@ TEST(SharedMemoryCalcSize, LargerWindowGivesLargerSize) { EXPECT_GT(large_size, small_size); } -TEST(SharedMemoryCalcSize, HeaderAligned) { EXPECT_EQ(sizeof(SharedMemoryHeader) % PTO2_ALIGN_SIZE, 0u); } +TEST(SharedMemoryCalcSize, HeaderAligned) { EXPECT_EQ(sizeof(SharedMemoryHeader) % CHIP_ALIGN_SIZE, 0u); } TEST(SharedMemoryCalcSize, PerRingDifferentSizes) { uint64_t ws[CHIP_MAX_RING_DEPTH] = {128, 256, 512, 1024}; @@ -215,7 +215,7 @@ TEST(SharedMemoryLayout, InitPerRingWritesHeaderValues) { DeviceArena arena; const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); - const size_t off_buffer = arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); + const size_t off_buffer = arena.reserve(static_cast(sm_size), CHIP_ALIGN_SIZE); ASSERT_NE(arena.commit(), nullptr); auto *handle = static_cast(arena.region_ptr(off_handle)); @@ -246,7 +246,7 @@ TEST(RuntimeArenaLayout, PerRingConfigInitializesRuntimeComponents) { ASSERT_NE(runtime_arena.commit(DeviceArena::kDefaultBaseAlign), nullptr); DeviceArena sm_arena; - const size_t sm_off = sm_arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); + const size_t sm_off = sm_arena.reserve(static_cast(sm_size), CHIP_ALIGN_SIZE); ASSERT_NE(sm_arena.commit(), nullptr); void *sm = sm_arena.region_ptr(sm_off); std::memset(sm, 0, static_cast(sm_size)); @@ -293,7 +293,7 @@ TEST(RuntimeArenaLayout, RejectsOverflowingPerRingHeapSum) { // Zero window size: all ring descriptor pointers collapse to the same address. TEST(SharedMemoryBoundary, ZeroWindowSize) { uint64_t size = SharedMemoryHandle::calculate_size(0); - uint64_t header_size = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t header_size = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); EXPECT_EQ(size, header_size); DeviceArena arena; diff --git a/tests/ut/cpp/a2a3/test_task_allocator.cpp b/tests/ut/cpp/a2a3/test_task_allocator.cpp index 7cd9622975..81802ad68e 100644 --- a/tests/ut/cpp/a2a3/test_task_allocator.cpp +++ b/tests/ut/cpp/a2a3/test_task_allocator.cpp @@ -109,8 +109,8 @@ TEST_F(TaskAllocatorTest, AllocNonZeroSize) { EXPECT_EQ(result.task_id, 0); EXPECT_EQ(result.slot, 0); EXPECT_NE(result.packed_base, nullptr); - // 100 bytes aligned up to PTO2_ALIGN_SIZE (64) = 128 - uint64_t expected_aligned = PTO2_ALIGN_UP(100u, PTO2_ALIGN_SIZE); + // 100 bytes aligned up to CHIP_ALIGN_SIZE (64) = 128 + uint64_t expected_aligned = CHIP_ALIGN_UP(100u, CHIP_ALIGN_SIZE); EXPECT_EQ(expected_aligned, 128u); EXPECT_EQ(allocator.heap_top(), expected_aligned); EXPECT_EQ( diff --git a/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp b/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp index 9ac3cdeeff..7d848e21c6 100644 --- a/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp +++ b/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp @@ -161,7 +161,7 @@ TEST(A5AICoreCompletionMailbox, ConditionAttachesToExistingEntry) { } TEST(A5AICoreCompletionMailbox, SdmaPostDoneRecordUsesMonotonicPostId) { - alignas(PTO2_ALIGN_SIZE) uint64_t completed_post_id = (uint64_t{1} << 40) + 16; + alignas(CHIP_ALIGN_SIZE) uint64_t completed_post_id = (uint64_t{1} << 40) + 16; CompletionCondition cond{}; cond.completion_type = COMPLETION_TYPE_SDMA_EVENT_RECORD; cond.addr = reinterpret_cast(&completed_post_id); diff --git a/tests/ut/cpp/a5/test_shared_memory.cpp b/tests/ut/cpp/a5/test_shared_memory.cpp index 7b7696ee7f..7692aca926 100644 --- a/tests/ut/cpp/a5/test_shared_memory.cpp +++ b/tests/ut/cpp/a5/test_shared_memory.cpp @@ -32,7 +32,7 @@ namespace { SharedMemoryHandle *make_handle(DeviceArena &arena, uint64_t task_window_size, uint64_t heap_size) { const uint64_t sm_size = SharedMemoryHandle::calculate_size(task_window_size); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); - const size_t off_buffer = arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); + const size_t off_buffer = arena.reserve(static_cast(sm_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; auto *handle = static_cast(arena.region_ptr(off_handle)); @@ -153,13 +153,13 @@ TEST_F(SharedMemoryTest, PerRingIndependence) { TEST_F(SharedMemoryTest, PointerAlignment) { for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { auto addr = reinterpret_cast(handle->header->rings[r].task_descriptors); - EXPECT_EQ(addr % PTO2_ALIGN_SIZE, 0u) << "Ring " << r << " descriptors not aligned"; + EXPECT_EQ(addr % CHIP_ALIGN_SIZE, 0u) << "Ring " << r << " descriptors not aligned"; } } TEST_F(SharedMemoryTest, HeaderAlignment) { uintptr_t header_addr = (uintptr_t)handle->header; - EXPECT_EQ(header_addr % PTO2_ALIGN_SIZE, 0u) << "Header must be cache-line aligned"; + EXPECT_EQ(header_addr % CHIP_ALIGN_SIZE, 0u) << "Header must be cache-line aligned"; } // Descriptor and payload regions don't overlap within or across rings. @@ -198,7 +198,7 @@ TEST(SharedMemoryCalcSize, LargerWindowGivesLargerSize) { EXPECT_GT(large_size, small_size); } -TEST(SharedMemoryCalcSize, HeaderAligned) { EXPECT_EQ(sizeof(SharedMemoryHeader) % PTO2_ALIGN_SIZE, 0u); } +TEST(SharedMemoryCalcSize, HeaderAligned) { EXPECT_EQ(sizeof(SharedMemoryHeader) % CHIP_ALIGN_SIZE, 0u); } TEST(SharedMemoryCalcSize, PerRingDifferentSizes) { uint64_t ws[CHIP_MAX_RING_DEPTH] = {128, 256, 512, 1024}; @@ -215,7 +215,7 @@ TEST(SharedMemoryLayout, InitPerRingWritesHeaderValues) { DeviceArena arena; const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); - const size_t off_buffer = arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); + const size_t off_buffer = arena.reserve(static_cast(sm_size), CHIP_ALIGN_SIZE); ASSERT_NE(arena.commit(), nullptr); auto *handle = static_cast(arena.region_ptr(off_handle)); @@ -246,7 +246,7 @@ TEST(RuntimeArenaLayout, PerRingConfigInitializesRuntimeComponents) { ASSERT_NE(runtime_arena.commit(DeviceArena::kDefaultBaseAlign), nullptr); DeviceArena sm_arena; - const size_t sm_off = sm_arena.reserve(static_cast(sm_size), PTO2_ALIGN_SIZE); + const size_t sm_off = sm_arena.reserve(static_cast(sm_size), CHIP_ALIGN_SIZE); ASSERT_NE(sm_arena.commit(), nullptr); void *sm = sm_arena.region_ptr(sm_off); std::memset(sm, 0, static_cast(sm_size)); @@ -340,7 +340,7 @@ TEST(RuntimeArenaLayout, RejectsOverflowingPerRingHeapSum) { // Zero window size: all ring descriptor pointers collapse to the same address. TEST(SharedMemoryBoundary, ZeroWindowSize) { uint64_t size = SharedMemoryHandle::calculate_size(0); - uint64_t header_size = PTO2_ALIGN_UP(sizeof(SharedMemoryHeader), PTO2_ALIGN_SIZE); + uint64_t header_size = CHIP_ALIGN_UP(sizeof(SharedMemoryHeader), CHIP_ALIGN_SIZE); EXPECT_EQ(size, header_size); DeviceArena arena; diff --git a/tests/ut/cpp/a5/test_task_allocator.cpp b/tests/ut/cpp/a5/test_task_allocator.cpp index 7cd9622975..81802ad68e 100644 --- a/tests/ut/cpp/a5/test_task_allocator.cpp +++ b/tests/ut/cpp/a5/test_task_allocator.cpp @@ -109,8 +109,8 @@ TEST_F(TaskAllocatorTest, AllocNonZeroSize) { EXPECT_EQ(result.task_id, 0); EXPECT_EQ(result.slot, 0); EXPECT_NE(result.packed_base, nullptr); - // 100 bytes aligned up to PTO2_ALIGN_SIZE (64) = 128 - uint64_t expected_aligned = PTO2_ALIGN_UP(100u, PTO2_ALIGN_SIZE); + // 100 bytes aligned up to CHIP_ALIGN_SIZE (64) = 128 + uint64_t expected_aligned = CHIP_ALIGN_UP(100u, CHIP_ALIGN_SIZE); EXPECT_EQ(expected_aligned, 128u); EXPECT_EQ(allocator.heap_top(), expected_aligned); EXPECT_EQ( diff --git a/tests/ut/cpp/common/test_hbg_graph_cache.cpp b/tests/ut/cpp/common/test_hbg_graph_cache.cpp index 9e107a9232..7cdaf20e52 100644 --- a/tests/ut/cpp/common/test_hbg_graph_cache.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_cache.cpp @@ -32,7 +32,7 @@ namespace { template uint32_t append_section(std::vector &image, const std::vector &values) { if (values.empty()) return 0; - const size_t offset = PTO2_ALIGN_UP(image.size(), alignof(T)); + const size_t offset = CHIP_ALIGN_UP(image.size(), alignof(T)); image.resize(offset + values.size() * sizeof(T)); std::memcpy(image.data() + offset, values.data(), values.size() * sizeof(T)); return static_cast(offset); @@ -201,7 +201,7 @@ class OuterHeap { public: static constexpr size_t TENSOR_SLOTS = graph_boundary_tensor_pool_slots(GRAPH_MAX_TENSOR_ARGS); static constexpr size_t SCALAR_SPAN = - PTO2_ALIGN_UP(static_cast(GRAPH_MAX_SCALAR_ARGS), ARG_POOL_ALIGN / sizeof(uint64_t)); + CHIP_ALIGN_UP(static_cast(GRAPH_MAX_SCALAR_ARGS), ARG_POOL_ALIGN / sizeof(uint64_t)); OuterHeap(const std::vector &definition_image, uint8_t fill = 0) { const auto *definition = reinterpret_cast(definition_image.data()); @@ -403,7 +403,7 @@ TEST(GraphBoundaryPool, WidestBoundaryExceedsOneSlotBudget) { EXPECT_GT(graph_boundary_tensor_pool_slots(GRAPH_MAX_TENSOR_ARGS), static_cast(MAX_TENSOR_ARGS)); EXPECT_GT( static_cast( - PTO2_ALIGN_UP(static_cast(GRAPH_MAX_SCALAR_ARGS), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)) + CHIP_ALIGN_UP(static_cast(GRAPH_MAX_SCALAR_ARGS), ARG_POOL_ALIGN / (int32_t)sizeof(uint64_t)) ), static_cast(MAX_SCALAR_ARGS) ); diff --git a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp index ebb51c991f..e871ad9006 100644 --- a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp @@ -141,7 +141,7 @@ TEST_F(HbgGraphSubmitFailureTest, InFlightGraphInvocationsReserveHeapOnlyAtCommi ASSERT_EQ(definitions.entries[0].full_key, first_upload->full_key); const GraphDefinition *definition = definition_image(definitions.entries[0]); const uint64_t expected_extent = - PTO2_ALIGN_UP(definition->required_heap + definition->execution_storage_bytes, PTO2_ALIGN_SIZE); + CHIP_ALIGN_UP(definition->required_heap + definition->execution_storage_bytes, CHIP_ALIGN_SIZE); EXPECT_EQ(static_cast(first_end - first_base), expected_extent); EXPECT_EQ(static_cast(second_end - second_base), expected_extent); EXPECT_TRUE(first_end <= second_base || second_end <= first_base) << "two shells must not share heap bytes"; @@ -250,7 +250,7 @@ TEST_F(HbgGraphSubmitFailureTest, WorkerRecordsWhileMainThreadSubmitsSameHashShe ASSERT_EQ(definitions.entries.size(), 1u); const GraphDefinition *definition = definition_image(definitions.entries[0]); const uint64_t expected_extent = - PTO2_ALIGN_UP(definition->required_heap + definition->execution_storage_bytes, PTO2_ALIGN_SIZE); + CHIP_ALIGN_UP(definition->required_heap + definition->execution_storage_bytes, CHIP_ALIGN_SIZE); std::vector> ranges; for (size_t i = 0; i < 3; ++i) { diff --git a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp index d9152017f0..e1a98989e0 100644 --- a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp +++ b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp @@ -43,14 +43,14 @@ constexpr int32_t FANIN_STRIDE = static_cast(ARG_POOL_ALIGN / sizeof(in // A buffer aligned the way both the arena mirror and the device SM base are; // ChipTaskSlotState is alignas(64) and every segment offset is a multiple of -// PTO2_ALIGN_SIZE. +// CHIP_ALIGN_SIZE. class AlignedImage { public: explicit AlignedImage(uint64_t bytes, uint8_t fill = 0) : - storage_(bytes + PTO2_ALIGN_SIZE, std::byte{0}) { + storage_(bytes + CHIP_ALIGN_SIZE, std::byte{0}) { base_ = reinterpret_cast( - (reinterpret_cast(storage_.data()) + PTO2_ALIGN_SIZE - 1) & - ~static_cast(PTO2_ALIGN_SIZE - 1) + (reinterpret_cast(storage_.data()) + CHIP_ALIGN_SIZE - 1) & + ~static_cast(CHIP_ALIGN_SIZE - 1) ); if (fill != 0) std::memset(base_, fill, bytes); } From d7c67d849e03ce78adb5376a1e85a65f83dfc483 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:50:29 -0700 Subject: [PATCH 17/22] Refactor: retire the PTO2 prefix on the remaining runtime constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last of the identifier work: the waits `TENSOR_DATA_TIMEOUT_CYCLES` / `TENSOR_DATA_TIMEOUT_MS`, the caps `DEFERRED_RELEASE_CAP` and `EXT_PARAMS_COUNT`, the three `MODE_*` runtime modes, the types `LaunchSpec` and `OutputLayout`, the `ORCHESTRATION_CONFIG_DEFINED` include guard, and the file-local `read_runtime_status` helper. 239 occurrences across 83 files. Four keep a prefix because they are object-like macros rather than `constexpr`: `CHIP_TASK_WINDOW_SIZE`, `CHIP_HEAP_SIZE`, `CHIP_BLOCK_NOTIFY_INTERVAL` and `CHIP_PUBLICATION_REQUEST_TIMEOUT_CYCLES`. `HEAP_SIZE` is the one that proves the rule rather than merely satisfying it: three cpput fixtures declare `static constexpr uint64_t HEAP_SIZE = 4096;`, and `#define HEAP_SIZE (256 * 1024 * 1024)` rewrites that declaration into `static constexpr uint64_t (256 * 1024 * 1024) = 4096;`. The runtime build never sees it — the C++ unit tests are a separate CMake project — so the gate for a rename has to compile both. `read_runtime_status` is `static` in all four `host/runtime_maker.cpp` copies, so the prefix bought it nothing that internal linkage did not already provide. Co-Authored-By: Claude Opus 5 (1M context) --- .../device-error-codes/stall.md | 2 +- .../paged_attention_ringbuffer/README.md | 2 +- .../paged_attention_ringbuffer/README.md | 2 +- .../qwen3_14b_decode/README.md | 2 +- .../host_build_graph/common/intrinsic.h | 2 +- .../host_build_graph/host/runtime_maker.cpp | 10 +++++----- .../orchestration/orchestration_api.h | 4 ++-- .../runtime/dispatch_payload.h | 2 +- .../runtime/orchestrator_core/orchestrator.cpp | 10 +++++----- .../runtime/orchestrator_core/runtime_core.cpp | 13 ++++++------- .../host_build_graph/runtime/runtime_core.h | 10 +++++----- .../host_build_graph/runtime/runtime_types.h | 10 +++++----- .../runtime/scheduler/scheduler_cold_path.cpp | 4 ++-- .../runtime/shared/shared_memory.cpp | 4 ++-- .../host_build_graph/runtime/shared_memory.h | 2 +- .../host_build_graph/runtime/submit_types.h | 4 ++-- .../runtime/host_build_graph/runtime/types.h | 2 +- .../common/intrinsic.h | 2 +- .../docs/MULTI_RING.md | 4 ++-- .../docs/SCALAR_DATA_ACCESS.md | 2 +- .../host/runtime_maker.cpp | 10 +++++----- .../orchestration/orchestration_api.h | 4 ++-- .../runtime/dispatch_payload.h | 2 +- .../runtime/orchestrator.cpp | 10 +++++----- .../runtime/ring_buffer.h | 4 ++-- .../runtime/runtime_core.cpp | 11 +++++------ .../runtime/runtime_core.h | 10 +++++----- .../runtime/runtime_types.h | 18 +++++++++--------- .../runtime/scheduler/scheduler_completion.cpp | 4 ++-- .../runtime/scheduler/scheduler_dispatch.cpp | 9 ++++----- .../runtime/shared/shared_memory.cpp | 4 ++-- .../runtime/shared_memory.h | 2 +- .../runtime/submit_types.h | 4 ++-- .../tensormap_and_ringbuffer/runtime/types.h | 2 +- .../host_build_graph/common/intrinsic.h | 2 +- .../host_build_graph/host/runtime_maker.cpp | 10 +++++----- .../orchestration/orchestration_api.h | 4 ++-- .../runtime/dispatch_payload.h | 2 +- .../runtime/orchestrator_core/orchestrator.cpp | 10 +++++----- .../runtime/orchestrator_core/runtime_core.cpp | 13 ++++++------- .../host_build_graph/runtime/runtime_core.h | 10 +++++----- .../host_build_graph/runtime/runtime_types.h | 10 +++++----- .../runtime/scheduler/scheduler_cold_path.cpp | 4 ++-- .../runtime/shared/shared_memory.cpp | 4 ++-- .../host_build_graph/runtime/shared_memory.h | 2 +- .../host_build_graph/runtime/submit_types.h | 4 ++-- .../runtime/host_build_graph/runtime/types.h | 2 +- .../common/intrinsic.h | 2 +- .../docs/MULTI_RING.md | 4 ++-- .../docs/SCALAR_DATA_ACCESS.md | 2 +- .../host/runtime_maker.cpp | 10 +++++----- .../orchestration/orchestration_api.h | 4 ++-- .../runtime/dispatch_payload.h | 2 +- .../runtime/orchestrator.cpp | 12 ++++++------ .../runtime/ring_buffer.cpp | 4 ++-- .../runtime/ring_buffer.h | 8 ++++---- .../runtime/runtime_core.cpp | 11 +++++------ .../runtime/runtime_core.h | 10 +++++----- .../runtime/runtime_types.h | 18 +++++++++--------- .../runtime/scheduler/scheduler_completion.cpp | 4 ++-- .../runtime/scheduler/scheduler_dispatch.cpp | 9 ++++----- .../runtime/shared/shared_memory.cpp | 4 ++-- .../runtime/shared_memory.h | 2 +- .../runtime/submit_types.h | 4 ++-- .../tensormap_and_ringbuffer/runtime/types.h | 2 +- src/common/runtime_status/error_names.h | 2 +- .../available_aicore_counts_orch.cpp | 4 ++-- .../available_aicore_counts_orch.cpp | 4 ++-- .../available_aicore_counts_orch.cpp | 4 ++-- .../orchestration/require_sync_start_orch.cpp | 2 +- .../orchestration/tensor_wait_timeout_orch.cpp | 2 +- .../test_runtime_fatal_codes.py | 2 +- tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp | 2 +- tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp | 10 +++++----- tests/ut/cpp/a2a3/test_scheduler_state.cpp | 2 +- tests/ut/cpp/a2a3/test_shared_memory.cpp | 6 +++--- tests/ut/cpp/a5/test_hbg_submit_poison.cpp | 2 +- tests/ut/cpp/a5/test_orchestrator_fanin.cpp | 10 +++++----- tests/ut/cpp/a5/test_scheduler_state.cpp | 2 +- tests/ut/cpp/a5/test_shared_memory.cpp | 8 ++++---- .../common/test_hbg_graph_definition_arena.cpp | 2 +- .../common/test_hbg_graph_submit_failure.cpp | 4 ++-- tests/ut/cpp/common/test_hbg_slot_claim.cpp | 2 +- tests/ut/cpp/common/test_hbg_sm_compaction.cpp | 6 +++--- 84 files changed, 223 insertions(+), 229 deletions(-) diff --git a/docs/troubleshooting/device-error-codes/stall.md b/docs/troubleshooting/device-error-codes/stall.md index 92e0dccd17..20eb9a50e6 100644 --- a/docs/troubleshooting/device-error-codes/stall.md +++ b/docs/troubleshooting/device-error-codes/stall.md @@ -62,7 +62,7 @@ exist) before reading. See the "Device logs" section of ## Code 8, specifically -The tensor-data wait defaults to 15 s (`PTO2_TENSOR_DATA_TIMEOUT_MS`, +The tensor-data wait defaults to 15 s (`TENSOR_DATA_TIMEOUT_MS`, frequency-scaled). It means either the producer never completed, or a consumer never released its fanout reference. Check for a hung producer first (that is S1 above), then verify the consumer really declares the dependency and exits. If the kernel is diff --git a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md index f52bafbb2d..d16cedf0d1 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md +++ b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md @@ -18,7 +18,7 @@ One case, `ringbuffer_stress`, sized to hurt: | `runtime_env` key | Value here | Note | | ----------------- | ---------- | ---- | -| `ring_task_window` | 64 | Default is 16384 (`PTO2_TASK_WINDOW_SIZE`, `src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h`) — **256× smaller** | +| `ring_task_window` | 64 | Default is 16384 (`CHIP_TASK_WINDOW_SIZE`, `src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h`) — **256× smaller** | | `ring_heap` | 4 MiB | bytes per ring | | `ring_dep_pool` | 256 | | diff --git a/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md b/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md index c92707d9d6..8cce365ba1 100644 --- a/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md +++ b/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md @@ -18,7 +18,7 @@ One case, `ringbuffer_stress`, sized to hurt: | `runtime_env` key | Value here | Note | | ----------------- | ---------- | ---- | -| `ring_task_window` | 64 | Default is 16384 (`PTO2_TASK_WINDOW_SIZE`, `src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h`) — **256× smaller** | +| `ring_task_window` | 64 | Default is 16384 (`CHIP_TASK_WINDOW_SIZE`, `src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h`) — **256× smaller** | | `ring_heap` | 4 MiB | bytes per ring | | `ring_dep_pool` | 256 | | diff --git a/examples/a5/tensormap_and_ringbuffer/qwen3_14b_decode/README.md b/examples/a5/tensormap_and_ringbuffer/qwen3_14b_decode/README.md index 99a1d5ede2..64a58c8874 100644 --- a/examples/a5/tensormap_and_ringbuffer/qwen3_14b_decode/README.md +++ b/examples/a5/tensormap_and_ringbuffer/qwen3_14b_decode/README.md @@ -227,7 +227,7 @@ so a refresh is still "re-harvest a2a3, then re-apply this table". | Delta | Where | Why a5 needs it | | ----- | ----- | --------------- | -| `set_block_num` → `set_core_num` (×18) | orchestration | a5 tmr `PTO2LaunchSpec` names the accessor `set_core_num` (hbg still has `set_block_num`) | +| `set_block_num` → `set_core_num` (×18) | orchestration | a5 tmr `LaunchSpec` names the accessor `set_core_num` (hbg still has `set_block_num`) | | Left(A) tile `BLayout::RowMajor` → `ColMajor` (×288) | AIC kernels | a5 cube takes L0A in the ColMajor fractal: pto-isa's own `TileLeft` alias is `BLayout::ColMajor` off a2a3, and `CheckMadValid` asserts `!TileLeft::isRowMajor`. SFractal stays `RowMajor`, and the L1 `Mat` tiles the `TEXTRACT` reads from are unchanged | | drop `pipe_barrier(PIPE_V)` (×135), `pipe_barrier(PIPE_MTE1)` (×32), `pipe_barrier(PIPE_FIX)` (×3) | AIV + AIC kernels | a5 `pipe_barrier` accepts only `MTE3` / `M` / `ALL`; anything else is a compile error. Nothing is lost: a5's own `pto::Event` emits no barrier for a same-pipe V event either, and every cross-pipe order here is already carried by a `set_flag` / `wait_flag` pair | | `__DAV_C220_{VEC,CUBE}__` → `__DAV_{VEC,CUBE}__` (3 files) | attention extern | `__DAV_C220_*` is the a2a3 (`dav-c220`) ccec predefine. a5 builds as `dav-c310`, so on a5 those guards silently compiled out the whole RoPE/QK-norm/KV-write phase **and** the vendor FAI's cube and vector bodies — the task ran to completion writing nothing. `__DAV_VEC__` / `__DAV_CUBE__` are predefined on both arches and are what every other kernel here already uses | diff --git a/src/a2a3/runtime/host_build_graph/common/intrinsic.h b/src/a2a3/runtime/host_build_graph/common/intrinsic.h index 8dcd4ca5fd..a9184f4e5f 100644 --- a/src/a2a3/runtime/host_build_graph/common/intrinsic.h +++ b/src/a2a3/runtime/host_build_graph/common/intrinsic.h @@ -93,7 +93,7 @@ #endif /** Number of extra pointer slots appended to the args[] tail (LocalContext + GlobalContext). */ -static constexpr int32_t PTO2_EXT_PARAMS_COUNT = 2; +static constexpr int32_t EXT_PARAMS_COUNT = 2; /** * Args[] suffix indices for context pointers. diff --git a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp index e48e4431f4..a323ecb722 100644 --- a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp @@ -277,7 +277,7 @@ static uint64_t read_ring_override(const uint64_t *base, int idx) { // reclaiming runtime's knobs; this function does not resolve them, and the caller // reads them only to warn that they reach nothing here. static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t *eff_task_window_size) { - *eff_task_window_size = PTO2_TASK_WINDOW_SIZE; + *eff_task_window_size = CHIP_TASK_WINDOW_SIZE; apply_env_ring_value("PTO2_RING_TASK_WINDOW", 4, static_cast(INT32_MAX), true, eff_task_window_size); @@ -307,7 +307,7 @@ static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t return true; } -static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { +static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { if (runtime == nullptr || api == nullptr || host_header == nullptr) { return 0; } @@ -625,7 +625,7 @@ int32_t run_host_orchestration( runtime_bind_ops(rt); orchestrator.total_cluster_count = block_dim * PLATFORM_AIC_CORES_PER_BLOCKDIM; orchestrator.total_aiv_count = block_dim * PLATFORM_AIV_CORES_PER_BLOCKDIM; - rt->mode = PTO2_MODE_EXECUTE; + rt->mode = MODE_EXECUTE; const auto *entry_points = reinterpret_cast(host_orch_func_ptr); if (entry_points->bind == nullptr) { @@ -1194,7 +1194,7 @@ extern "C" int bind_callable_to_runtime_impl( // No SM base: the scheduler and sm_handle are device-written now, so nothing // here stores one, and the region is not even committed yet. RuntimeContext *rt = - runtime_init_data_from_layout(host_arena, layout, PTO2_MODE_EXECUTE, /*sm_dev_base=*/nullptr, sm_size); + runtime_init_data_from_layout(host_arena, layout, MODE_EXECUTE, /*sm_dev_base=*/nullptr, sm_size); if (rt == nullptr) { LOG_ERROR("runtime_init_data_from_layout failed"); return PTO_RUNTIME_ERR_INTERNAL; @@ -1292,7 +1292,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e memset(&host_header, 0, sizeof(host_header)); if (execution_rc != 0) { - runtime_status = pto2_read_runtime_status(runtime, api, &host_header); + runtime_status = read_runtime_status(runtime, api, &host_header); } if (runtime_status != 0) { int32_t orch_error_code = host_header.orch_error_code.load(std::memory_order_relaxed); diff --git a/src/a2a3/runtime/host_build_graph/orchestration/orchestration_api.h b/src/a2a3/runtime/host_build_graph/orchestration/orchestration_api.h index 903c7aa32e..84a90a6151 100644 --- a/src/a2a3/runtime/host_build_graph/orchestration/orchestration_api.h +++ b/src/a2a3/runtime/host_build_graph/orchestration/orchestration_api.h @@ -870,8 +870,8 @@ rt_submit_graph(GraphFunctionWithConfig function, const GraphTaskArgs * This struct is defined identically in runtime_core.h (with an include * guard) so the executor can use the same type without including this header. */ -#ifndef PTO2_ORCHESTRATION_CONFIG_DEFINED -#define PTO2_ORCHESTRATION_CONFIG_DEFINED +#ifndef ORCHESTRATION_CONFIG_DEFINED +#define ORCHESTRATION_CONFIG_DEFINED struct OrchestrationConfig { int expected_arg_count; }; diff --git a/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h b/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h index 021a4d7cb4..5d1a6a2439 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h +++ b/src/a2a3/runtime/host_build_graph/runtime/dispatch_payload.h @@ -43,7 +43,7 @@ /** Max dispatch arguments: 16 scalars + up to 32 tensor pointers + ext params */ #ifndef DISPATCH_MAX_ARGS -#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) +#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + EXT_PARAMS_COUNT) #endif #ifndef CHIP_ALIGN_UP diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 2f382301bd..9b27e5f806 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1224,8 +1224,8 @@ struct PreparedTask { ChipTaskSlotState *slot_state = nullptr; }; -static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { - PTO2OutputLayout layout; +static OutputLayout calculate_output_layout(const CoreTaskArgs &args) { + OutputLayout layout; for (int32_t i = 0; i < args.tensor_count(); i++) { if (args.tag(i) != TensorArgType::OUTPUT) { continue; @@ -1454,7 +1454,7 @@ static TaskOutputTensors submit_task_common( CYCLE_COUNT_START(); ORCH_PHASE_START(); TaskOutputTensors result; - PTO2OutputLayout layout = calculate_output_layout(args); + OutputLayout layout = calculate_output_layout(args); PreparedTask prepared; if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; @@ -1980,7 +1980,7 @@ TaskOutputTensors graph_record_submit_node( recording.unsupported = true; } - const PTO2OutputLayout layout = calculate_output_layout(args); + const OutputLayout layout = calculate_output_layout(args); const uint64_t aligned_output = layout.total_output_size > 0 ? CHIP_ALIGN_UP(static_cast(layout.total_output_size), CHIP_ALIGN_SIZE) : 0; @@ -2690,7 +2690,7 @@ TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { ); } - PTO2OutputLayout layout = calculate_output_layout(args); + OutputLayout layout = calculate_output_layout(args); PreparedTask prepared; // Kernel-less alloc task: no active subtasks, no dispatch-time attributes. The // early-dispatch hint is force-set below (see the flag-the-creator note). diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 553c64f8fa..dc786698d4 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -55,7 +55,7 @@ host_tensor_write(HostTensorAccessor *, uint64_t dev_addr, const void *src, uint // device register unavailable on the host, so return a monotonic wall-clock // scaled to that counter's cycle units (PLATFORM_PROF_SYS_CNT_FREQ). The // cycle-denominated timeouts that run during host orchestration -// (PTO2_TENSOR_DATA_TIMEOUT_CYCLES in wait_for_tensor_ready, the fanin spill +// (TENSOR_DATA_TIMEOUT_CYCLES in wait_for_tensor_ready, the fanin spill // pool's backstop) then fire at their intended wall-clock; a constant 0 would // make them no-ops and spin forever. The AICPU build links the strong device // counter from device_time.cpp; hidden visibility keeps this off the global @@ -74,8 +74,7 @@ __attribute__((weak, visibility("hidden"))) uint64_t get_sys_cnt_aicpu() { // that define PLATFORM_PROF_SYS_CNT_FREQ locally, so pulling the platform header into // it caused a redefinition conflict (#1189). Scaling MS by the counter frequency (like // SCHEDULER_TIMEOUT_CYCLES) keeps the data-wait wall-clock identical across arches. -static constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_CYCLES = - (PTO2_TENSOR_DATA_TIMEOUT_MS * PLATFORM_PROF_SYS_CNT_FREQ) / 1000; +static constexpr uint64_t TENSOR_DATA_TIMEOUT_CYCLES = (TENSOR_DATA_TIMEOUT_MS * PLATFORM_PROF_SYS_CNT_FREQ) / 1000; // ============================================================================= // Orchestration Ops Table (function-pointer dispatch for orchestration .so) @@ -225,11 +224,11 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo failed = true; return; } - if (get_sys_cnt_aicpu() - t0 > PTO2_TENSOR_DATA_TIMEOUT_CYCLES) { + if (get_sys_cnt_aicpu() - t0 > TENSOR_DATA_TIMEOUT_CYCLES) { orch.report_fatal( SIMPLER_ERROR_TENSOR_WAIT_TIMEOUT, caller, "Timeout (%llu cycles): producer (ring=%d, local=%d) not completed", - (unsigned long long)PTO2_TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id + (unsigned long long)TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id ); failed = true; return; @@ -257,11 +256,11 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo failed = true; return; } - if (get_sys_cnt_aicpu() - t0 > PTO2_TENSOR_DATA_TIMEOUT_CYCLES) { + if (get_sys_cnt_aicpu() - t0 > TENSOR_DATA_TIMEOUT_CYCLES) { orch.report_fatal( SIMPLER_ERROR_TENSOR_WAIT_TIMEOUT, caller, "Timeout (%llu cycles): consumers of producer (ring=%d, local=%d) not done", - (unsigned long long)PTO2_TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id + (unsigned long long)TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id ); failed = true; return; diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h index 62e258d9f0..32f82e7335 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_core.h @@ -55,9 +55,9 @@ * Runtime execution mode */ enum RuntimeMode { - PTO2_MODE_EXECUTE = 0, // Execute tasks on workers - PTO2_MODE_SIMULATE = 1, // Simulate task execution with cycle counting - PTO2_MODE_GRAPH_ONLY = 2 // Build graph only, no execution + MODE_EXECUTE = 0, // Execute tasks on workers + MODE_SIMULATE = 1, // Simulate task execution with cycle counting + MODE_GRAPH_ONLY = 2 // Build graph only, no execution }; /** @@ -339,8 +339,8 @@ void set_tensor_data( * Slim config struct exported by orchestration .so via aicpu_orchestration_config(). * Shared definition with orchestration_api.h (same layout, guarded). */ -#ifndef PTO2_ORCHESTRATION_CONFIG_DEFINED -#define PTO2_ORCHESTRATION_CONFIG_DEFINED +#ifndef ORCHESTRATION_CONFIG_DEFINED +#define ORCHESTRATION_CONFIG_DEFINED struct OrchestrationConfig { int expected_arg_count; }; diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index 39a8766a79..06446d6c32 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -72,7 +72,7 @@ // Task management // Actual window size is passed at runtime to runtime_create_from_sm(). // Use pto2_task_slot(sched, task_id) for slot calculation. -#define PTO2_TASK_WINDOW_SIZE 16384 // Default task window size (power of 2) +#define CHIP_TASK_WINDOW_SIZE 16384 // Default task window size (power of 2) // host_build_graph has one ring, and carries no per-ring dimension anywhere: // host-orch builds the whole graph on the host, it fits that ring, and the @@ -151,14 +151,14 @@ inline constexpr int32_t ARG_POOL_ALIGN = 64; #define CHIP_DEP_DEGREE_WARN_THRESHOLD 16 // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle -// count (PTO2_TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp +// count (TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp // — its only user — by scaling with the platform counter frequency, like // SCHEDULER_TIMEOUT_CYCLES, so it reaps at the same wall-clock on every arch (a // fixed raw cycle count would be 15 s on a5 at 1 GHz but 300 s on a2a3 at 50 MHz). // PLATFORM_PROF_SYS_CNT_FREQ is deliberately NOT pulled into this header: it is // included by orchestrations that define that constant locally, so doing so caused // a redefinition conflict. See issue #1189. -constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s +constexpr uint64_t TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s // ============================================================================= // Task States @@ -207,7 +207,7 @@ enum class TaskKind : uint8_t { GRAPH_NODE = 3, }; -struct PTO2OutputLayout { +struct OutputLayout { uint64_t offsets[MAX_TENSOR_ARGS] = {}; uint64_t buffer_sizes[MAX_TENSOR_ARGS] = {}; int32_t total_output_size = 0; @@ -429,7 +429,7 @@ struct TaskPayload { * @param result Materialized output tensors (from TensorCreateInfo path) */ void - init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, PTO2OutputLayout &layout) { + init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, OutputLayout &layout) { tensor_count = args.tensor_count(); scalar_count = args.scalar_count(); diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index 8956105da6..efbd4e2fc1 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -911,12 +911,12 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // Read at one-time boot init, before the SM is reset for the run, so a ring // not yet written holds uninitialized memory (0xbe... under ASAN's // malloc-fill). Only a plausible count is taken — (0, - // PTO2_TASK_WINDOW_SIZE], since the ring cannot hold more than its task + // CHIP_TASK_WINDOW_SIZE], since the ring cannot hold more than its task // window — so any garbage pattern, negative or positive, leaves the count // at 0, which is the correct value at boot. int32_t pto2_count = 0; int32_t ring_tasks = header->ring.fc.current_task_index.load(std::memory_order_acquire); - if (ring_tasks > 0 && ring_tasks <= PTO2_TASK_WINDOW_SIZE) pto2_count = ring_tasks; + if (ring_tasks > 0 && ring_tasks <= CHIP_TASK_WINDOW_SIZE) pto2_count = ring_tasks; total_tasks_ = pto2_count; } else { total_tasks_ = 0; diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 8ad041e91d..70350a5603 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -111,7 +111,7 @@ bool SharedMemoryHandle::attach_populated( } SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { - const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); + const uint64_t buffer_size = calculate_size(CHIP_TASK_WINDOW_SIZE); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(buffer_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; @@ -120,7 +120,7 @@ SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &are memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); memset(buffer, 0, static_cast(buffer_size)); - if (!handle->init(buffer, buffer_size, PTO2_TASK_WINDOW_SIZE)) return nullptr; + if (!handle->init(buffer, buffer_size, CHIP_TASK_WINDOW_SIZE)) return nullptr; return handle; } diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h index f65fc338ef..55e7704258 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a2a3/runtime/host_build_graph/runtime/shared_memory.h @@ -214,7 +214,7 @@ struct SharedMemoryHandle { static uint64_t calculate_size(uint64_t task_window_size); // UT convenience: reserve wrapper + sm_base on `arena`, commit, and init using - // default PTO2_TASK_WINDOW_SIZE. Only valid when the arena is otherwise empty + // default CHIP_TASK_WINDOW_SIZE. Only valid when the arena is otherwise empty // (the call performs the single commit). All memory is owned by the arena — // caller must not call destroy(). static SharedMemoryHandle *create_and_init_default(DeviceArena &arena); diff --git a/src/a2a3/runtime/host_build_graph/runtime/submit_types.h b/src/a2a3/runtime/host_build_graph/runtime/submit_types.h index 21756a3e8b..e8be0426ae 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/submit_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/submit_types.h @@ -258,9 +258,9 @@ struct MixedKernels { * is expanded into at dispatch time. Each block receives a unique * block_idx in [0, block_num) via the per-dispatch LocalContext. */ -class PTO2LaunchSpec { +class LaunchSpec { public: - constexpr PTO2LaunchSpec() = default; + constexpr LaunchSpec() = default; int16_t block_num() const { return block_num_; } void set_block_num(int16_t n) { block_num_ = n; } diff --git a/src/a2a3/runtime/host_build_graph/runtime/types.h b/src/a2a3/runtime/host_build_graph/runtime/types.h index e6106e1bd4..50348f0b73 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/types.h @@ -247,7 +247,7 @@ struct Arg : TaskArgsTpl { bool has_error{false}; const char *error_msg{nullptr}; - PTO2LaunchSpec launch_spec; // SPMD launch parameters (block_num, etc.) + LaunchSpec launch_spec; // SPMD launch parameters (block_num, etc.) // Speculative early-dispatch hint (codegen-author set, off by default). When // true, the scheduler may stage this task on an idle core before its producer diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/common/intrinsic.h b/src/a2a3/runtime/tensormap_and_ringbuffer/common/intrinsic.h index 5229098552..eb613f7fe1 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/common/intrinsic.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/common/intrinsic.h @@ -97,7 +97,7 @@ #endif /** Number of extra pointer slots appended to the args[] tail (LocalContext + GlobalContext). */ -static constexpr int32_t PTO2_EXT_PARAMS_COUNT = 2; +static constexpr int32_t EXT_PARAMS_COUNT = 2; /** * Args[] suffix indices for context pointers. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 2df9338cc2..3f78c2ee29 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -232,8 +232,8 @@ AICore uses `last_reg_val` to detect new dispatches — identical values cause s | Constant | Default | Total (×4 rings) | | -------- | ------- | ---------------- | -| `PTO2_TASK_WINDOW_SIZE` | 16384 | 65536 | -| `PTO2_HEAP_SIZE` | 256 MB | 1 GB | +| `CHIP_TASK_WINDOW_SIZE` | 16384 | 65536 | +| `CHIP_HEAP_SIZE` | 256 MB | 1 GB | | `CHIP_DEP_LIST_POOL_SIZE` | 16384 | 65536 | ### 7.2 Runtime Overrides diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md index 046dd13447..08ec728d45 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md @@ -47,7 +47,7 @@ One extra step versus get_tensor_data: wait for all consumers to finish (`fanout ### 3.3 Timeout - Uses cycle counter (`get_sys_cnt_aicpu()`), checked every 1024 spins -- Threshold: `PTO2_TENSOR_DATA_TIMEOUT_CYCLES` (~10 s at 1.5 GHz) +- Threshold: `TENSOR_DATA_TIMEOUT_CYCLES` (~10 s at 1.5 GHz) - On timeout: sets `orch.fatal = true`, preventing further task submission ## 4. Seeding a Runtime-Created Output diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index 3dc8660818..130183d402 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -240,8 +240,8 @@ static bool resolve_ring_config( ) { uint64_t dep_pool_values[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - eff_task_window_sizes[r] = PTO2_TASK_WINDOW_SIZE; - eff_heap_sizes[r] = PTO2_HEAP_SIZE; + eff_task_window_sizes[r] = CHIP_TASK_WINDOW_SIZE; + eff_heap_sizes[r] = CHIP_HEAP_SIZE; dep_pool_values[r] = CHIP_DEP_LIST_POOL_SIZE; } @@ -284,7 +284,7 @@ static bool resolve_ring_config( return true; } -static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { +static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { if (runtime == nullptr || host_header == nullptr) { return 0; } @@ -748,7 +748,7 @@ static bool build_runtime_image( } RuntimeContext *rt = runtime_init_data_from_layout( - *host_arena, layout, PTO2_MODE_EXECUTE, ptrs.gm_sm, sizes.sm_size, ptrs.gm_heap, sizing.heap_sizes + *host_arena, layout, MODE_EXECUTE, ptrs.gm_sm, sizes.sm_size, ptrs.gm_heap, sizing.heap_sizes ); if (rt == nullptr) { LOG_ERROR("runtime_init_data_from_layout failed"); @@ -1021,7 +1021,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e memset(&host_header, 0, sizeof(host_header)); if (execution_rc != 0) { - runtime_status = pto2_read_runtime_status(runtime, api, &host_header); + runtime_status = read_runtime_status(runtime, api, &host_header); } if (runtime_status != 0) { int32_t orch_error_code = host_header.orch_error_code.load(std::memory_order_relaxed); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h b/src/a2a3/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h index c89a2e2fb8..b8070f0c43 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h @@ -379,8 +379,8 @@ class ScopeGuard { * This struct is defined identically in runtime_core.h (with an include * guard) so the executor can use the same type without including this header. */ -#ifndef PTO2_ORCHESTRATION_CONFIG_DEFINED -#define PTO2_ORCHESTRATION_CONFIG_DEFINED +#ifndef ORCHESTRATION_CONFIG_DEFINED +#define ORCHESTRATION_CONFIG_DEFINED struct OrchestrationConfig { int expected_arg_count; }; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h index b9ad62f2b7..101a3bb0e7 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h @@ -44,7 +44,7 @@ /** Max dispatch arguments: 16 scalars + up to 32 tensor pointers + ext params */ #ifndef DISPATCH_MAX_ARGS -#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) +#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + EXT_PARAMS_COUNT) #endif #ifndef CHIP_ALIGN_UP diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 1dab2c0053..4b041f7394 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -545,8 +545,8 @@ struct PreparedTask { ChipTaskSlotState *slot_state = nullptr; }; -static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { - PTO2OutputLayout layout; +static OutputLayout calculate_output_layout(const CoreTaskArgs &args) { + OutputLayout layout; for (int32_t i = 0; i < args.tensor_count(); i++) { if (args.tag(i) != TensorArgType::OUTPUT) { continue; @@ -583,7 +583,7 @@ static bool check_scope_can_accept_task(OrchestratorState *orch, TaskAllocator & LOG_ERROR("Solution:"); LOG_ERROR(" 1. Reduce tasks per scope (use batching/unroll)"); LOG_ERROR(" 2. Increase task window (current: %d)", allocator.window_size()); - LOG_ERROR(" Compile-time: PTO2_TASK_WINDOW_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_TASK_WINDOW_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_TASK_WINDOW="); LOG_ERROR(" 3. Split work across multiple scopes"); LOG_ERROR("========================================"); @@ -886,7 +886,7 @@ static TaskOutputTensors submit_task_common( ) { CYCLE_COUNT_START(); TaskOutputTensors result; - PTO2OutputLayout layout = calculate_output_layout(args); + OutputLayout layout = calculate_output_layout(args); PreparedTask prepared; if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; @@ -1289,7 +1289,7 @@ TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { return TaskOutputTensors{}; } - PTO2OutputLayout layout = calculate_output_layout(args); + OutputLayout layout = calculate_output_layout(args); PreparedTask prepared; // Kernel-less alloc task: no active subtasks, no dispatch-time markers. The // early-dispatch hint is force-set below (see the flag-the-creator note). diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 4d2db6b714..b98cfddc9a 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -52,7 +52,7 @@ #endif // Block notification interval (in spin counts) -#define PTO2_BLOCK_NOTIFY_INTERVAL 10000 +#define CHIP_BLOCK_NOTIFY_INTERVAL 10000 // Heap/task deadlock is detected structurally when the reclaim head is the // oldest task owned by an open scope on the blocked ring. This wall-clock value // is the backstop for all other cases; it is an ABSOLUTE TIME (not a spin @@ -200,7 +200,7 @@ class TaskAllocator { report_deadlock(output_size, blocked_on_heap, /*scope_gated=*/false); return {-1, -1, nullptr, nullptr}; } - if (spin_count % PTO2_BLOCK_NOTIFY_INTERVAL == 0) { + if (spin_count % CHIP_BLOCK_NOTIFY_INTERVAL == 0) { LOG_WARN( "[TaskAllocator ring=%u] BLOCKED: tasks=%d/%d, heap_used=%" PRIu64 "/%" PRIu64 ", heap_available=%" PRIu64 ", heap_cursor=%" PRIu64 ", on=%s, spins=%d", diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 174e2879d7..546beb0473 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -42,8 +42,7 @@ __attribute__((weak, visibility("hidden"))) uint64_t get_sys_cnt_aicpu() { retur // that define PLATFORM_PROF_SYS_CNT_FREQ locally, so pulling the platform header into // it caused a redefinition conflict (#1189). Scaling MS by the counter frequency (like // SCHEDULER_TIMEOUT_CYCLES) keeps the data-wait wall-clock identical across arches. -static constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_CYCLES = - (PTO2_TENSOR_DATA_TIMEOUT_MS * PLATFORM_PROF_SYS_CNT_FREQ) / 1000; +static constexpr uint64_t TENSOR_DATA_TIMEOUT_CYCLES = (TENSOR_DATA_TIMEOUT_MS * PLATFORM_PROF_SYS_CNT_FREQ) / 1000; // ============================================================================= // Orchestration Ops Table (function-pointer dispatch for orchestration .so) @@ -125,11 +124,11 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo failed = true; return; } - if (get_sys_cnt_aicpu() - t0 > PTO2_TENSOR_DATA_TIMEOUT_CYCLES) { + if (get_sys_cnt_aicpu() - t0 > TENSOR_DATA_TIMEOUT_CYCLES) { orch.report_fatal( SIMPLER_ERROR_TENSOR_WAIT_TIMEOUT, caller, "Timeout (%llu cycles): producer (ring=%d, local=%d) not completed", - (unsigned long long)PTO2_TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id + (unsigned long long)TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id ); failed = true; return; @@ -152,11 +151,11 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo failed = true; return; } - if (get_sys_cnt_aicpu() - t0 > PTO2_TENSOR_DATA_TIMEOUT_CYCLES) { + if (get_sys_cnt_aicpu() - t0 > TENSOR_DATA_TIMEOUT_CYCLES) { orch.report_fatal( SIMPLER_ERROR_TENSOR_WAIT_TIMEOUT, caller, "Timeout (%llu cycles): consumers of producer (ring=%d, local=%d) not done", - (unsigned long long)PTO2_TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id + (unsigned long long)TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id ); failed = true; return; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index 2f643bd1b9..6f87a0c48d 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -52,9 +52,9 @@ * Runtime execution mode */ enum RuntimeMode { - PTO2_MODE_EXECUTE = 0, // Execute tasks on workers - PTO2_MODE_SIMULATE = 1, // Simulate task execution with cycle counting - PTO2_MODE_GRAPH_ONLY = 2 // Build graph only, no execution + MODE_EXECUTE = 0, // Execute tasks on workers + MODE_SIMULATE = 1, // Simulate task execution with cycle counting + MODE_GRAPH_ONLY = 2 // Build graph only, no execution }; /** @@ -310,8 +310,8 @@ void set_tensor_data( * Slim config struct exported by orchestration .so via aicpu_orchestration_config(). * Shared definition with orchestration_api.h (same layout, guarded). */ -#ifndef PTO2_ORCHESTRATION_CONFIG_DEFINED -#define PTO2_ORCHESTRATION_CONFIG_DEFINED +#ifndef ORCHESTRATION_CONFIG_DEFINED +#define ORCHESTRATION_CONFIG_DEFINED struct OrchestrationConfig { int expected_arg_count; }; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 2e3c92457f..e9aedb4f5e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -59,17 +59,17 @@ // ============================================================================= // Task management -// NOTE: PTO2_TASK_WINDOW_SIZE is now a per-ring default value. +// NOTE: CHIP_TASK_WINDOW_SIZE is now a per-ring default value. // Actual window size is passed at runtime to runtime_reserve_layout(). // Use pto2_task_slot(sched, task_id) for slot calculation. -#define PTO2_TASK_WINDOW_SIZE 16384 // Default per-ring task window size (power of 2) +#define CHIP_TASK_WINDOW_SIZE 16384 // Default per-ring task window size (power of 2) // Multi-ring: number of independent ring layers (HeapRing + TaskRing + DepPool per layer) // Scope depth maps to ring index via: min(scope_depth, CHIP_MAX_RING_DEPTH - 1) #define CHIP_MAX_RING_DEPTH 4 // Memory pools (per-ring defaults; total = value × CHIP_MAX_RING_DEPTH) -#define PTO2_HEAP_SIZE (256 * 1024 * 1024) // 256MB per ring (1GB total) +#define CHIP_HEAP_SIZE (256 * 1024 * 1024) // 256MB per ring (1GB total) #define CHIP_DEP_LIST_POOL_SIZE 16384 // Per-ring dependency list pool entries #define CHIP_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool #define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) @@ -77,11 +77,11 @@ // Scope management #define CHIP_MAX_SCOPE_DEPTH 64 // Maximum nesting depth // Hard cap for the scope_tasks buffer. Equals the total in-flight ring slot -// budget (PTO2_TASK_WINDOW_SIZE × CHIP_MAX_RING_DEPTH): once every ring slot +// budget (CHIP_TASK_WINDOW_SIZE × CHIP_MAX_RING_DEPTH): once every ring slot // is in flight, no more tasks can ever be pushed regardless of buffer size. // scope_tasks_push fatals on overflow rather than growing the arena-owned // buffer (which would be UB on the arena's malloc'd backing). -#define CHIP_SCOPE_TASKS_CAP (PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) +#define CHIP_SCOPE_TASKS_CAP (CHIP_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) // Ready queue #define CHIP_READY_QUEUE_SIZE 65536 // Per-shape queue size @@ -106,14 +106,14 @@ #define CHIP_DEP_POOL_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle -// count (PTO2_TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp +// count (TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp // — its only user — by scaling with the platform counter frequency, like // SCHEDULER_TIMEOUT_CYCLES, so it reaps at the same wall-clock on every arch (a // fixed raw cycle count would be 15 s on a5 at 1 GHz but 300 s on a2a3 at 50 MHz). // PLATFORM_PROF_SYS_CNT_FREQ is deliberately NOT pulled into this header: it is // included by orchestrations that define that constant locally, so doing so caused // a redefinition conflict. See issue #1189. -constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s +constexpr uint64_t TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s // ============================================================================= // Task States @@ -152,7 +152,7 @@ struct TaskAllocResult { bool failed() const { return task_id < 0; } }; -struct PTO2OutputLayout { +struct OutputLayout { uint64_t offsets[MAX_TENSOR_ARGS] = {}; uint64_t buffer_sizes[MAX_TENSOR_ARGS] = {}; int32_t total_output_size = 0; @@ -375,7 +375,7 @@ struct TaskPayload { * @param result Materialized output tensors (from TensorCreateInfo path) */ void - init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, PTO2OutputLayout &layout) { + init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, OutputLayout &layout) { tensor_count = args.tensor_count(); scalar_count = args.scalar_count(); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index fa51e936c6..97fbda33b4 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -33,7 +33,7 @@ // ============================================================================= namespace { -inline constexpr int32_t PTO2_DEFERRED_RELEASE_CAP = 256; +inline constexpr int32_t DEFERRED_RELEASE_CAP = 256; } // Pure function: read register result -> SlotTransition (no side effects). @@ -228,7 +228,7 @@ void SchedulerContext::complete_slot_task( } chip_swimlane.phase_complete_count++; #endif - if (deferred_release_count < PTO2_DEFERRED_RELEASE_CAP) { + if (deferred_release_count < DEFERRED_RELEASE_CAP) { deferred_release_slot_states[deferred_release_count++] = &slot_state; } else { while (deferred_release_count > 0) { diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index c73fc463eb..42157339b6 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -51,7 +51,7 @@ static_assert(sizeof(ChipTensor) == TASKPAYLOAD_TENSOR_STRIDE); // ============================================================================= namespace { -inline constexpr int32_t PTO2_DEFERRED_RELEASE_CAP = 256; +inline constexpr int32_t DEFERRED_RELEASE_CAP = 256; } // The early-dispatch core bitmask (EARLY_DISPATCH_CORE_MASK_WORDS * 64 bits) must cover @@ -914,7 +914,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ chip_swimlane.chip_swimlane_enabled = (chip_swimlane_level_ != ChipSwimlaneLevel::DISABLED); #endif - ChipTaskSlotState *deferred_release_slot_states[PTO2_DEFERRED_RELEASE_CAP]; + ChipTaskSlotState *deferred_release_slot_states[DEFERRED_RELEASE_CAP]; int32_t deferred_release_count = 0; // PMU runs require single-issue dispatch — overlapping in-flight tasks @@ -1103,8 +1103,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ if (rt_ != nullptr && rt_->aicore_mailbox != nullptr && (sched_->async_wait_list.count > 0 || rt_->aicore_mailbox->has_pending())) { AsyncPollResult poll_result = sched_->async_wait_list.poll_and_complete( - rt_->aicore_mailbox, sched_, deferred_release_slot_states, deferred_release_count, - PTO2_DEFERRED_RELEASE_CAP + rt_->aicore_mailbox, sched_, deferred_release_slot_states, deferred_release_count, DEFERRED_RELEASE_CAP #if SIMPLER_SCHED_PROFILING , thread_idx @@ -1254,7 +1253,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // and no fanout pre-conditions beyond their own producers; release // self-reference so the slot can reach CONSUMED once all consumers drain. deferred_release_slot_states[deferred_release_count++] = &dummy_slot; - if (deferred_release_count >= PTO2_DEFERRED_RELEASE_CAP) { + if (deferred_release_count >= DEFERRED_RELEASE_CAP) { while (deferred_release_count > 0) { #if SIMPLER_SCHED_PROFILING (void)sched_->on_task_release( diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index d1d01d984b..132e298a2d 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -95,7 +95,7 @@ bool SharedMemoryHandle::init_per_ring( } SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { - const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); + const uint64_t buffer_size = calculate_size(CHIP_TASK_WINDOW_SIZE); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(buffer_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; @@ -104,7 +104,7 @@ SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &are memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); memset(buffer, 0, static_cast(buffer_size)); - if (!handle->init(buffer, buffer_size, PTO2_TASK_WINDOW_SIZE, PTO2_HEAP_SIZE)) return nullptr; + if (!handle->init(buffer, buffer_size, CHIP_TASK_WINDOW_SIZE, CHIP_HEAP_SIZE)) return nullptr; return handle; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 4cb9d8387c..2bf29330e9 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -193,7 +193,7 @@ struct SharedMemoryHandle { static uint64_t calculate_size_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]); // UT convenience: reserve wrapper + sm_base on `arena`, commit, and init - // using default PTO2_TASK_WINDOW_SIZE / PTO2_HEAP_SIZE. Only valid when the + // using default CHIP_TASK_WINDOW_SIZE / CHIP_HEAP_SIZE. Only valid when the // arena is otherwise empty (the call performs the single commit). All // memory is owned by the arena — caller must not call destroy(). static SharedMemoryHandle *create_and_init_default(DeviceArena &arena); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h index fb5feb3abc..69249b2068 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/submit_types.h @@ -254,9 +254,9 @@ struct MixedKernels { * is expanded into at dispatch time. Each block receives a unique * block_idx in [0, block_num) via the per-dispatch LocalContext. */ -class PTO2LaunchSpec { +class LaunchSpec { public: - constexpr PTO2LaunchSpec() = default; + constexpr LaunchSpec() = default; int16_t block_num() const { return block_num_; } void set_block_num(int16_t n) { block_num_ = n; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h index 8c4785b1e8..fd9fab708e 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/types.h @@ -280,7 +280,7 @@ struct Arg : TaskArgsTpl { bool has_error{false}; const char *error_msg{nullptr}; - PTO2LaunchSpec launch_spec; // SPMD launch parameters (block_num, etc.) + LaunchSpec launch_spec; // SPMD launch parameters (block_num, etc.) // Early-dispatch hint (codegen-author set, off by default). When // true, the scheduler may stage this task on an idle core before its producer diff --git a/src/a5/runtime/host_build_graph/common/intrinsic.h b/src/a5/runtime/host_build_graph/common/intrinsic.h index 8dcd4ca5fd..a9184f4e5f 100644 --- a/src/a5/runtime/host_build_graph/common/intrinsic.h +++ b/src/a5/runtime/host_build_graph/common/intrinsic.h @@ -93,7 +93,7 @@ #endif /** Number of extra pointer slots appended to the args[] tail (LocalContext + GlobalContext). */ -static constexpr int32_t PTO2_EXT_PARAMS_COUNT = 2; +static constexpr int32_t EXT_PARAMS_COUNT = 2; /** * Args[] suffix indices for context pointers. diff --git a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp index 54c3f455e3..efcfeb179a 100644 --- a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp @@ -277,7 +277,7 @@ static uint64_t read_ring_override(const uint64_t *base, int idx) { // reclaiming runtime's knobs; this function does not resolve them, and the caller // reads them only to warn that they reach nothing here. static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t *eff_task_window_size) { - *eff_task_window_size = PTO2_TASK_WINDOW_SIZE; + *eff_task_window_size = CHIP_TASK_WINDOW_SIZE; apply_env_ring_value("PTO2_RING_TASK_WINDOW", 4, static_cast(INT32_MAX), true, eff_task_window_size); @@ -307,7 +307,7 @@ static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t return true; } -static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { +static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { if (runtime == nullptr || api == nullptr || host_header == nullptr) { return 0; } @@ -636,7 +636,7 @@ int32_t run_host_orchestration( runtime_bind_ops(rt); orchestrator.total_cluster_count = block_dim * PLATFORM_AIC_CORES_PER_BLOCKDIM; orchestrator.total_aiv_count = block_dim * PLATFORM_AIV_CORES_PER_BLOCKDIM; - rt->mode = PTO2_MODE_EXECUTE; + rt->mode = MODE_EXECUTE; // get_tensor_data/set_tensor_data resolve buffer.addr through the host // views registered at staging time (runtime/host_tensor_access.h), so the // host orchestrator can read control tensors (e.g. paged_attention's @@ -1210,7 +1210,7 @@ extern "C" int bind_callable_to_runtime_impl( // No SM base: the scheduler and sm_handle are device-written now, so nothing // here stores one, and the region is not even committed yet. RuntimeContext *rt = - runtime_init_data_from_layout(host_arena, layout, PTO2_MODE_EXECUTE, /*sm_dev_base=*/nullptr, sm_size); + runtime_init_data_from_layout(host_arena, layout, MODE_EXECUTE, /*sm_dev_base=*/nullptr, sm_size); if (rt == nullptr) { LOG_ERROR("runtime_init_data_from_layout failed"); return PTO_RUNTIME_ERR_INTERNAL; @@ -1315,7 +1315,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e memset(&host_header, 0, sizeof(host_header)); if (execution_rc != 0) { - runtime_status = pto2_read_runtime_status(runtime, api, &host_header); + runtime_status = read_runtime_status(runtime, api, &host_header); } if (runtime_status != 0) { int32_t orch_error_code = host_header.orch_error_code.load(std::memory_order_relaxed); diff --git a/src/a5/runtime/host_build_graph/orchestration/orchestration_api.h b/src/a5/runtime/host_build_graph/orchestration/orchestration_api.h index 903c7aa32e..84a90a6151 100644 --- a/src/a5/runtime/host_build_graph/orchestration/orchestration_api.h +++ b/src/a5/runtime/host_build_graph/orchestration/orchestration_api.h @@ -870,8 +870,8 @@ rt_submit_graph(GraphFunctionWithConfig function, const GraphTaskArgs * This struct is defined identically in runtime_core.h (with an include * guard) so the executor can use the same type without including this header. */ -#ifndef PTO2_ORCHESTRATION_CONFIG_DEFINED -#define PTO2_ORCHESTRATION_CONFIG_DEFINED +#ifndef ORCHESTRATION_CONFIG_DEFINED +#define ORCHESTRATION_CONFIG_DEFINED struct OrchestrationConfig { int expected_arg_count; }; diff --git a/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h b/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h index 021a4d7cb4..5d1a6a2439 100644 --- a/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h +++ b/src/a5/runtime/host_build_graph/runtime/dispatch_payload.h @@ -43,7 +43,7 @@ /** Max dispatch arguments: 16 scalars + up to 32 tensor pointers + ext params */ #ifndef DISPATCH_MAX_ARGS -#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) +#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + EXT_PARAMS_COUNT) #endif #ifndef CHIP_ALIGN_UP diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 2f382301bd..9b27e5f806 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1224,8 +1224,8 @@ struct PreparedTask { ChipTaskSlotState *slot_state = nullptr; }; -static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { - PTO2OutputLayout layout; +static OutputLayout calculate_output_layout(const CoreTaskArgs &args) { + OutputLayout layout; for (int32_t i = 0; i < args.tensor_count(); i++) { if (args.tag(i) != TensorArgType::OUTPUT) { continue; @@ -1454,7 +1454,7 @@ static TaskOutputTensors submit_task_common( CYCLE_COUNT_START(); ORCH_PHASE_START(); TaskOutputTensors result; - PTO2OutputLayout layout = calculate_output_layout(args); + OutputLayout layout = calculate_output_layout(args); PreparedTask prepared; if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; @@ -1980,7 +1980,7 @@ TaskOutputTensors graph_record_submit_node( recording.unsupported = true; } - const PTO2OutputLayout layout = calculate_output_layout(args); + const OutputLayout layout = calculate_output_layout(args); const uint64_t aligned_output = layout.total_output_size > 0 ? CHIP_ALIGN_UP(static_cast(layout.total_output_size), CHIP_ALIGN_SIZE) : 0; @@ -2690,7 +2690,7 @@ TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { ); } - PTO2OutputLayout layout = calculate_output_layout(args); + OutputLayout layout = calculate_output_layout(args); PreparedTask prepared; // Kernel-less alloc task: no active subtasks, no dispatch-time attributes. The // early-dispatch hint is force-set below (see the flag-the-creator note). diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp index 553c64f8fa..dc786698d4 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/runtime_core.cpp @@ -55,7 +55,7 @@ host_tensor_write(HostTensorAccessor *, uint64_t dev_addr, const void *src, uint // device register unavailable on the host, so return a monotonic wall-clock // scaled to that counter's cycle units (PLATFORM_PROF_SYS_CNT_FREQ). The // cycle-denominated timeouts that run during host orchestration -// (PTO2_TENSOR_DATA_TIMEOUT_CYCLES in wait_for_tensor_ready, the fanin spill +// (TENSOR_DATA_TIMEOUT_CYCLES in wait_for_tensor_ready, the fanin spill // pool's backstop) then fire at their intended wall-clock; a constant 0 would // make them no-ops and spin forever. The AICPU build links the strong device // counter from device_time.cpp; hidden visibility keeps this off the global @@ -74,8 +74,7 @@ __attribute__((weak, visibility("hidden"))) uint64_t get_sys_cnt_aicpu() { // that define PLATFORM_PROF_SYS_CNT_FREQ locally, so pulling the platform header into // it caused a redefinition conflict (#1189). Scaling MS by the counter frequency (like // SCHEDULER_TIMEOUT_CYCLES) keeps the data-wait wall-clock identical across arches. -static constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_CYCLES = - (PTO2_TENSOR_DATA_TIMEOUT_MS * PLATFORM_PROF_SYS_CNT_FREQ) / 1000; +static constexpr uint64_t TENSOR_DATA_TIMEOUT_CYCLES = (TENSOR_DATA_TIMEOUT_MS * PLATFORM_PROF_SYS_CNT_FREQ) / 1000; // ============================================================================= // Orchestration Ops Table (function-pointer dispatch for orchestration .so) @@ -225,11 +224,11 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo failed = true; return; } - if (get_sys_cnt_aicpu() - t0 > PTO2_TENSOR_DATA_TIMEOUT_CYCLES) { + if (get_sys_cnt_aicpu() - t0 > TENSOR_DATA_TIMEOUT_CYCLES) { orch.report_fatal( SIMPLER_ERROR_TENSOR_WAIT_TIMEOUT, caller, "Timeout (%llu cycles): producer (ring=%d, local=%d) not completed", - (unsigned long long)PTO2_TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id + (unsigned long long)TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id ); failed = true; return; @@ -257,11 +256,11 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo failed = true; return; } - if (get_sys_cnt_aicpu() - t0 > PTO2_TENSOR_DATA_TIMEOUT_CYCLES) { + if (get_sys_cnt_aicpu() - t0 > TENSOR_DATA_TIMEOUT_CYCLES) { orch.report_fatal( SIMPLER_ERROR_TENSOR_WAIT_TIMEOUT, caller, "Timeout (%llu cycles): consumers of producer (ring=%d, local=%d) not done", - (unsigned long long)PTO2_TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id + (unsigned long long)TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id ); failed = true; return; diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_core.h b/src/a5/runtime/host_build_graph/runtime/runtime_core.h index 6fac6296fd..9aec559321 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_core.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_core.h @@ -55,9 +55,9 @@ * Runtime execution mode */ enum RuntimeMode { - PTO2_MODE_EXECUTE = 0, // Execute tasks on workers - PTO2_MODE_SIMULATE = 1, // Simulate task execution with cycle counting - PTO2_MODE_GRAPH_ONLY = 2 // Build graph only, no execution + MODE_EXECUTE = 0, // Execute tasks on workers + MODE_SIMULATE = 1, // Simulate task execution with cycle counting + MODE_GRAPH_ONLY = 2 // Build graph only, no execution }; /** @@ -340,8 +340,8 @@ void set_tensor_data( * Slim config struct exported by orchestration .so via aicpu_orchestration_config(). * Shared definition with orchestration_api.h (same layout, guarded). */ -#ifndef PTO2_ORCHESTRATION_CONFIG_DEFINED -#define PTO2_ORCHESTRATION_CONFIG_DEFINED +#ifndef ORCHESTRATION_CONFIG_DEFINED +#define ORCHESTRATION_CONFIG_DEFINED struct OrchestrationConfig { int expected_arg_count; }; diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index fb4b3555bb..9b32e39a67 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -72,7 +72,7 @@ // Task management // Actual window size is passed at runtime to runtime_create_from_sm(). // Use pto2_task_slot(sched, task_id) for slot calculation. -#define PTO2_TASK_WINDOW_SIZE 16384 // Default task window size (power of 2) +#define CHIP_TASK_WINDOW_SIZE 16384 // Default task window size (power of 2) // host_build_graph has one ring, and carries no per-ring dimension anywhere: // host-orch builds the whole graph on the host, it fits that ring, and the @@ -152,14 +152,14 @@ inline constexpr int32_t ARG_POOL_ALIGN = 64; #define CHIP_DEP_DEGREE_WARN_THRESHOLD 16 // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle -// count (PTO2_TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp +// count (TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp // — its only user — by scaling with the platform counter frequency, like // SCHEDULER_TIMEOUT_CYCLES, so it reaps at the same wall-clock on every arch (a // fixed raw cycle count would be 15 s on a5 at 1 GHz but 300 s on a2a3 at 50 MHz). // PLATFORM_PROF_SYS_CNT_FREQ is deliberately NOT pulled into this header: it is // included by orchestrations that define that constant locally, so doing so caused // a redefinition conflict. See issue #1189. -constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s +constexpr uint64_t TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s // ============================================================================= // Task States @@ -208,7 +208,7 @@ enum class TaskKind : uint8_t { GRAPH_NODE = 3, }; -struct PTO2OutputLayout { +struct OutputLayout { uint64_t offsets[MAX_TENSOR_ARGS] = {}; uint64_t buffer_sizes[MAX_TENSOR_ARGS] = {}; int32_t total_output_size = 0; @@ -430,7 +430,7 @@ struct TaskPayload { * @param result Materialized output tensors (from TensorCreateInfo path) */ void - init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, PTO2OutputLayout &layout) { + init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, OutputLayout &layout) { tensor_count = args.tensor_count(); scalar_count = args.scalar_count(); diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index 8956105da6..efbd4e2fc1 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -911,12 +911,12 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // Read at one-time boot init, before the SM is reset for the run, so a ring // not yet written holds uninitialized memory (0xbe... under ASAN's // malloc-fill). Only a plausible count is taken — (0, - // PTO2_TASK_WINDOW_SIZE], since the ring cannot hold more than its task + // CHIP_TASK_WINDOW_SIZE], since the ring cannot hold more than its task // window — so any garbage pattern, negative or positive, leaves the count // at 0, which is the correct value at boot. int32_t pto2_count = 0; int32_t ring_tasks = header->ring.fc.current_task_index.load(std::memory_order_acquire); - if (ring_tasks > 0 && ring_tasks <= PTO2_TASK_WINDOW_SIZE) pto2_count = ring_tasks; + if (ring_tasks > 0 && ring_tasks <= CHIP_TASK_WINDOW_SIZE) pto2_count = ring_tasks; total_tasks_ = pto2_count; } else { total_tasks_ = 0; diff --git a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 8ad041e91d..70350a5603 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -111,7 +111,7 @@ bool SharedMemoryHandle::attach_populated( } SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { - const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); + const uint64_t buffer_size = calculate_size(CHIP_TASK_WINDOW_SIZE); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(buffer_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; @@ -120,7 +120,7 @@ SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &are memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); memset(buffer, 0, static_cast(buffer_size)); - if (!handle->init(buffer, buffer_size, PTO2_TASK_WINDOW_SIZE)) return nullptr; + if (!handle->init(buffer, buffer_size, CHIP_TASK_WINDOW_SIZE)) return nullptr; return handle; } diff --git a/src/a5/runtime/host_build_graph/runtime/shared_memory.h b/src/a5/runtime/host_build_graph/runtime/shared_memory.h index f65fc338ef..55e7704258 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared_memory.h +++ b/src/a5/runtime/host_build_graph/runtime/shared_memory.h @@ -214,7 +214,7 @@ struct SharedMemoryHandle { static uint64_t calculate_size(uint64_t task_window_size); // UT convenience: reserve wrapper + sm_base on `arena`, commit, and init using - // default PTO2_TASK_WINDOW_SIZE. Only valid when the arena is otherwise empty + // default CHIP_TASK_WINDOW_SIZE. Only valid when the arena is otherwise empty // (the call performs the single commit). All memory is owned by the arena — // caller must not call destroy(). static SharedMemoryHandle *create_and_init_default(DeviceArena &arena); diff --git a/src/a5/runtime/host_build_graph/runtime/submit_types.h b/src/a5/runtime/host_build_graph/runtime/submit_types.h index 21756a3e8b..e8be0426ae 100644 --- a/src/a5/runtime/host_build_graph/runtime/submit_types.h +++ b/src/a5/runtime/host_build_graph/runtime/submit_types.h @@ -258,9 +258,9 @@ struct MixedKernels { * is expanded into at dispatch time. Each block receives a unique * block_idx in [0, block_num) via the per-dispatch LocalContext. */ -class PTO2LaunchSpec { +class LaunchSpec { public: - constexpr PTO2LaunchSpec() = default; + constexpr LaunchSpec() = default; int16_t block_num() const { return block_num_; } void set_block_num(int16_t n) { block_num_ = n; } diff --git a/src/a5/runtime/host_build_graph/runtime/types.h b/src/a5/runtime/host_build_graph/runtime/types.h index e6106e1bd4..50348f0b73 100644 --- a/src/a5/runtime/host_build_graph/runtime/types.h +++ b/src/a5/runtime/host_build_graph/runtime/types.h @@ -247,7 +247,7 @@ struct Arg : TaskArgsTpl { bool has_error{false}; const char *error_msg{nullptr}; - PTO2LaunchSpec launch_spec; // SPMD launch parameters (block_num, etc.) + LaunchSpec launch_spec; // SPMD launch parameters (block_num, etc.) // Speculative early-dispatch hint (codegen-author set, off by default). When // true, the scheduler may stage this task on an idle core before its producer diff --git a/src/a5/runtime/tensormap_and_ringbuffer/common/intrinsic.h b/src/a5/runtime/tensormap_and_ringbuffer/common/intrinsic.h index 7d80e44cef..04e6feed9b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/common/intrinsic.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/common/intrinsic.h @@ -97,7 +97,7 @@ #endif /** Number of extra pointer slots appended to the args[] tail (LocalContext + GlobalContext). */ -static constexpr int32_t PTO2_EXT_PARAMS_COUNT = 2; +static constexpr int32_t EXT_PARAMS_COUNT = 2; /** * Args[] suffix indices for context pointers. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 615e946db3..04d4e904e8 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -234,8 +234,8 @@ AICore uses `last_reg_val` to detect new dispatches — identical values cause s | Constant | Default | Total (×4 rings) | | -------- | ------- | ---------------- | -| `PTO2_TASK_WINDOW_SIZE` | 16384 | 65536 | -| `PTO2_HEAP_SIZE` | 256 MB | 1 GB | +| `CHIP_TASK_WINDOW_SIZE` | 16384 | 65536 | +| `CHIP_HEAP_SIZE` | 256 MB | 1 GB | | `CHIP_DEP_LIST_POOL_SIZE` | 16384 | 65536 | ### 7.2 Runtime Overrides diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md index 046dd13447..08ec728d45 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/SCALAR_DATA_ACCESS.md @@ -47,7 +47,7 @@ One extra step versus get_tensor_data: wait for all consumers to finish (`fanout ### 3.3 Timeout - Uses cycle counter (`get_sys_cnt_aicpu()`), checked every 1024 spins -- Threshold: `PTO2_TENSOR_DATA_TIMEOUT_CYCLES` (~10 s at 1.5 GHz) +- Threshold: `TENSOR_DATA_TIMEOUT_CYCLES` (~10 s at 1.5 GHz) - On timeout: sets `orch.fatal = true`, preventing further task submission ## 4. Seeding a Runtime-Created Output diff --git a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index 1b165480f6..384028ccbf 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -240,8 +240,8 @@ static bool resolve_ring_config( ) { uint64_t dep_pool_values[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - eff_task_window_sizes[r] = PTO2_TASK_WINDOW_SIZE; - eff_heap_sizes[r] = PTO2_HEAP_SIZE; + eff_task_window_sizes[r] = CHIP_TASK_WINDOW_SIZE; + eff_heap_sizes[r] = CHIP_HEAP_SIZE; dep_pool_values[r] = CHIP_DEP_LIST_POOL_SIZE; } @@ -284,7 +284,7 @@ static bool resolve_ring_config( return true; } -static int32_t pto2_read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { +static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedMemoryHeader *host_header) { if (runtime == nullptr || host_header == nullptr) { return 0; } @@ -748,7 +748,7 @@ static bool build_runtime_image( } RuntimeContext *rt = runtime_init_data_from_layout( - *host_arena, layout, PTO2_MODE_EXECUTE, ptrs.gm_sm, sizes.sm_size, ptrs.gm_heap, sizing.heap_sizes + *host_arena, layout, MODE_EXECUTE, ptrs.gm_sm, sizes.sm_size, ptrs.gm_heap, sizing.heap_sizes ); if (rt == nullptr) { LOG_ERROR("runtime_init_data_from_layout failed"); @@ -1021,7 +1021,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e memset(&host_header, 0, sizeof(host_header)); if (execution_rc != 0) { - runtime_status = pto2_read_runtime_status(runtime, api, &host_header); + runtime_status = read_runtime_status(runtime, api, &host_header); } if (runtime_status != 0) { int32_t orch_error_code = host_header.orch_error_code.load(std::memory_order_relaxed); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h b/src/a5/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h index c89a2e2fb8..b8070f0c43 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/orchestration/orchestration_api.h @@ -379,8 +379,8 @@ class ScopeGuard { * This struct is defined identically in runtime_core.h (with an include * guard) so the executor can use the same type without including this header. */ -#ifndef PTO2_ORCHESTRATION_CONFIG_DEFINED -#define PTO2_ORCHESTRATION_CONFIG_DEFINED +#ifndef ORCHESTRATION_CONFIG_DEFINED +#define ORCHESTRATION_CONFIG_DEFINED struct OrchestrationConfig { int expected_arg_count; }; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h index 1434658bb5..4806422de3 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/dispatch_payload.h @@ -44,7 +44,7 @@ /** Max dispatch arguments: 16 scalars + up to 32 tensor pointers + ext params */ #ifndef DISPATCH_MAX_ARGS -#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + PTO2_EXT_PARAMS_COUNT) +#define DISPATCH_MAX_ARGS (MAX_TENSOR_ARGS + MAX_SCALAR_ARGS + EXT_PARAMS_COUNT) #endif #ifndef CHIP_ALIGN_UP diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 030c4999c7..52c78653e2 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -537,8 +537,8 @@ struct PreparedTask { ChipTaskSlotState *slot_state = nullptr; }; -static PTO2OutputLayout calculate_output_layout(const CoreTaskArgs &args) { - PTO2OutputLayout layout; +static OutputLayout calculate_output_layout(const CoreTaskArgs &args) { + OutputLayout layout; for (int32_t i = 0; i < args.tensor_count(); i++) { if (args.tag(i) != TensorArgType::OUTPUT) { continue; @@ -575,7 +575,7 @@ static bool check_scope_can_accept_task(OrchestratorState *orch, TaskAllocator & LOG_ERROR("Solution:"); LOG_ERROR(" 1. Reduce tasks per scope (use batching/unroll)"); LOG_ERROR(" 2. Increase task window (current: %d)", allocator.window_size()); - LOG_ERROR(" Compile-time: PTO2_TASK_WINDOW_SIZE in runtime_types.h"); + LOG_ERROR(" Compile-time: CHIP_TASK_WINDOW_SIZE in runtime_types.h"); LOG_ERROR(" Runtime env: PTO2_RING_TASK_WINDOW="); LOG_ERROR(" 3. Split work across multiple scopes"); LOG_ERROR("========================================"); @@ -850,7 +850,7 @@ static bool ensure_tensormap_capacity(OrchestratorState *orch, int32_t needed) { if (!block_timing) { block_cycle0 = now; block_timing = true; - } else if (now - block_cycle0 >= PTO2_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { + } else if (now - block_cycle0 >= CHIP_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { request_reclaim_publication(); } if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { @@ -888,7 +888,7 @@ static TaskOutputTensors submit_task_common( ) { CYCLE_COUNT_START(); TaskOutputTensors result; - PTO2OutputLayout layout = calculate_output_layout(args); + OutputLayout layout = calculate_output_layout(args); PreparedTask prepared; if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; @@ -1291,7 +1291,7 @@ TaskOutputTensors OrchestratorState::alloc_tensors(const CoreTaskArgs &args) { return TaskOutputTensors{}; } - PTO2OutputLayout layout = calculate_output_layout(args); + OutputLayout layout = calculate_output_layout(args); PreparedTask prepared; // Kernel-less alloc task: no active subtasks, no dispatch-time attributes. The // early-dispatch hint is force-set below (see the flag-the-creator note). diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index 1e206b6dc4..d0778b67d5 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -94,7 +94,7 @@ bool FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { if (!block_timing) { block_cycle0 = now; block_timing = true; - } else if (!watermark_synchronized && now - block_cycle0 >= PTO2_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { + } else if (!watermark_synchronized && now - block_cycle0 >= CHIP_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { publication_request.request(); } if (now - block_cycle0 >= ALLOC_DEADLOCK_TIMEOUT_CYCLES) { @@ -229,7 +229,7 @@ bool DepListPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed, Chi if (!block_timing) { block_cycle0 = now; block_timing = true; - } else if (!watermark_synchronized && now - block_cycle0 >= PTO2_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { + } else if (!watermark_synchronized && now - block_cycle0 >= CHIP_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { publication_request.request(); } bool head_is_oldest_open_task = ring.slot_states != nullptr && diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 9bb7271c91..9d545df03f 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -52,10 +52,10 @@ #endif // Block notification interval (in spin counts) -#define PTO2_BLOCK_NOTIFY_INTERVAL 10000 +#define CHIP_BLOCK_NOTIFY_INTERVAL 10000 // Productive-loop publication is a liveness escape, not the normal pressure // path. Require sustained no-progress before bypassing K-batched publication. -#define PTO2_PUBLICATION_REQUEST_TIMEOUT_CYCLES (PLATFORM_PROF_SYS_CNT_FREQ / 100) // 10 ms +#define CHIP_PUBLICATION_REQUEST_TIMEOUT_CYCLES (PLATFORM_PROF_SYS_CNT_FREQ / 100) // 10 ms // Heap/task deadlock is detected structurally when the reclaim head is the // oldest task owned by an open scope on the blocked ring. This wall-clock value // is the backstop for all other cases; it is an ABSOLUTE TIME (not a spin @@ -254,7 +254,7 @@ class TaskAllocator { if (!block_timing) { block_cycle0 = now; block_timing = true; - } else if (!watermark_synchronized && now - block_cycle0 >= PTO2_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { + } else if (!watermark_synchronized && now - block_cycle0 >= CHIP_PUBLICATION_REQUEST_TIMEOUT_CYCLES) { publication_request.request(); } // (1) Structural, after publication acknowledgment: no open @@ -271,7 +271,7 @@ class TaskAllocator { report_deadlock(output_size, blocked_on_heap, /*scope_gated=*/false); return {-1, -1, nullptr, nullptr}; } - if (spin_count % PTO2_BLOCK_NOTIFY_INTERVAL == 0) { + if (spin_count % CHIP_BLOCK_NOTIFY_INTERVAL == 0) { LOG_WARN( "[TaskAllocator ring=%u] BLOCKED: tasks=%d/%d, heap_used=%" PRIu64 "/%" PRIu64 ", heap_available=%" PRIu64 ", heap_cursor=%" PRIu64 ", on=%s, spins=%d", diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp index 174e2879d7..546beb0473 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.cpp @@ -42,8 +42,7 @@ __attribute__((weak, visibility("hidden"))) uint64_t get_sys_cnt_aicpu() { retur // that define PLATFORM_PROF_SYS_CNT_FREQ locally, so pulling the platform header into // it caused a redefinition conflict (#1189). Scaling MS by the counter frequency (like // SCHEDULER_TIMEOUT_CYCLES) keeps the data-wait wall-clock identical across arches. -static constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_CYCLES = - (PTO2_TENSOR_DATA_TIMEOUT_MS * PLATFORM_PROF_SYS_CNT_FREQ) / 1000; +static constexpr uint64_t TENSOR_DATA_TIMEOUT_CYCLES = (TENSOR_DATA_TIMEOUT_MS * PLATFORM_PROF_SYS_CNT_FREQ) / 1000; // ============================================================================= // Orchestration Ops Table (function-pointer dispatch for orchestration .so) @@ -125,11 +124,11 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo failed = true; return; } - if (get_sys_cnt_aicpu() - t0 > PTO2_TENSOR_DATA_TIMEOUT_CYCLES) { + if (get_sys_cnt_aicpu() - t0 > TENSOR_DATA_TIMEOUT_CYCLES) { orch.report_fatal( SIMPLER_ERROR_TENSOR_WAIT_TIMEOUT, caller, "Timeout (%llu cycles): producer (ring=%d, local=%d) not completed", - (unsigned long long)PTO2_TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id + (unsigned long long)TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id ); failed = true; return; @@ -152,11 +151,11 @@ wait_for_tensor_ready(RuntimeContext *rt, const ChipTensor &tensor, bool wait_fo failed = true; return; } - if (get_sys_cnt_aicpu() - t0 > PTO2_TENSOR_DATA_TIMEOUT_CYCLES) { + if (get_sys_cnt_aicpu() - t0 > TENSOR_DATA_TIMEOUT_CYCLES) { orch.report_fatal( SIMPLER_ERROR_TENSOR_WAIT_TIMEOUT, caller, "Timeout (%llu cycles): consumers of producer (ring=%d, local=%d) not done", - (unsigned long long)PTO2_TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id + (unsigned long long)TENSOR_DATA_TIMEOUT_CYCLES, ring_id, local_id ); failed = true; return; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h index 2f643bd1b9..6f87a0c48d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_core.h @@ -52,9 +52,9 @@ * Runtime execution mode */ enum RuntimeMode { - PTO2_MODE_EXECUTE = 0, // Execute tasks on workers - PTO2_MODE_SIMULATE = 1, // Simulate task execution with cycle counting - PTO2_MODE_GRAPH_ONLY = 2 // Build graph only, no execution + MODE_EXECUTE = 0, // Execute tasks on workers + MODE_SIMULATE = 1, // Simulate task execution with cycle counting + MODE_GRAPH_ONLY = 2 // Build graph only, no execution }; /** @@ -310,8 +310,8 @@ void set_tensor_data( * Slim config struct exported by orchestration .so via aicpu_orchestration_config(). * Shared definition with orchestration_api.h (same layout, guarded). */ -#ifndef PTO2_ORCHESTRATION_CONFIG_DEFINED -#define PTO2_ORCHESTRATION_CONFIG_DEFINED +#ifndef ORCHESTRATION_CONFIG_DEFINED +#define ORCHESTRATION_CONFIG_DEFINED struct OrchestrationConfig { int expected_arg_count; }; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 6d8033c227..71ac588a28 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -59,17 +59,17 @@ // ============================================================================= // Task management -// NOTE: PTO2_TASK_WINDOW_SIZE is now a per-ring default value. +// NOTE: CHIP_TASK_WINDOW_SIZE is now a per-ring default value. // Actual window size is passed at runtime to runtime_reserve_layout(). // Use pto2_task_slot(sched, task_id) for slot calculation. -#define PTO2_TASK_WINDOW_SIZE 16384 // Default per-ring task window size (power of 2) +#define CHIP_TASK_WINDOW_SIZE 16384 // Default per-ring task window size (power of 2) // Multi-ring: number of independent ring layers (HeapRing + TaskRing + DepPool per layer) // Scope depth maps to ring index via: min(scope_depth, CHIP_MAX_RING_DEPTH - 1) #define CHIP_MAX_RING_DEPTH 4 // Memory pools (per-ring defaults; total = value × CHIP_MAX_RING_DEPTH) -#define PTO2_HEAP_SIZE (256 * 1024 * 1024) // 256MB per ring (1GB total) +#define CHIP_HEAP_SIZE (256 * 1024 * 1024) // 256MB per ring (1GB total) #define CHIP_DEP_LIST_POOL_SIZE 16384 // Per-ring dependency list pool entries #define CHIP_TENSORMAP_POOL_SIZE (65536) // TensorMap entry pool #define CHIP_TENSORMAP_NUM_BUCKETS 4096 // Power of 2 for fast hash (4096×8B=32KB fits L1) @@ -77,11 +77,11 @@ // Scope management #define CHIP_MAX_SCOPE_DEPTH 64 // Maximum nesting depth // Hard cap for the scope_tasks buffer. Equals the total in-flight ring slot -// budget (PTO2_TASK_WINDOW_SIZE × CHIP_MAX_RING_DEPTH): once every ring slot +// budget (CHIP_TASK_WINDOW_SIZE × CHIP_MAX_RING_DEPTH): once every ring slot // is in flight, no more tasks can ever be pushed regardless of buffer size. // scope_tasks_push fatals on overflow rather than growing the arena-owned // buffer (which would be UB on the arena's malloc'd backing). -#define CHIP_SCOPE_TASKS_CAP (PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) +#define CHIP_SCOPE_TASKS_CAP (CHIP_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH) // Ready queue #define CHIP_READY_QUEUE_SIZE 65536 // Per-shape queue size @@ -106,14 +106,14 @@ #define CHIP_DEP_POOL_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks // get_tensor_data/set_tensor_data spin-wait timeout, expressed in time. The cycle -// count (PTO2_TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp +// count (TENSOR_DATA_TIMEOUT_CYCLES) is derived from this in runtime_core.cpp // — its only user — by scaling with the platform counter frequency, like // SCHEDULER_TIMEOUT_CYCLES, so it reaps at the same wall-clock on every arch (a // fixed raw cycle count would be 15 s on a5 at 1 GHz but 300 s on a2a3 at 50 MHz). // PLATFORM_PROF_SYS_CNT_FREQ is deliberately NOT pulled into this header: it is // included by orchestrations that define that constant locally, so doing so caused // a redefinition conflict. See issue #1189. -constexpr uint64_t PTO2_TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s +constexpr uint64_t TENSOR_DATA_TIMEOUT_MS = 15000; // 15 s // ============================================================================= // Task States @@ -152,7 +152,7 @@ struct TaskAllocResult { bool failed() const { return task_id < 0; } }; -struct PTO2OutputLayout { +struct OutputLayout { uint64_t offsets[MAX_TENSOR_ARGS] = {}; uint64_t buffer_sizes[MAX_TENSOR_ARGS] = {}; int32_t total_output_size = 0; @@ -375,7 +375,7 @@ struct TaskPayload { * @param result Materialized output tensors (from TensorCreateInfo path) */ void - init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, PTO2OutputLayout &layout) { + init(const CoreTaskArgs &args, TaskOutputTensors &result, TaskAllocResult &alloc_result, OutputLayout &layout) { tensor_count = args.tensor_count(); scalar_count = args.scalar_count(); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index 3085111aea..bb93f1a4fc 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -33,7 +33,7 @@ // ============================================================================= namespace { -inline constexpr int32_t PTO2_DEFERRED_RELEASE_CAP = 256; +inline constexpr int32_t DEFERRED_RELEASE_CAP = 256; } // Pure function: read register result -> SlotTransition (no side effects). @@ -198,7 +198,7 @@ void SchedulerContext::complete_slot_task( } chip_swimlane.phase_complete_count++; #endif - if (deferred_release_count < PTO2_DEFERRED_RELEASE_CAP) { + if (deferred_release_count < DEFERRED_RELEASE_CAP) { deferred_release_slot_states[deferred_release_count++] = &slot_state; } else { while (deferred_release_count > 0) { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 5f59a2c610..43ca085a19 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -38,7 +38,7 @@ // ============================================================================= namespace { -inline constexpr int32_t PTO2_DEFERRED_RELEASE_CAP = 256; +inline constexpr int32_t DEFERRED_RELEASE_CAP = 256; } // AICore materializes args[] from src_payload on the gated path using these @@ -865,7 +865,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ chip_swimlane.chip_swimlane_enabled = (chip_swimlane_level_ != ChipSwimlaneLevel::DISABLED); #endif - ChipTaskSlotState *deferred_release_slot_states[PTO2_DEFERRED_RELEASE_CAP]; + ChipTaskSlotState *deferred_release_slot_states[DEFERRED_RELEASE_CAP]; int32_t deferred_release_count = 0; // PMU runs require single-issue dispatch — overlapping in-flight tasks @@ -1045,8 +1045,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ if (rt_ != nullptr && rt_->aicore_mailbox != nullptr && (sched_->async_wait_list.count > 0 || rt_->aicore_mailbox->has_pending())) { AsyncPollResult poll_result = sched_->async_wait_list.poll_and_complete( - rt_->aicore_mailbox, sched_, deferred_release_slot_states, deferred_release_count, - PTO2_DEFERRED_RELEASE_CAP + rt_->aicore_mailbox, sched_, deferred_release_slot_states, deferred_release_count, DEFERRED_RELEASE_CAP #if SIMPLER_SCHED_PROFILING , thread_idx @@ -1178,7 +1177,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ // beyond their own producers; release self-reference so the slot can // reach CONSUMED once all consumers drain. deferred_release_slot_states[deferred_release_count++] = &dummy_slot; - if (deferred_release_count >= PTO2_DEFERRED_RELEASE_CAP) { + if (deferred_release_count >= DEFERRED_RELEASE_CAP) { while (deferred_release_count > 0) { #if SIMPLER_SCHED_PROFILING (void)sched_->on_task_release( diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index d1d01d984b..132e298a2d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -95,7 +95,7 @@ bool SharedMemoryHandle::init_per_ring( } SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &arena) { - const uint64_t buffer_size = calculate_size(PTO2_TASK_WINDOW_SIZE); + const uint64_t buffer_size = calculate_size(CHIP_TASK_WINDOW_SIZE); const size_t off_handle = arena.reserve(sizeof(SharedMemoryHandle), alignof(SharedMemoryHandle)); const size_t off_buffer = arena.reserve(static_cast(buffer_size), CHIP_ALIGN_SIZE); if (arena.commit() == nullptr) return nullptr; @@ -104,7 +104,7 @@ SharedMemoryHandle *SharedMemoryHandle::create_and_init_default(DeviceArena &are memset(handle, 0, sizeof(*handle)); void *buffer = arena.region_ptr(off_buffer); memset(buffer, 0, static_cast(buffer_size)); - if (!handle->init(buffer, buffer_size, PTO2_TASK_WINDOW_SIZE, PTO2_HEAP_SIZE)) return nullptr; + if (!handle->init(buffer, buffer_size, CHIP_TASK_WINDOW_SIZE, CHIP_HEAP_SIZE)) return nullptr; return handle; } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h index 4cb9d8387c..2bf29330e9 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared_memory.h @@ -193,7 +193,7 @@ struct SharedMemoryHandle { static uint64_t calculate_size_per_ring(const uint64_t task_window_sizes[CHIP_MAX_RING_DEPTH]); // UT convenience: reserve wrapper + sm_base on `arena`, commit, and init - // using default PTO2_TASK_WINDOW_SIZE / PTO2_HEAP_SIZE. Only valid when the + // using default CHIP_TASK_WINDOW_SIZE / CHIP_HEAP_SIZE. Only valid when the // arena is otherwise empty (the call performs the single commit). All // memory is owned by the arena — caller must not call destroy(). static SharedMemoryHandle *create_and_init_default(DeviceArena &arena); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h index df671e0f88..30fb4a14d7 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/submit_types.h @@ -254,9 +254,9 @@ struct MixedKernels { * is expanded into at dispatch time. Each block receives a unique * block_idx in [0, core_num) via the per-dispatch LocalContext. */ -class PTO2LaunchSpec { +class LaunchSpec { public: - constexpr PTO2LaunchSpec() = default; + constexpr LaunchSpec() = default; int16_t core_num() const { return core_num_; } void set_core_num(int16_t n) { core_num_ = n; } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h index 7faae66a16..89183308c6 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/types.h @@ -281,7 +281,7 @@ struct Arg : TaskArgsTpl { bool has_error{false}; const char *error_msg{nullptr}; - PTO2LaunchSpec launch_spec; // SPMD launch parameters (block_num, etc.) + LaunchSpec launch_spec; // SPMD launch parameters (block_num, etc.) // Early-dispatch hint (codegen-author set, off by default). When // true, the scheduler may stage this task on an idle core before its producer diff --git a/src/common/runtime_status/error_names.h b/src/common/runtime_status/error_names.h index 40382e54de..765e63fb16 100644 --- a/src/common/runtime_status/error_names.h +++ b/src/common/runtime_status/error_names.h @@ -153,7 +153,7 @@ static inline const char *error_hint(int32_t code) { "(total AIV count, or total cluster count for MIX/AIC), or drop the sync-start request"; case SIMPLER_ERROR_TENSOR_WAIT_TIMEOUT: return "find the producing kernel and check it for a hang (see SCHEDULER_TIMEOUT sub_class S1); " - "verify the consumer declares the dependency and exits; raise PTO2_TENSOR_DATA_TIMEOUT_MS " + "verify the consumer declares the dependency and exits; raise TENSOR_DATA_TIMEOUT_MS " "to tell a slow kernel apart from a stuck one"; case SIMPLER_ERROR_EXPLICIT_ORCH_FATAL: return "self-inflicted -- follow the message passed to the rt_report_fatal() call site"; diff --git a/tests/st/a2a3/host_build_graph/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp b/tests/st/a2a3/host_build_graph/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp index 141677fd9a..c3a4743832 100644 --- a/tests/st/a2a3/host_build_graph/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp +++ b/tests/st/a2a3/host_build_graph/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp @@ -31,7 +31,7 @@ #define FUNC_SPMD_MIX_AIV0 1 #define FUNC_SPMD_MIX_AIV1 2 -// PTO2LaunchSpec spells the SPMD block-count setter differently per arch +// LaunchSpec spells the SPMD block-count setter differently per arch // (a5: set_core_num, a2a3: set_block_num) for the same field. Bridge it so this // one fixture compiles on both; keyed off the arch's types.h include guard, // which orchestration_api.h pulls in transitively. @@ -76,7 +76,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip // straight through. Giving it one would hang host_build_graph: its // orchestrator runs to completion on the host before the device executes // anything, so a producer's task_state can never reach COMPLETED and - // wait_for_tensor_ready would spin to PTO2_TENSOR_DATA_TIMEOUT_CYCLES. + // wait_for_tensor_ready would spin to TENSOR_DATA_TIMEOUT_CYCLES. uint32_t idx[1] = {0}; set_tensor_data(shape, 1, idx, cluster_count); idx[0] = 1; diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp b/tests/st/a2a3/tensormap_and_ringbuffer/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp index 141677fd9a..c3a4743832 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp +++ b/tests/st/a2a3/tensormap_and_ringbuffer/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp @@ -31,7 +31,7 @@ #define FUNC_SPMD_MIX_AIV0 1 #define FUNC_SPMD_MIX_AIV1 2 -// PTO2LaunchSpec spells the SPMD block-count setter differently per arch +// LaunchSpec spells the SPMD block-count setter differently per arch // (a5: set_core_num, a2a3: set_block_num) for the same field. Bridge it so this // one fixture compiles on both; keyed off the arch's types.h include guard, // which orchestration_api.h pulls in transitively. @@ -76,7 +76,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip // straight through. Giving it one would hang host_build_graph: its // orchestrator runs to completion on the host before the device executes // anything, so a producer's task_state can never reach COMPLETED and - // wait_for_tensor_ready would spin to PTO2_TENSOR_DATA_TIMEOUT_CYCLES. + // wait_for_tensor_ready would spin to TENSOR_DATA_TIMEOUT_CYCLES. uint32_t idx[1] = {0}; set_tensor_data(shape, 1, idx, cluster_count); idx[0] = 1; diff --git a/tests/st/a5/tensormap_and_ringbuffer/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp b/tests/st/a5/tensormap_and_ringbuffer/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp index 141677fd9a..c3a4743832 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp +++ b/tests/st/a5/tensormap_and_ringbuffer/available_aicore_counts/kernels/orchestration/available_aicore_counts_orch.cpp @@ -31,7 +31,7 @@ #define FUNC_SPMD_MIX_AIV0 1 #define FUNC_SPMD_MIX_AIV1 2 -// PTO2LaunchSpec spells the SPMD block-count setter differently per arch +// LaunchSpec spells the SPMD block-count setter differently per arch // (a5: set_core_num, a2a3: set_block_num) for the same field. Bridge it so this // one fixture compiles on both; keyed off the arch's types.h include guard, // which orchestration_api.h pulls in transitively. @@ -76,7 +76,7 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const Chip // straight through. Giving it one would hang host_build_graph: its // orchestrator runs to completion on the host before the device executes // anything, so a producer's task_state can never reach COMPLETED and - // wait_for_tensor_ready would spin to PTO2_TENSOR_DATA_TIMEOUT_CYCLES. + // wait_for_tensor_ready would spin to TENSOR_DATA_TIMEOUT_CYCLES. uint32_t idx[1] = {0}; set_tensor_data(shape, 1, idx, cluster_count); idx[0] = 1; diff --git a/tests/st/runtime_fatal_codes/kernels/orchestration/require_sync_start_orch.cpp b/tests/st/runtime_fatal_codes/kernels/orchestration/require_sync_start_orch.cpp index f3b23a18f3..c1912f64f8 100644 --- a/tests/st/runtime_fatal_codes/kernels/orchestration/require_sync_start_orch.cpp +++ b/tests/st/runtime_fatal_codes/kernels/orchestration/require_sync_start_orch.cpp @@ -26,7 +26,7 @@ #define FUNC_NOOP_KERNEL 0 -// PTO2LaunchSpec spells the SPMD block-count setter differently per arch +// LaunchSpec spells the SPMD block-count setter differently per arch // (a5: set_core_num, a2a3: set_block_num) for the same field. Bridge it so this // one fixture compiles on both; keyed off the arch's types.h include guard, // which orchestration_api.h pulls in transitively. diff --git a/tests/st/runtime_fatal_codes/kernels/orchestration/tensor_wait_timeout_orch.cpp b/tests/st/runtime_fatal_codes/kernels/orchestration/tensor_wait_timeout_orch.cpp index 75715623e1..89a4a8d440 100644 --- a/tests/st/runtime_fatal_codes/kernels/orchestration/tensor_wait_timeout_orch.cpp +++ b/tests/st/runtime_fatal_codes/kernels/orchestration/tensor_wait_timeout_orch.cpp @@ -15,7 +15,7 @@ * Submits an AIC kernel that spins forever, then reads its output with * get_tensor_data. Because the output has a producer in the TensorMap, * get_tensor_data spin-waits for that producer to complete — which never - * happens — so after PTO2_TENSOR_DATA_TIMEOUT_CYCLES (15e9 cycles == 15 s at the + * happens — so after TENSOR_DATA_TIMEOUT_CYCLES (15e9 cycles == 15 s at the * 1 GHz AICPU counter) the orchestrator latches TENSOR_WAIT_TIMEOUT. * * Onboard only (the hang kernel would spin the simulator forever). The test diff --git a/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py b/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py index e0ccef3261..376bc21df1 100644 --- a/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py +++ b/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py @@ -129,7 +129,7 @@ kernel_core="aic", onboard_only=True, # a while(true) kernel would hang the simulator # The data-wait timeout is 15 s on both arches now that it is frequency- - # scaled (PTO2_TENSOR_DATA_TIMEOUT_MS, #1189) -- before that it was 15 s on + # scaled (TENSOR_DATA_TIMEOUT_MS, #1189) -- before that it was 15 s on # a5 but 300 s on a2a3, so this case used to be a5-only. Raise every other # watchdog above 15 s so the tensor-data wait wins the race and latches # code 8 before they reap the hung core. diff --git a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp index e0557bbe94..165d600af8 100644 --- a/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a2a3/test_hbg_submit_poison.cpp @@ -65,7 +65,7 @@ class HbgSubmitPoisonTest : public ::testing::Test { // Same order the AICPU boots in: the slot arrays are not part of the // uploaded image, so nothing can push until they carry their ramp. sched.seed_queue_slots(); - ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), 4096, PTO2_TASK_WINDOW_SIZE, &sched)); + ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), 4096, CHIP_TASK_WINDOW_SIZE, &sched)); } void TearDown() override { diff --git a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp index c3223620ce..cfdb3543a6 100644 --- a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp @@ -37,7 +37,7 @@ class OrchestratorFaninTest : public ::testing::Test { int32_t task_window_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - task_window_sizes[r] = static_cast(PTO2_TASK_WINDOW_SIZE); + task_window_sizes[r] = static_cast(CHIP_TASK_WINDOW_SIZE); } orch_layout = OrchestratorState::reserve_layout(runtime_arena, task_window_sizes); @@ -45,7 +45,7 @@ class OrchestratorFaninTest : public ::testing::Test { ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(orch.init_data_from_layout( - orch_layout, runtime_arena, sm_handle->sm_base, gm_heap.data(), 4096, PTO2_TASK_WINDOW_SIZE + orch_layout, runtime_arena, sm_handle->sm_base, gm_heap.data(), 4096, CHIP_TASK_WINDOW_SIZE )); ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); sched.wire_arena_pointers(sched_layout, runtime_arena); @@ -353,8 +353,8 @@ TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { // Default window: cap == the old compile-time value (no behavior change). for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) - windows[r] = PTO2_TASK_WINDOW_SIZE; - EXPECT_EQ(cap_for(windows), PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH); + windows[r] = CHIP_TASK_WINDOW_SIZE; + EXPECT_EQ(cap_for(windows), CHIP_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH); EXPECT_EQ(cap_for(windows), CHIP_SCOPE_TASKS_CAP); // Shrunk window: cap shrinks to the real budget (no over-allocation). @@ -364,7 +364,7 @@ TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { // Enlarged window past the compile default: cap grows to match the rings, so a // large scope no longer hits a premature SCOPE_TASKS_OVERFLOW (the bug fixed). - const int32_t big = PTO2_TASK_WINDOW_SIZE * 2; + const int32_t big = CHIP_TASK_WINDOW_SIZE * 2; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) windows[r] = big; EXPECT_EQ(cap_for(windows), big * CHIP_MAX_RING_DEPTH); diff --git a/tests/ut/cpp/a2a3/test_scheduler_state.cpp b/tests/ut/cpp/a2a3/test_scheduler_state.cpp index 98403bf1d2..ecb5e80c23 100644 --- a/tests/ut/cpp/a2a3/test_scheduler_state.cpp +++ b/tests/ut/cpp/a2a3/test_scheduler_state.cpp @@ -254,7 +254,7 @@ TEST_F(SchedulerStateTest, ContendedConsumedHeadSetsPendingAndIdleDrainAdvances) } TEST_F(SchedulerStateTest, ContendedConsumedHeadIdleDrainStress) { - const int32_t head_task_ids[] = {0, 1, 127, PTO2_TASK_WINDOW_SIZE - 2, PTO2_TASK_WINDOW_SIZE + 3}; + const int32_t head_task_ids[] = {0, 1, 127, CHIP_TASK_WINDOW_SIZE - 2, CHIP_TASK_WINDOW_SIZE + 3}; for (int32_t ring_id = 0; ring_id < CHIP_MAX_RING_DEPTH; ring_id++) { for (int32_t head_task_id : head_task_ids) { diff --git a/tests/ut/cpp/a2a3/test_shared_memory.cpp b/tests/ut/cpp/a2a3/test_shared_memory.cpp index f94a35f03d..694b5a5189 100644 --- a/tests/ut/cpp/a2a3/test_shared_memory.cpp +++ b/tests/ut/cpp/a2a3/test_shared_memory.cpp @@ -188,7 +188,7 @@ TEST(SharedMemoryLayout, RegionsNonOverlapping) { // ============================================================================= TEST(SharedMemoryCalcSize, NonZero) { - uint64_t size = SharedMemoryHandle::calculate_size(PTO2_TASK_WINDOW_SIZE); + uint64_t size = SharedMemoryHandle::calculate_size(CHIP_TASK_WINDOW_SIZE); EXPECT_GT(size, 0u); } @@ -253,7 +253,7 @@ TEST(RuntimeArenaLayout, PerRingConfigInitializesRuntimeComponents) { std::vector gm(static_cast(total_heap)); RuntimeContext *rt = - runtime_init_data_from_layout(runtime_arena, layout, PTO2_MODE_EXECUTE, sm, sm_size, gm.data(), heaps); + runtime_init_data_from_layout(runtime_arena, layout, MODE_EXECUTE, sm, sm_size, gm.data(), heaps); ASSERT_NE(rt, nullptr); runtime_wire_arena_pointers(runtime_arena, layout, rt); @@ -280,7 +280,7 @@ TEST(RuntimeArenaLayout, RejectsOverflowingPerRingHeapSum) { char sm = 0; char gm = 0; - EXPECT_EQ(runtime_init_data_from_layout(runtime_arena, layout, PTO2_MODE_EXECUTE, &sm, 0, &gm, heaps), nullptr); + EXPECT_EQ(runtime_init_data_from_layout(runtime_arena, layout, MODE_EXECUTE, &sm, 0, &gm, heaps), nullptr); OrchestratorState orch{}; EXPECT_FALSE(orch.init_data_from_layout(layout.offsets.orch, runtime_arena, &sm, &gm, heaps, ws)); diff --git a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp index e0557bbe94..165d600af8 100644 --- a/tests/ut/cpp/a5/test_hbg_submit_poison.cpp +++ b/tests/ut/cpp/a5/test_hbg_submit_poison.cpp @@ -65,7 +65,7 @@ class HbgSubmitPoisonTest : public ::testing::Test { // Same order the AICPU boots in: the slot arrays are not part of the // uploaded image, so nothing can push until they carry their ramp. sched.seed_queue_slots(); - ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), 4096, PTO2_TASK_WINDOW_SIZE, &sched)); + ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), 4096, CHIP_TASK_WINDOW_SIZE, &sched)); } void TearDown() override { diff --git a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp index c6bd7810cc..0f94bf3ebe 100644 --- a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp @@ -37,7 +37,7 @@ class OrchestratorFaninTest : public ::testing::Test { int32_t task_window_sizes[CHIP_MAX_RING_DEPTH]; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - task_window_sizes[r] = static_cast(PTO2_TASK_WINDOW_SIZE); + task_window_sizes[r] = static_cast(CHIP_TASK_WINDOW_SIZE); } orch_layout = OrchestratorState::reserve_layout(runtime_arena, task_window_sizes); @@ -45,7 +45,7 @@ class OrchestratorFaninTest : public ::testing::Test { ASSERT_NE(runtime_arena.commit(), nullptr); ASSERT_TRUE(orch.init_data_from_layout( - orch_layout, runtime_arena, sm_handle->sm_base, gm_heap.data(), 4096, PTO2_TASK_WINDOW_SIZE + orch_layout, runtime_arena, sm_handle->sm_base, gm_heap.data(), 4096, CHIP_TASK_WINDOW_SIZE )); ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); sched.wire_arena_pointers(sched_layout, runtime_arena); @@ -367,8 +367,8 @@ TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { // Default window: cap == the old compile-time value (no behavior change). for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) - windows[r] = PTO2_TASK_WINDOW_SIZE; - EXPECT_EQ(cap_for(windows), PTO2_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH); + windows[r] = CHIP_TASK_WINDOW_SIZE; + EXPECT_EQ(cap_for(windows), CHIP_TASK_WINDOW_SIZE * CHIP_MAX_RING_DEPTH); EXPECT_EQ(cap_for(windows), CHIP_SCOPE_TASKS_CAP); // Shrunk window: cap shrinks to the real budget (no over-allocation). @@ -378,7 +378,7 @@ TEST(OrchestratorLayoutScopeTasksCap, FollowsRuntimeWindowSum) { // Enlarged window past the compile default: cap grows to match the rings, so a // large scope no longer hits a premature SCOPE_TASKS_OVERFLOW (the bug fixed). - const int32_t big = PTO2_TASK_WINDOW_SIZE * 2; + const int32_t big = CHIP_TASK_WINDOW_SIZE * 2; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) windows[r] = big; EXPECT_EQ(cap_for(windows), big * CHIP_MAX_RING_DEPTH); diff --git a/tests/ut/cpp/a5/test_scheduler_state.cpp b/tests/ut/cpp/a5/test_scheduler_state.cpp index b9fac69df4..2cb54304d0 100644 --- a/tests/ut/cpp/a5/test_scheduler_state.cpp +++ b/tests/ut/cpp/a5/test_scheduler_state.cpp @@ -307,7 +307,7 @@ TEST_F(SchedulerStateTest, PublicationRequestDoesNotConsumeDeferredAdvance) { } TEST_F(SchedulerStateTest, ContendedConsumedHeadIdleDrainStress) { - const int32_t head_task_ids[] = {0, 1, 127, PTO2_TASK_WINDOW_SIZE - 2, PTO2_TASK_WINDOW_SIZE + 3}; + const int32_t head_task_ids[] = {0, 1, 127, CHIP_TASK_WINDOW_SIZE - 2, CHIP_TASK_WINDOW_SIZE + 3}; for (int32_t ring_id = 0; ring_id < CHIP_MAX_RING_DEPTH; ring_id++) { for (int32_t head_task_id : head_task_ids) { diff --git a/tests/ut/cpp/a5/test_shared_memory.cpp b/tests/ut/cpp/a5/test_shared_memory.cpp index 7692aca926..4b4922d008 100644 --- a/tests/ut/cpp/a5/test_shared_memory.cpp +++ b/tests/ut/cpp/a5/test_shared_memory.cpp @@ -188,7 +188,7 @@ TEST(SharedMemoryLayout, RegionsNonOverlapping) { // ============================================================================= TEST(SharedMemoryCalcSize, NonZero) { - uint64_t size = SharedMemoryHandle::calculate_size(PTO2_TASK_WINDOW_SIZE); + uint64_t size = SharedMemoryHandle::calculate_size(CHIP_TASK_WINDOW_SIZE); EXPECT_GT(size, 0u); } @@ -253,7 +253,7 @@ TEST(RuntimeArenaLayout, PerRingConfigInitializesRuntimeComponents) { std::vector gm(static_cast(total_heap)); RuntimeContext *rt = - runtime_init_data_from_layout(runtime_arena, layout, PTO2_MODE_EXECUTE, sm, sm_size, gm.data(), heaps); + runtime_init_data_from_layout(runtime_arena, layout, MODE_EXECUTE, sm, sm_size, gm.data(), heaps); ASSERT_NE(rt, nullptr); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { EXPECT_FALSE(rt->scheduler.ring_sched_states[r].publication_batching_enabled); @@ -292,7 +292,7 @@ TEST(RuntimeArenaLayout, RewiresReclaimPublicationPointersAfterRelocation) { std::vector sm(static_cast(sm_size)); std::vector gm(100 * 1024); RuntimeContext *source_rt = - runtime_init_data_from_layout(source_arena, layout, PTO2_MODE_EXECUTE, sm.data(), sm_size, gm.data(), heaps); + runtime_init_data_from_layout(source_arena, layout, MODE_EXECUTE, sm.data(), sm_size, gm.data(), heaps); ASSERT_NE(source_rt, nullptr); runtime_wire_arena_pointers(source_arena, layout, source_rt); @@ -327,7 +327,7 @@ TEST(RuntimeArenaLayout, RejectsOverflowingPerRingHeapSum) { char sm = 0; char gm = 0; - EXPECT_EQ(runtime_init_data_from_layout(runtime_arena, layout, PTO2_MODE_EXECUTE, &sm, 0, &gm, heaps), nullptr); + EXPECT_EQ(runtime_init_data_from_layout(runtime_arena, layout, MODE_EXECUTE, &sm, 0, &gm, heaps), nullptr); OrchestratorState orch{}; EXPECT_FALSE(orch.init_data_from_layout(layout.offsets.orch, runtime_arena, &sm, &gm, heaps, ws)); diff --git a/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp b/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp index 90b1dee9ac..26c3df91ff 100644 --- a/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_definition_arena.cpp @@ -62,7 +62,7 @@ class HbgGraphDefinitionArenaTest : public ::testing::Test { ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); sched.wire_arena_pointers(sched_layout, runtime_arena); - ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), HEAP_BYTES, PTO2_TASK_WINDOW_SIZE, &sched)); + ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), HEAP_BYTES, CHIP_TASK_WINDOW_SIZE, &sched)); } void TearDown() override { diff --git a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp index e871ad9006..2578bc71a8 100644 --- a/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_submit_failure.cpp @@ -71,7 +71,7 @@ class HbgGraphSubmitFailureTest : public ::testing::Test { ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); sched.wire_arena_pointers(sched_layout, runtime_arena); - ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), HEAP_BYTES, PTO2_TASK_WINDOW_SIZE, &sched)); + ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), HEAP_BYTES, CHIP_TASK_WINDOW_SIZE, &sched)); definition_staging.assign(STAGING_BYTES, std::byte{0}); arena.base = definition_staging.data(); @@ -893,7 +893,7 @@ TEST_F(HbgGraphSubmitFailureTest, AHiddenAllocTaskLeavesItsDispatchPredicateDefi ASSERT_TRUE(outputs.task_id().is_valid()); ASSERT_FALSE(orch.fatal); - const uint64_t slot = outputs.task_id().local() & (PTO2_TASK_WINDOW_SIZE - 1); + const uint64_t slot = outputs.task_id().local() & (CHIP_TASK_WINDOW_SIZE - 1); ASSERT_LT(slot, static_cast(kPoisonedSlots)) << "the submitted slot must be one this test poisoned"; EXPECT_EQ(payloads[slot].predicate.op, PredicateOp::NONE); EXPECT_EQ(payloads[slot].predicate.addr, 0u); diff --git a/tests/ut/cpp/common/test_hbg_slot_claim.cpp b/tests/ut/cpp/common/test_hbg_slot_claim.cpp index 61f78e5679..6508bc3688 100644 --- a/tests/ut/cpp/common/test_hbg_slot_claim.cpp +++ b/tests/ut/cpp/common/test_hbg_slot_claim.cpp @@ -55,7 +55,7 @@ class HbgSlotClaimTest : public ::testing::Test { ASSERT_TRUE(sched.init_data_from_layout(sched_layout, runtime_arena, sm_handle->sm_base)); sched.wire_arena_pointers(sched_layout, runtime_arena); - ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), HEAP_BYTES, PTO2_TASK_WINDOW_SIZE, &sched)); + ASSERT_TRUE(orch.init(sm_handle->sm_base, gm_heap.data(), HEAP_BYTES, CHIP_TASK_WINDOW_SIZE, &sched)); definition_staging.assign(STAGING_BYTES, std::byte{0}); GraphDefinitionArena arena{}; diff --git a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp index e1a98989e0..205606587e 100644 --- a/tests/ut/cpp/common/test_hbg_sm_compaction.cpp +++ b/tests/ut/cpp/common/test_hbg_sm_compaction.cpp @@ -404,15 +404,15 @@ TEST(HbgSmCompaction, LayoutStaysWithinDeltaReach) { EXPECT_LT(compacted.bytes, static_cast(INT32_MAX)); constexpr uint64_t REACH = static_cast(INT32_MAX); - EXPECT_LE(sm_layout::ring_segment_offsets(PTO2_TASK_WINDOW_SIZE).end, REACH); + EXPECT_LE(sm_layout::ring_segment_offsets(CHIP_TASK_WINDOW_SIZE).end, REACH); // The bound is a property of the layout, not of the capacity alone: there is a // capacity past which the mirror no longer fits, and it is above the default. - uint64_t window = PTO2_TASK_WINDOW_SIZE; + uint64_t window = CHIP_TASK_WINDOW_SIZE; while (window <= (UINT64_MAX / 2) && sm_layout::ring_segment_offsets(window).end <= REACH) { window *= 2; } - EXPECT_GT(window, static_cast(PTO2_TASK_WINDOW_SIZE)); + EXPECT_GT(window, static_cast(CHIP_TASK_WINDOW_SIZE)); EXPECT_GT(sm_layout::ring_segment_offsets(window).end, REACH); } From 58b6a6fb285a5883991b775ecbdc4d4a481bbb73 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:32:34 -0700 Subject: [PATCH 18/22] Fix: point two diagnostics at the environment variables that exist Both named a `PTO2_*` variable that nothing reads, and both failed silently because an unrecognised environment variable is simply ignored. `tools/benchmark_rounds.sh --serial-orch-sched` set `PTO2_SERIAL_ORCH_SCHED=1` for its second arm. The runtime reads `SIMPLER_TMR_SERIAL_ORCH_SCHED_ENABLE` (`tensormap_and_ringbuffer/host/runtime_maker.cpp`), so the serial arm ran with serial orch->sched still off: the script measured the parallel path twice and reported the pair as a serial-versus-parallel comparison. Any measurement taken through that flag is void. `error_hint(SIMPLER_ERROR_SCHEDULER_TIMEOUT)` told the user to raise `PTO2_SCHEDULER_TIMEOUT_MS`. The knob is `SIMPLER_SCHEDULER_TIMEOUT_MS` (`common/platform/include/host/runtime_timeout_config.h`), which CI already sets by that name, so following the hint on a scheduler-timeout fatal changed nothing and made a slow kernel look like a real deadlock. Found while sweeping the `PTO2` prefix out of the runtime: both survived only because they are strings, so no compiler ever checked them. Co-Authored-By: Claude Opus 5 (1M context) --- src/common/runtime_status/error_names.h | 2 +- tools/benchmark_rounds.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/runtime_status/error_names.h b/src/common/runtime_status/error_names.h index 765e63fb16..f2cf4ed4f1 100644 --- a/src/common/runtime_status/error_names.h +++ b/src/common/runtime_status/error_names.h @@ -166,7 +166,7 @@ static inline const char *error_hint(int32_t code) { "or shrink the graph; in a reclaiming runtime also inspect entry-reclaim progress"; case SIMPLER_ERROR_SCHEDULER_TIMEOUT: return "read the sub_class= line below first, then follow the S1-S5 table in the doc; raise " - "PTO2_SCHEDULER_TIMEOUT_MS to tell a true deadlock apart from a merely slow kernel"; + "SIMPLER_SCHEDULER_TIMEOUT_MS to tell a true deadlock apart from a merely slow kernel"; case SIMPLER_ERROR_ASYNC_COMPLETION_INVALID: return "recheck the register_completion_condition() arguments in the kernel, in particular the " "completion type and the counter address"; diff --git a/tools/benchmark_rounds.sh b/tools/benchmark_rounds.sh index 7d1f19756e..6987dba50f 100755 --- a/tools/benchmark_rounds.sh +++ b/tools/benchmark_rounds.sh @@ -130,7 +130,7 @@ Options: -v, --verbose Save detailed test_*.py output to a timestamped log file --serial-orch-sched Run each TMR case twice: default parallel mode, then serial - orch->sched mode with PTO2_SERIAL_ORCH_SCHED=1. + orch->sched mode with SIMPLER_TMR_SERIAL_ORCH_SCHED_ENABLE=1. -h, --help Show this help All other options are passed through to the underlying `python test_*.py` @@ -290,8 +290,8 @@ run_bench() { vlog "Running: ${run_cmd[*]}" local rc=0 if [[ "$mode" == "serial" ]]; then - vlog "Environment: PTO2_SERIAL_ORCH_SCHED=1" - PTO2_SERIAL_ORCH_SCHED=1 "${run_cmd[@]}" > "$fw_stdout_file" 2>&1 || rc=$? + vlog "Environment: SIMPLER_TMR_SERIAL_ORCH_SCHED_ENABLE=1" + SIMPLER_TMR_SERIAL_ORCH_SCHED_ENABLE=1 "${run_cmd[@]}" > "$fw_stdout_file" 2>&1 || rc=$? else "${run_cmd[@]}" > "$fw_stdout_file" 2>&1 || rc=$? fi From 3c2aeb659409fa2b5fad6920d1de5f39b5d72ac6 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:52:17 -0700 Subject: [PATCH 19/22] Refactor: retire the PTO2 brand from comments, logs and docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last of the prefix, and the only tier of it that was never an identifier: the brand as a word. `PTO2 shared memory` becomes `shared memory`, `PTO2 integration:` becomes what that map is for, `Extension to the PTO2 runtime` names `tensormap_and_ringbuffer`, and the doc titles say what they document. Log text moves with the docs that quote it, so the documented string keeps matching the emitted one: `progress: completed=`, `total submitted tasks`, `completed tasks`, `dispatch starting with`, `scheduler timeout sub_class=`, `=== Shared Memory Layout ===` and two failure messages, together with every mention of them in `profiling_levels.md` and `device_log_profiling.md`. Four file-local identifiers go too: `read_pto2_runtime_status`, `normalize_pto2_task_id_int`, and the locals `pto2_sm` and `pto2_count` — the last renamed to `window_tasks`, which is what it counts. Three references named things that do not exist, which is how a stale comment becomes a lie rather than just noise: - `pto2_task_slot(sched, task_id)`, cited by `runtime_types.h` and `runtime_init.cpp` as the way to compute a slot. There is no such function; the slot is `local_id` masked by the window size, and that is what the comment now says — including why the window has to be a power of two. - `pto2_run_runtime`, which `docs/task-flow.md` names five times as the L2 ABI edge. The entry point is `simpler_run`. - `pto2_init_*` flags, which `docs/dynamic-linking.md` lists among the fields `SchedulerContext::deinit()` resets. It resets no such flag, and none exists under any spelling, so the clause is gone. `docs/dfx/profiling-config-naming.md` claimed `PTO2_` was "the device runtime's internal namespace" and then illustrated it with `CHIP_MAX_RING_DEPTH`, `SIMPLER_ERROR_*` and `RuntimeContext` — three names that had already left it. It now states the convention that actually holds, including why an object-like macro takes `CHIP_` where a `constexpr` does not, and records why `PTO2_RING_TASK_WINDOW` / `PTO2_RING_HEAP` / `PTO2_RING_DEP_POOL` keep the legacy spelling: they are read from the environment by name, so a rename is ignored silently instead of failing to compile, and retiring them needs a cross-repo migration. `.claude/rules/codestyle.md` is untouched. Its `PTO2` mentions are the rule that bans the prefix. Co-Authored-By: Claude Opus 5 (1M context) --- docs/dfx/profiling-config-naming.md | 18 ++++++++--- docs/dynamic-linking.md | 2 +- docs/orchestrator.md | 2 +- docs/task-flow.md | 10 +++---- docs/troubleshooting/device-error-codes.md | 2 +- .../tools/sched_overhead_analysis.py | 4 +-- simpler_setup/tools/swimlane_converter.py | 30 +++++++++---------- src/a2a3/docs/runtimes.md | 2 +- .../aicore/aicore_executor.cpp | 2 +- .../host_build_graph/aicpu/aicpu_executor.cpp | 4 +-- .../host_build_graph/docs/profiling_levels.md | 10 +++---- .../host_build_graph/host/runtime_maker.cpp | 12 ++++---- .../host_build_graph/runtime/runtime.h | 6 ++-- .../host_build_graph/runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler_cold_path.cpp | 11 ++++--- .../runtime/scheduler/scheduler_context.h | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 8 ++--- .../runtime/shared/runtime_init.cpp | 2 +- .../runtime/shared/shared_memory.cpp | 2 +- .../aicore/aicore_executor.cpp | 2 +- .../aicpu/aicpu_executor.cpp | 4 +-- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 2 +- .../docs/SUBMIT_BY_CLUSTER.md | 4 +-- .../docs/device_log_profiling.md | 8 ++--- .../docs/profiling_levels.md | 14 ++++----- .../host/runtime_maker.cpp | 16 +++++----- .../runtime/runtime.h | 6 ++-- .../runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler_cold_path.cpp | 9 +++--- .../runtime/scheduler/scheduler_context.h | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 6 ++-- .../runtime/shared/shared_memory.cpp | 2 +- src/a5/docs/runtimes.md | 2 +- .../aicore/aicore_executor.cpp | 2 +- .../host_build_graph/aicpu/aicpu_executor.cpp | 4 +-- .../host_build_graph/docs/profiling_levels.md | 10 +++---- .../host_build_graph/host/runtime_maker.cpp | 12 ++++---- .../host_build_graph/runtime/runtime.h | 6 ++-- .../host_build_graph/runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler_cold_path.cpp | 11 ++++--- .../scheduler/scheduler_completion.cpp | 2 +- .../runtime/scheduler/scheduler_context.h | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 8 ++--- .../runtime/shared/runtime_init.cpp | 2 +- .../runtime/shared/shared_memory.cpp | 2 +- .../aicore/aicore_executor.cpp | 2 +- .../aicpu/aicpu_executor.cpp | 4 +-- .../docs/MULTI_RING.md | 2 +- .../docs/RUNTIME_LOGIC.md | 2 +- .../docs/SUBMIT_BY_CLUSTER.md | 4 +-- .../docs/device_log_profiling.md | 8 ++--- .../docs/profiling_levels.md | 14 ++++----- .../host/runtime_maker.cpp | 16 +++++----- .../runtime/runtime.h | 6 ++-- .../runtime/runtime_types.h | 2 +- .../runtime/scheduler/scheduler_cold_path.cpp | 3 +- .../scheduler/scheduler_completion.cpp | 4 +-- .../runtime/scheduler/scheduler_context.h | 2 +- .../runtime/scheduler/scheduler_dispatch.cpp | 6 ++-- .../runtime/shared/shared_memory.cpp | 2 +- src/common/hierarchical/orchestrator.h | 2 +- .../aicpu/chip_swimlane_collector_aicpu.h | 6 ++-- .../platform/include/common/args_dump.h | 2 +- .../include/common/chip_swimlane_profiling.h | 6 ++-- src/common/platform/include/common/dep_gen.h | 2 +- .../onboard/host/device_runner_base.cpp | 6 ++-- .../onboard/host/device_runner_base.h | 6 ++-- .../shared/aicpu/dep_gen_collector_aicpu.cpp | 2 +- .../platform/sim/host/device_runner_base.cpp | 2 +- .../platform/sim/host/device_runner_base.h | 2 +- src/common/task_interface/task_args.h | 2 +- tests/ut/cpp/CMakeLists.txt | 2 +- tests/ut/cpp/a2a3/test_a2a3_fatal.cpp | 2 +- .../common/test_trb_runtime_temp_buffer.cpp | 2 +- tests/ut/py/test_resource_failure_summary.py | 4 +-- tests/ut/py/test_worker/test_host_worker.py | 2 +- 77 files changed, 204 insertions(+), 198 deletions(-) diff --git a/docs/dfx/profiling-config-naming.md b/docs/dfx/profiling-config-naming.md index ad787396c1..93155d8bff 100644 --- a/docs/dfx/profiling-config-naming.md +++ b/docs/dfx/profiling-config-naming.md @@ -63,10 +63,20 @@ runtime-specific ones) unambiguous about ownership. ### 4. Everything carries the `SIMPLER_` project prefix -`PTO2_` is the device runtime's internal namespace (89 identifiers: -`CHIP_MAX_RING_DEPTH`, `SIMPLER_ERROR_*`, `RuntimeContext`, …). Configuration -surface exposed to users / CI / external consumers is unified under -`SIMPLER_`, regardless of which internal namespace implements it. +The device runtime's internal names carry no project prefix at all — they are +plain domain names (`RuntimeContext`, `SchedulerState`, `TaskDescriptor`), taking +a `Chip` / `CHIP_` prefix in two cases: where the host orchestrator owns the bare +name (`ChipTensorMap`, `ChipReadyQueue`, `CHIP_MAX_SCOPE_DEPTH`), and for +object-like macros, which capture their identifier textually in every including +translation unit (`CHIP_HEAP_SIZE`, `CHIP_ALIGN_UP`). Configuration surface exposed +to users / CI / external consumers is unified under `SIMPLER_`, and status codes +under `SIMPLER_ERROR_*`. + +The exception is `PTO2_RING_TASK_WINDOW` / `PTO2_RING_HEAP` / +`PTO2_RING_DEP_POOL`. Those three are read from the environment by name, and other +repositories and shell scripts set them, so renaming them would be ignored +silently rather than failing to compile. They keep the legacy spelling until a +cross-repo migration retires them. ## Compile-time gates vs runtime emission diff --git a/docs/dynamic-linking.md b/docs/dynamic-linking.md index 79a5b06f3f..0e6f4f6377 100644 --- a/docs/dynamic-linking.md +++ b/docs/dynamic-linking.md @@ -232,7 +232,7 @@ SchedulerContext owns its own teardown: (`sync_start_pending` / `drain_attempt` / `drain_ack_tokens_` / `pending_task` / parallel-stage state), task counters, worker-id lists, core trackers, `cores_total_num_` / `aic_count_` / `aiv_count_`, - `regs_`, `sched_`, `func_id_to_addr_`, and the `pto2_init_*` flags. + `regs_`, `sched_`, and `func_id_to_addr_`. - `AicpuExecutor::deinit()` calls `sched_ctx_.deinit()` first, then resets only its own fields: `thread_num_`, `sched_thread_num_`, `orch_func_`, `orch_args_cached_`, `orch_so_handle_`, `orch_so_path_`, diff --git a/docs/orchestrator.md b/docs/orchestrator.md index 5e7fd60e7c..d2c19d7879 100644 --- a/docs/orchestrator.md +++ b/docs/orchestrator.md @@ -522,7 +522,7 @@ state machine). counter, then returns. A task whose `fanout_released` now meets the threshold transitions to CONSUMED inline; others stay COMPLETED or PENDING until the scheduler and consumers finish their own releases. This mirrors -L2's `pto2_scope_end`. +the chip runtime's `rt_scope_end`. The internal run fence, not `scope_end`, provides synchronous completion. `Worker.run` closes its outer scope, closes submission, and waits for that diff --git a/docs/task-flow.md b/docs/task-flow.md index 619aa94ff1..2f8c54525b 100644 --- a/docs/task-flow.md +++ b/docs/task-flow.md @@ -133,7 +133,7 @@ remote sidecars; the remote framed path encodes the sidecar as a | **① User submit** | `TaskArgs` object (builder) | Python/C++ parent heap | user orch fn | Orchestrator | | **② Slot storage** | `TaskArgs` object (inside `slot.task_args`) | parent heap | Orchestrator.submit moves it here | WorkerThread at dispatch | | **③ Dispatch wire (PROCESS only)** | length-prefixed blob | shm mailbox (MAP_SHARED) | parent WorkerThread encodes | forked child decodes | -| **④ L2 ABI edge** | `ChipStorageTaskArgs` POD | child stack | `ChipWorker::run` assembles | `pto2_run_runtime` consumes | +| **④ L2 ABI edge** | `ChipStorageTaskArgs` POD | child stack | `ChipWorker::run` assembles | `simpler_run` consumes | ### Tags stripped at submit @@ -201,7 +201,7 @@ View does **not** own memory. Valid for the duration of a single ④ ChipStorageTaskArgs POD — child stack │ memcpy view.tensors, view.scalars into struct ▼ - pto2_run_runtime(local_slot, &chip_storage, &config) + simpler_run(local_slot, &chip_storage, &config) ``` --- @@ -232,7 +232,7 @@ Propagated by value throughout: 5. Child reads `CallConfig` from mailbox by value, or the remote session runner reconstructs it from `CallConfigWire` 6. `ChipWorker::run` receives `const CallConfig&`; passed on to - `pto2_run_runtime` at the L2 edge + `simpler_run` at the L2 edge Same type at every level. Used directly at the L2 runtime ABI. @@ -248,7 +248,7 @@ concrete leaves, each consumed by its own Python child loop. Wraps a dlsym'd `runtime.so`. `_chip_process_loop` instantiates one `ChipWorker` per chip child and calls its `run` on every dispatch. `run()` assembles a `ChipStorageTaskArgs` POD from the decoded view and -calls `pto2_run_runtime`: +calls `simpler_run`: ```cpp void ChipWorker::run(int32_t local_slot, TaskArgsView view, const CallConfig &config) { @@ -257,7 +257,7 @@ void ChipWorker::run(int32_t local_slot, TaskArgsView view, const CallConfig &co chip_storage.scalar_count_ = view.scalar_count; memcpy(chip_storage.tensors_, view.tensors, view.tensor_count * sizeof(ChipTensor)); memcpy(chip_storage.scalars_, view.scalars, view.scalar_count * sizeof(uint64_t)); - pto2_run_runtime(local_slot, &chip_storage, &config); + simpler_run(local_slot, &chip_storage, &config); } ``` diff --git a/docs/troubleshooting/device-error-codes.md b/docs/troubleshooting/device-error-codes.md index 5f7b73e751..a16ae593a3 100644 --- a/docs/troubleshooting/device-error-codes.md +++ b/docs/troubleshooting/device-error-codes.md @@ -116,7 +116,7 @@ Code 100 is a funnel. The runtime sub-classifies it on device and prints the verdict plus locators, so you rarely need the device log: ```text -[ERROR] PTO2 scheduler timeout sub_class=S1:running-stalled (detail=1) completed=0/1 \ +[ERROR] scheduler timeout sub_class=S1:running-stalled (detail=1) completed=0/1 \ running=1 ready=0 waiting=0 orch_done=1 stuck_task_id=42 stuck_core=5 ``` diff --git a/simpler_setup/tools/sched_overhead_analysis.py b/simpler_setup/tools/sched_overhead_analysis.py index 77e731d7e4..285e38e213 100644 --- a/simpler_setup/tools/sched_overhead_analysis.py +++ b/simpler_setup/tools/sched_overhead_analysis.py @@ -7,7 +7,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. # ----------------------------------------------------------------------------------------------------------- -"""Scheduler overhead analysis for PTO2 — is the scheduler the bottleneck, or starved? +"""Scheduler overhead analysis — is the scheduler the bottleneck, or starved? Inputs (BOTH required, captured in SEPARATE runs — do not co-run the flags, as dep_gen perturbs the swimlane timing): @@ -1004,7 +1004,7 @@ def _pct(x, denom): def main(): parser = argparse.ArgumentParser( - description="Scheduler overhead analysis for PTO2", + description="Scheduler overhead analysis", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: diff --git a/simpler_setup/tools/swimlane_converter.py b/simpler_setup/tools/swimlane_converter.py index 9a277565d0..d484c066a5 100644 --- a/simpler_setup/tools/swimlane_converter.py +++ b/simpler_setup/tools/swimlane_converter.py @@ -86,8 +86,8 @@ def _task_display_name(func_id, func_id_to_name, tdisp, *, spmd=False): return label -def normalize_pto2_task_id_int(v): - """Unsigned 64-bit PTO2 task id (matches host JSON / device ``task_id.raw``). +def normalize_task_id_int(v): + """Unsigned 64-bit task id (matches host JSON / device ``task_id.raw``). Normalizes signed values to unsigned for ``(ring_id << 32) | local_id``. Returns None if ``v`` is not convertible to int. @@ -102,7 +102,7 @@ def normalize_pto2_task_id_int(v): def format_task_display(task_id): - """Format PTO2 task_id for human-readable labels. + """Format a task_id for human-readable labels. Layout: 64-bit raw = (ring_id << 32) | local_id (same as runtime TaskId). @@ -111,7 +111,7 @@ def format_task_display(task_id): For invalid or non-numeric values, returns str(task_id). """ - tid = normalize_pto2_task_id_int(task_id) + tid = normalize_task_id_int(task_id) if tid is None: return str(task_id) ring = (tid >> 32) & 0xFF @@ -127,7 +127,7 @@ def _decode_graph_node_task_id(task_id): Graph nodes use ``local=(outer_local << 10) | node_index`` while the stream-visible outer Graph task remains on ring 0. """ - tid = normalize_pto2_task_id_int(task_id) + tid = normalize_task_id_int(task_id) if tid is None or ((tid >> 32) & 0xFFFFFFFF) != 1: return None local = tid & 0xFFFFFFFF @@ -142,7 +142,7 @@ def _collect_graph_execution_instances(tasks, scheduler_phases): # noqa: PLR091 for record in records: phase = record.get("phase") if phase == "graph_prepare": - outer_task_id = normalize_pto2_task_id_int(record.get("task_id")) + outer_task_id = normalize_task_id_int(record.get("task_id")) if outer_task_id is not None and (outer_task_id >> 32) == 0: prepare_by_outer[outer_task_id].append(record) elif phase == "dummy_task": @@ -348,7 +348,7 @@ def read_perf_data(filepath): # noqa: PLR0912, PLR0915 host_composite_end_us = (max(host_timestamps) - host_origin_ns) / 1000.0 if host_timestamps else 0.0 # AICore lookup keyed by (core_id, reg_task_id). Two dispatches of the - # same PTO2 task_token_raw to the same core (SPMD over-subscription, MIX + # same task_token_raw to the same core (SPMD over-subscription, MIX # cluster spread) each get their own reg_task_id, so this key is unique # per dispatch even when task_token_raw collides. # @@ -666,8 +666,8 @@ def load_deps_json(deps_path): for edge in edges: if not isinstance(edge, dict): continue - pred = normalize_pto2_task_id_int(edge.get("pred")) - succ = normalize_pto2_task_id_int(edge.get("succ")) + pred = normalize_task_id_int(edge.get("pred")) + succ = normalize_task_id_int(edge.get("succ")) if pred is None or succ is None: continue key = (pred, succ) @@ -711,7 +711,7 @@ def load_deps_kernel_map(deps_path): for task in tasks: if not isinstance(task, dict): continue - tid = normalize_pto2_task_id_int(task.get("task_id")) + tid = normalize_task_id_int(task.get("task_id")) kids = task.get("kernel_ids") if tid is None or not isinstance(kids, list) or len(kids) != 3: continue @@ -742,7 +742,7 @@ def load_deps_block_map(deps_path): for task in tasks: if not isinstance(task, dict): continue - tid = normalize_pto2_task_id_int(task.get("task_id")) + tid = normalize_task_id_int(task.get("task_id")) if tid is None: continue try: @@ -1834,7 +1834,7 @@ def _find_containing_complete(thread_idx: int, finish_us: float): start_us = record["start_time_us"] end_us = record["end_time_us"] dur = max(end_us - start_us, AICPU_WORKER_MARKER_MIN_DUR_US) - task_id = normalize_pto2_task_id_int(record.get("task_id")) + task_id = normalize_task_id_int(record.get("task_id")) task_label = format_task_display(task_id) if task_id is not None else "unknown" if phase == "dummy_task": event_name = f"dummy({task_label})" @@ -2094,7 +2094,7 @@ def _find_containing_complete(thread_idx: int, finish_us: float): for rec in thread_records: phase = rec.get("phase") if phase in ("dummy_task", "predicated_skip"): - task_id = normalize_pto2_task_id_int(rec.get("task_id")) + task_id = normalize_task_id_int(rec.get("task_id")) if task_id is not None: if phase == "dummy_task": dummy_task_ids.add(task_id) @@ -2703,7 +2703,7 @@ def _find_containing_complete(thread_idx: int, finish_us: float): task_id = record.get("task_id", -1) if task_id < 0: continue - tid_k = normalize_pto2_task_id_int(task_id) + tid_k = normalize_task_id_int(task_id) if tid_k is None: continue # First-seen orch_submit wins; legacy orch_fanin / orch_params @@ -2721,7 +2721,7 @@ def _find_containing_complete(thread_idx: int, finish_us: float): if has_aicpu_data and orch_anchor_by_task: for task in tasks: - tid = normalize_pto2_task_id_int(task.get("task_id")) + tid = normalize_task_id_int(task.get("task_id")) if tid is None: continue diff --git a/src/a2a3/docs/runtimes.md b/src/a2a3/docs/runtimes.md index 85941234e1..3129f26f1f 100644 --- a/src/a2a3/docs/runtimes.md +++ b/src/a2a3/docs/runtimes.md @@ -26,7 +26,7 @@ The simplest runtime. The host CPU builds the complete task dependency graph bef See [host_build_graph/docs/RUNTIME_LOGIC.md](../runtime/host_build_graph/docs/RUNTIME_LOGIC.md) for details. -## tensormap_and_ringbuffer (PTO2) +## tensormap_and_ringbuffer The primary production runtime. Uses ring buffers for task slots and output memory, with a TensorMap for automatic dependency tracking. diff --git a/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp b/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp index 968b97640b..066d724db0 100644 --- a/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp +++ b/src/a2a3/runtime/host_build_graph/aicore/aicore_executor.cpp @@ -273,7 +273,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // Performance profiling: record task execution. // Two identity fields go into the record (different roles): - // - task_token_raw (PTO2 ring/local) is pulled from the dispatch + // - task_token_raw (ring/local) is pulled from the dispatch // payload's LocalContext.async_ctx — already in AICore cache // from the just-completed task, no extra GM load. Host uses // it as the canonical task identity for JSON output / ring diff --git a/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp b/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp index 1ab6986a84..15c2b6a72f 100644 --- a/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp +++ b/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp @@ -60,7 +60,7 @@ // here (cleared on teardown before runtime_destroy). extern "C" void framework_bind_runtime(RuntimeContext *rt); -static int32_t read_pto2_runtime_status(Runtime *runtime) { +static int32_t read_runtime_status(Runtime *runtime) { if (runtime == nullptr) { return 0; } @@ -480,7 +480,7 @@ extern "C" int32_t aicpu_execute(Runtime *runtime) { LOG_ERROR("aicpu_execute: Thread execution failed with rc=%d", rc); } - int32_t runtime_rc = read_pto2_runtime_status(runtime); + int32_t runtime_rc = read_runtime_status(runtime); // The finalizer publishes cleanup eligibility only after runtime destruction. if (g_aicpu_executor.completion_gate_.claim_cleanup()) { diff --git a/src/a2a3/runtime/host_build_graph/docs/profiling_levels.md b/src/a2a3/runtime/host_build_graph/docs/profiling_levels.md index 238594e542..7534a297af 100644 --- a/src/a2a3/runtime/host_build_graph/docs/profiling_levels.md +++ b/src/a2a3/runtime/host_build_graph/docs/profiling_levels.md @@ -59,7 +59,7 @@ Each sub-level macro requires `SIMPLER_DFX=1`: **What's compiled:** - Debug/diagnostic logs (always present) -- Progress tracking (`PTO2 progress: completed=...`) +- Progress tracking (`progress: completed=...`) - Stall detection and dump (triggered after the `SCHEDULER_TIMEOUT_MS` wall-clock no-progress budget) - Deadlock/livelock detection (`diagnose_stuck_state`, called on stall) @@ -74,8 +74,8 @@ Each sub-level macro requires `SIMPLER_DFX=1`: - No `sched_start/sched_end/sched_cost` timestamps - No `orch_start/orch_end/orch_cost` timestamps - No `Scheduler summary: total_time=...` -- No `PTO2 total submitted tasks` log -- `PTO2 progress: completed=... total=...` may appear (thread 0 only, at task completion milestones) +- No `total submitted tasks` log +- `progress: completed=... total=...` may appear (thread 0 only, at task completion milestones) --- @@ -83,7 +83,7 @@ Each sub-level macro requires `SIMPLER_DFX=1`: host_build_graph boots **scheduler-only** — the orchestrator runs on the host, so the device log carries no `orch_start`/`orch_end`/`orch_cost` lines and no -`PTO2 total submitted tasks` line (see the note at the top of this file). Every +`total submitted tasks` line (see the note at the top of this file). Every AICPU thread schedules its own core slice, so `N_sched == aicpu_thread_num`. **What's compiled:** @@ -393,7 +393,7 @@ header just like on onboard. | 3 | + Scheduler phases (`SCHED_*`) | | 4 | + Orchestrator phases (full) | -At level 1 the AICore record carries the full PTO2 `task_token_raw` +At level 1 the AICore record carries the full `task_token_raw` (`(ring_id << 32) | local_id`), read straight from `LocalContext.async_ctx.task_token.raw` inside the AICore helper — already in cache from the dispatch payload, so no extra GM load. diff --git a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp index a323ecb722..a52b693d87 100644 --- a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp @@ -312,14 +312,14 @@ static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedM return 0; } - void *pto2_sm = runtime->get_gm_sm_ptr(); - if (pto2_sm == nullptr) { + void *device_sm = runtime->get_gm_sm_ptr(); + if (device_sm == nullptr) { return 0; } - int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(SharedMemoryHeader)); + int hdr_rc = api->copy_from_device(host_header, device_sm, sizeof(SharedMemoryHeader)); if (hdr_rc != 0) { - LOG_WARN("Failed to copy PTO2 header from device"); + LOG_WARN("Failed to copy the shared-memory header from device"); return 0; } @@ -373,7 +373,7 @@ bool create_orch_so_tempfile(const uint8_t *data, size_t size, std::string *out_ return true; } -// The orchestration .so exports these (PTO2 submit_task form). +// The orchestration .so exports these (submit_task form). typedef void (*OrchestrationEntryFunc)(const ChipTaskArgs &); typedef void (*OrchestrationBindFunc)(RuntimeContext *); typedef void (*OrchestrationPrewarmFunc)(); @@ -997,7 +997,7 @@ extern "C" int register_callable_impl(const ChipCallable *callable, const HostAp /** * Per-run binding: build device-side argument storage (tensor copy-out, GM - * heap, PTO2 shared memory) and publish it to the runtime. Assumes the + * heap, shared memory) and publish it to the runtime. Assumes the * callable-side state (kernel binaries, orch SO bytes, func/config names) * is already populated by register_callable_impl. * diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime.h b/src/a2a3/runtime/host_build_graph/runtime/runtime.h index 608b6c069f..7af1ce0b65 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime.h @@ -164,7 +164,7 @@ class Runtime { int32_t aicpu_allowed_cpu_count; int32_t aicpu_launch_count; - // PTO2 integration: kernel_id -> GM function_bin_addr mapping + // kernel binary resolution: kernel_id -> GM function_bin_addr mapping // NOTE: Made public for direct access from aicore code uint64_t func_id_to_addr_[RUNTIME_MAX_FUNC_ID]; @@ -184,7 +184,7 @@ class Runtime { private: // Kernel binary tracking for cleanup - void *gm_sm_ptr_; // GM pointer to PTO2 shared memory (device) + void *gm_sm_ptr_; // GM pointer to shared memory (device) void *slot_states_ptr_; // Pointer to ChipTaskSlotState array (scheduler-private, for profiling) ChipStorageTaskArgs orch_args_storage_; // Copy of args for device @@ -293,7 +293,7 @@ class Runtime { // Task graph is now managed by RuntimeContext, not Runtime // ========================================================================= - /** @deprecated Task count is now in PTO2 shared memory */ + /** @deprecated Task count is now in shared memory */ int get_task_count() const { return 0; } /** @deprecated RT2 uses DispatchPayload, not Task. Always returns nullptr. */ diff --git a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h index 06446d6c32..46f569857f 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/runtime_types.h @@ -71,7 +71,7 @@ // Task management // Actual window size is passed at runtime to runtime_create_from_sm(). -// Use pto2_task_slot(sched, task_id) for slot calculation. +// The slot is local_id masked by the window size, which is why the window is a power of two. #define CHIP_TASK_WINDOW_SIZE 16384 // Default task window size (power of 2) // host_build_graph has one ring, and carries no per-ring dimension anywhere: diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index efbd4e2fc1..703b18b835 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -83,8 +83,7 @@ LoopAction SchedulerContext::handle_orchestrator_exit( if (task_count > 0 && completed_tasks_.load(std::memory_order_relaxed) >= task_count) { completed_.store(true, std::memory_order_release); LOG_INFO( - "Thread %d: PTO2 completed tasks %d/%d", thread_idx, completed_tasks_.load(std::memory_order_relaxed), - task_count + "Thread %d: completed tasks %d/%d", thread_idx, completed_tasks_.load(std::memory_order_relaxed), task_count ); return LoopAction::BREAK_LOOP; } @@ -905,7 +904,7 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { } #endif - // Initialize task counters. Task count comes from PTO2 shared memory. + // Initialize task counters. Task count comes from shared memory. if (runtime->get_gm_sm_ptr()) { auto *header = static_cast(runtime->get_gm_sm_ptr()); // Read at one-time boot init, before the SM is reset for the run, so a ring @@ -914,10 +913,10 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // CHIP_TASK_WINDOW_SIZE], since the ring cannot hold more than its task // window — so any garbage pattern, negative or positive, leaves the count // at 0, which is the correct value at boot. - int32_t pto2_count = 0; + int32_t window_tasks = 0; int32_t ring_tasks = header->ring.fc.current_task_index.load(std::memory_order_acquire); - if (ring_tasks > 0 && ring_tasks <= CHIP_TASK_WINDOW_SIZE) pto2_count = ring_tasks; - total_tasks_ = pto2_count; + if (ring_tasks > 0 && ring_tasks <= CHIP_TASK_WINDOW_SIZE) window_tasks = ring_tasks; + total_tasks_ = window_tasks; } else { total_tasks_ = 0; } diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index 43965a8ca5..52eb8f0855 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -154,7 +154,7 @@ class SchedulerContext { // Run all post-orchestration scheduler bookkeeping: // - publishes core assignments to the perf collector (SIMPLER_DFX) - // - latches submitted task count from PTO2 shared memory + // - latches submitted task count from shared memory // - folds inline_completed_tasks into completed_tasks_ // (skipped on fatal error — emergency_shutdown runs instead) // Callers must invoke rt_orchestration_done(rt) before this — that diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 61f5ca99dc..11ae17c929 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -892,7 +892,7 @@ int32_t SchedulerContext::run_resolution_thread(Runtime *runtime, int32_t thread always_assert(sched_ != nullptr); SharedMemoryHeader *header = sched_->sm_header; if (!header) { - LOG_ERROR("PTO2 resolution: header is null"); + LOG_ERROR("resolution: header is null"); return -1; } LOG_INFO("Thread %d: resolution (P) thread starting, serving %d schedulers", thread_idx, active_sched_threads_); @@ -1064,13 +1064,13 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ SharedMemoryHeader *header = sched_->sm_header; if (!header) { - LOG_ERROR("PTO2 dispatch: header is null"); + LOG_ERROR("dispatch: header is null"); return -1; } Handshake *hank = static_cast(runtime->workers); - LOG_INFO("Thread %d: PTO2 dispatch starting with %d cores", thread_idx, core_trackers_[thread_idx].core_num()); + LOG_INFO("Thread %d: dispatch starting with %d cores", thread_idx, core_trackers_[thread_idx].core_num()); int32_t cur_thread_completed = 0; // Non-zero once a scheduler-hang timeout latches; returned in place of the // completed count so the caller still sees the negative error rc while the @@ -1224,7 +1224,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ if (new_total <= PROGRESS_VERBOSE_THRESHOLD || new_total / PROGRESS_LOG_INTERVAL != prev / PROGRESS_LOG_INTERVAL || new_total >= task_count) { LOG_INFO( - "PTO2 progress: completed=%d total=%d (%.1f%%)", new_total, task_count, + "progress: completed=%d total=%d (%.1f%%)", new_total, task_count, 100.0 * new_total / task_count ); } diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp index e9f7689253..a2acc0738f 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -203,7 +203,7 @@ bool OrchestratorState::init( auto *orch = this; *orch = OrchestratorState{}; - // A power-of-two window lets pto2_task_slot() mask instead of dividing. + // A power-of-two window lets the slot index mask instead of dividing. always_assert(task_window_size > 0 && (task_window_size & (task_window_size - 1)) == 0); orch->sm_header = reinterpret_cast(sm_base); diff --git a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 70350a5603..35e3e044fe 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -181,7 +181,7 @@ void SharedMemoryHandle::print_layout() { SharedMemoryHeader *h = header; - LOG_DEBUG("=== PTO2 Shared Memory Layout ==="); + LOG_DEBUG("=== Shared Memory Layout ==="); LOG_DEBUG("Base address: %p", sm_base); LOG_DEBUG("Total size: %" PRIu64 " bytes", h->total_size); LOG_DEBUG("Ring:"); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp index 32e5818568..2d48a6a093 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp @@ -251,7 +251,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in } // Two identity fields go into the record (different roles): - // - task_token_raw (PTO2 ring/local) is pulled from the dispatch + // - task_token_raw (ring/local) is pulled from the dispatch // payload's LocalContext.async_ctx — already in AICore cache // from the just-completed task, no extra GM load. Host uses // it as the canonical task identity for JSON output / ring diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index 12ba8680e2..8214c837fc 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -798,7 +798,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { #endif #endif // SIMPLER_ORCH_PROFILING - // Latch task count from PTO2 shared memory to hand off to the + // Latch task count from shared memory to hand off to the // scheduler. The orchestrator's run window (start_time / end_time / // submit_count) is no longer published to shared memory — the // device LOG_INFO "orch_start=… orch_end=… orch_cost=…" line @@ -837,7 +837,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { ); if (submitted_tasks >= 0) { LOG_INFO( - "PTO2 total submitted tasks = %d, already executed %d tasks", submitted_tasks, + "total submitted tasks = %d, already executed %d tasks", submitted_tasks, sched_ctx_.completed_tasks_count() ); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 3f78c2ee29..2c9802b507 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -1,6 +1,6 @@ # Multi-Ring Buffer Architecture -> Extension to the PTO2 runtime. For the base architecture, see [RUNTIME_LOGIC.md](RUNTIME_LOGIC.md). +> Extension to `tensormap_and_ringbuffer`. For the base architecture, see [RUNTIME_LOGIC.md](RUNTIME_LOGIC.md). ## 1. Problem diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 7f7f374465..e653a32b67 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -818,7 +818,7 @@ capacity because no task execution/reclaim happens during graph build. --- -## 11. PTO2 Orchestration API +## 11. Orchestration API The orchestration API is defined in `orchestration_api.h`. Orchestration code depends only on this header. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md index 6ac0f2f027..867b4ec87b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md @@ -2,7 +2,7 @@ ## 1. Goal -Define a single, main-branch-aligned specification for PTO2 cluster submission that combines: +Define a single, main-branch-aligned specification for cluster submission that combines: 1. Product requirements (what must be true). 2. Runtime design (how it is implemented on current main baseline). @@ -82,7 +82,7 @@ Rules: 4. `aiv0_kernel_id` and `aiv1_kernel_id` are semantically equivalent. 5. Wrappers are orchestration sugar only (inline in orchestration API); no dedicated runtime ops entries. 6. Submit-contract types are defined once in a shared header-only submit-types surface consumed by orchestration and runtime headers. -7. Invalid submits follow existing PTO2 behavior (`always_assert`), not a new recoverable return-code API. +7. Invalid submits follow existing behavior (`always_assert`), not a new recoverable return-code API. ## 7. Data Model (Requirements + Design) diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/device_log_profiling.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/device_log_profiling.md index df592a6bef..aa9ba62cda 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/device_log_profiling.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/device_log_profiling.md @@ -1,4 +1,4 @@ -# PTO2 Device Log Profiling Guide +# Device Log Profiling Guide ## How to Find Device Logs @@ -21,7 +21,7 @@ A single run produces two profiling blocks in the device log: | Block | Emitted by | Function | Content | | ----- | ---------- | -------- | ------- | | **Orchestrator Profiling** | Thread 3 (orchestrator) | `aicpu_orchestration_entry` | Time breakdown of graph construction on device | -| **PTO2 Scheduler Summary** | Threads 0/1/2 (schedulers) | `SchedulerContext::resolve_and_dispatch` | Per-thread scheduling statistics, phase timing, and lock contention | +| **Scheduler Summary** | Threads 0/1/2 (schedulers) | `SchedulerContext::resolve_and_dispatch` | Per-thread scheduling statistics, phase timing, and lock contention | All timing values are in microseconds (us), converted from AICPU cycle counters. @@ -47,7 +47,7 @@ Thread 3: fanin+ready : 1207.400us (8.3%) Thread 3: finalize+SM : 697.500us (4.8%) Thread 3: scope_end : 364.080us Thread 3: avg/task : 0.874us -Thread 3: PTO2 total submitted tasks = 16704 +Thread 3: total submitted tasks = 16704 ``` ### Field Reference @@ -75,7 +75,7 @@ Thread 3: PTO2 total submitted tasks = 16704 --- -## Block 2: PTO2 Scheduler Summary +## Block 2: Scheduler Summary Each of the 3 scheduler threads (Thread 0, 1, 2) prints its own summary after completing all tasks. The output has two sub-sections: **summary** and **phase breakdown**. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/profiling_levels.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/profiling_levels.md index 862e22d753..4db296a856 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/profiling_levels.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/profiling_levels.md @@ -47,7 +47,7 @@ Each sub-level macro requires `SIMPLER_DFX=1`: **What's compiled:** - Debug/diagnostic logs (always present) -- Progress tracking (`PTO2 progress: completed=...`) +- Progress tracking (`progress: completed=...`) - Stall detection and dump (triggered after the `SCHEDULER_TIMEOUT_MS` wall-clock no-progress budget) - Deadlock/livelock detection (`diagnose_stuck_state`, called on stall) @@ -62,8 +62,8 @@ Each sub-level macro requires `SIMPLER_DFX=1`: - No `sched_start/sched_end/sched_cost` timestamps - No `orch_start/orch_end/orch_cost` timestamps - No `Scheduler summary: total_time=...` -- No `PTO2 total submitted tasks` log -- `PTO2 progress: completed=... total=...` may appear (thread 0 only, at task completion milestones) +- No `total submitted tasks` log +- `progress: completed=... total=...` may appear (thread 0 only, at task completion milestones) --- @@ -161,7 +161,7 @@ captured at chip_swimlane_level >= 3) and `deps.json`; consume them via **Log output (per orchestrator thread, normal run):** the orchestrator phase breakdown, followed by the `orch_start/orch_end/orch_cost` line and the -`PTO2 total submitted tasks` line — all gated by `SIMPLER_ORCH_PROFILING`. This +`total submitted tasks` line — all gated by `SIMPLER_ORCH_PROFILING`. This level adds orchestrator-side logs only; the scheduler side is unchanged from Level 1 (add `SIMPLER_SCHED_PROFILING` / Level 2 for scheduler detail). @@ -180,7 +180,7 @@ Thread X: finalize+SM : XXXus (XX.X%) work=XXXus wait=XXXus atomics=XXX Thread X: scope_end : XXXus atomics=XXX Thread X: avg/task : XXXus Thread X: orch_start=XXX orch_end=XXX orch_cost=XXXus -PTO2 total submitted tasks = XXX, already executed XXX tasks +total submitted tasks = XXX, already executed XXX tasks ``` **Note:** Orchestrator logs always print when `SIMPLER_ORCH_PROFILING=1`, regardless of `enable_chip_swimlane` flag. @@ -240,7 +240,7 @@ header just like on onboard. | 3 | + Scheduler phases (`SCHED_*`) | | 4 | + Orchestrator phases (full) | -At level 1 the AICore record carries the full PTO2 `task_token_raw` +At level 1 the AICore record carries the full `task_token_raw` (`(ring_id << 32) | local_id`), read straight from `LocalContext.async_ctx.task_token.raw` inside the AICore helper — already in cache from the dispatch payload, so no extra GM load. @@ -405,7 +405,7 @@ definitions to runtime headers. | 0 | `SIMPLER_DFX=0` | 0 | No timing output | | 1 | `SIMPLER_DFX=1` | 0 | Host-side `Orch`/`Sched` `[STRACE]` windows only; no device logs | | 2 | `+SIMPLER_SCHED_PROFILING=1` | per sched thread | `sched_start` + phase breakdown + `Scheduler summary` | -| 3 | `+SIMPLER_ORCH_PROFILING=1` | per orch thread | Orchestrator phase breakdown + `orch_start` + `PTO2 total` | +| 3 | `+SIMPLER_ORCH_PROFILING=1` | per orch thread | Orchestrator phase breakdown + `orch_start` + `total submitted tasks` | | 4 | `+SIMPLER_TENSORMAP_PROFILING=1` | per orch thread | + TensorMap lookup stats (4 lines) | --- diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index 130183d402..4367aecc8f 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -289,14 +289,14 @@ static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedM return 0; } - void *pto2_sm = runtime->get_gm_sm_ptr(); - if (pto2_sm == nullptr) { + void *device_sm = runtime->get_gm_sm_ptr(); + if (device_sm == nullptr) { return 0; } - int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(SharedMemoryHeader)); + int hdr_rc = api->copy_from_device(host_header, device_sm, sizeof(SharedMemoryHeader)); if (hdr_rc != 0) { - LOG_WARN("Failed to copy PTO2 header from device"); + LOG_WARN("Failed to copy the shared-memory header from device"); return 0; } @@ -673,7 +673,7 @@ static void apply_orch_sched_env_flags(Runtime *runtime) { ); } -// per-(cid,config): reserve and acquire the static device pools. GM heap, PTO2 +// per-(cid,config): reserve and acquire the static device pools. GM heap, shared memory // shared memory, and the prebuilt runtime arena all live in one backing // allocation; setup_static_arena reserves the three regions and commits in one // shot. The runtime-arena size is recovered by replaying the (pure, cheap) @@ -705,7 +705,7 @@ static bool ensure_static_arenas( out->gm_sm = api->acquire_pooled_gm_sm(); int64_t t_sm_end = _now_ms(); if (out->gm_sm == nullptr) { - LOG_ERROR("Failed to acquire pooled PTO2 shared memory"); + LOG_ERROR("Failed to acquire pooled shared memory"); return false; } @@ -844,7 +844,7 @@ static bool build_and_cache_prebuilt_arena( /** * Per-run binding: build device-side argument storage (tensor copy-out, GM - * heap, PTO2 shared memory) and publish it to the runtime. Assumes the + * heap, shared memory) and publish it to the runtime. Assumes the * callable-side state (kernel binaries, orch SO bytes, func/config names) * is already populated by register_callable_impl. * @@ -1033,7 +1033,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e if (sched_error_code == SIMPLER_ERROR_SCHEDULER_TIMEOUT) { int32_t detail = host_header.sched_stall_detail.load(std::memory_order_acquire); LOG_ERROR( - "PTO2 scheduler timeout sub_class=%s (detail=%d) completed=%d/%d running=%d ready=%d waiting=%d " + "scheduler timeout sub_class=%s (detail=%d) completed=%d/%d running=%d ready=%d waiting=%d " "orch_done=%d stuck_task_id=%" PRId64 " stuck_core=%d", stall_detail_name(detail), detail, host_header.sched_stall_completed.load(std::memory_order_relaxed), host_header.sched_stall_total.load(std::memory_order_relaxed), diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime.h index 7f89df8daa..fb371f4827 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime.h @@ -179,7 +179,7 @@ struct alignas(64) DeviceRuntimeLaunchDesc { int32_t aicpu_allowed_cpu_count; int32_t aicpu_launch_count; - // PTO2 integration: kernel_id -> GM function_bin_addr mapping + // kernel binary resolution: kernel_id -> GM function_bin_addr mapping uint64_t func_id_to_addr_[RUNTIME_MAX_FUNC_ID]; // Serial orchestrator -> scheduler start control. @@ -188,7 +188,7 @@ struct alignas(64) DeviceRuntimeLaunchDesc { // Controlled via SIMPLER_TMR_SERIAL_ORCH_SCHED_ENABLE environment variable. bool serial_orch_sched; - void *gm_sm_ptr_; // GM pointer to PTO2 shared memory (device) + void *gm_sm_ptr_; // GM pointer to shared memory (device) ChipStorageTaskArgs orch_args_storage_; // Copy of args for device // Prebuilt-arena fast path (trb only). Set by the host before rtMemcpy'ing @@ -306,7 +306,7 @@ class Runtime { // Task graph is now managed by RuntimeContext, not Runtime // ========================================================================= - /** @deprecated Task count is now in PTO2 shared memory */ + /** @deprecated Task count is now in shared memory */ int get_task_count() const { return 0; } /** @deprecated RT2 uses DispatchPayload, not Task. Always returns nullptr. */ diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index e9aedb4f5e..a0449bafda 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -61,7 +61,7 @@ // Task management // NOTE: CHIP_TASK_WINDOW_SIZE is now a per-ring default value. // Actual window size is passed at runtime to runtime_reserve_layout(). -// Use pto2_task_slot(sched, task_id) for slot calculation. +// The slot is local_id masked by the window size, which is why the window is a power of two. #define CHIP_TASK_WINDOW_SIZE 16384 // Default per-ring task window size (power of 2) // Multi-ring: number of independent ring layers (HeapRing + TaskRing + DepPool per layer) diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index b096f97eb3..881d182413 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -84,8 +84,7 @@ LoopAction SchedulerContext::handle_orchestrator_exit( if (task_count > 0 && completed_tasks_.load(std::memory_order_relaxed) >= task_count) { completed_.store(true, std::memory_order_release); LOG_INFO( - "Thread %d: PTO2 completed tasks %d/%d", thread_idx, completed_tasks_.load(std::memory_order_relaxed), - task_count + "Thread %d: completed tasks %d/%d", thread_idx, completed_tasks_.load(std::memory_order_relaxed), task_count ); return LoopAction::BREAK_LOOP; } @@ -1180,12 +1179,12 @@ int32_t SchedulerContext::pre_handshake_init( // (on the leader, before any thread is released) rather than post-handshake. if (runtime->get_gm_sm_ptr()) { auto *header = static_cast(runtime->get_gm_sm_ptr()); - int64_t pto2_count = 0; + int64_t window_tasks = 0; for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { int32_t ring_tasks = header->rings[r].fc.current_task_index.load(std::memory_order_acquire); - if (ring_tasks > 0 && ring_tasks <= CHIP_SCOPE_TASKS_CAP) pto2_count += ring_tasks; + if (ring_tasks > 0 && ring_tasks <= CHIP_SCOPE_TASKS_CAP) window_tasks += ring_tasks; } - total_tasks_ = static_cast(pto2_count); + total_tasks_ = static_cast(window_tasks); } else { total_tasks_ = 0; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 7085728c97..b6192bd9c0 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -110,7 +110,7 @@ class SchedulerContext { // Run all post-orchestration scheduler bookkeeping: // - publishes core assignments to the perf collector (SIMPLER_DFX) - // - latches submitted task count from PTO2 shared memory + // - latches submitted task count from shared memory // - folds inline_completed_tasks into completed_tasks_ // - flips orchestrator_done_ and triggers core transition // (skipped on fatal error — emergency_shutdown runs instead) diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 42157339b6..c1657167a9 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -894,13 +894,13 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ SharedMemoryHeader *header = sched_->sm_header; if (!header) { - LOG_ERROR("PTO2 dispatch: header is null"); + LOG_ERROR("dispatch: header is null"); return -1; } Handshake *hank = static_cast(runtime->dev.workers); - LOG_INFO("Thread %d: PTO2 dispatch starting with %d cores", thread_idx, core_trackers_[thread_idx].core_num()); + LOG_INFO("Thread %d: dispatch starting with %d cores", thread_idx, core_trackers_[thread_idx].core_num()); int32_t cur_thread_completed = 0; // Non-zero once a scheduler-hang timeout latches; returned in place of the // completed count so the caller still sees the negative error rc while the @@ -1059,7 +1059,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ if (new_total <= PROGRESS_VERBOSE_THRESHOLD || new_total / PROGRESS_LOG_INTERVAL != prev / PROGRESS_LOG_INTERVAL || new_total >= task_count) { LOG_INFO( - "PTO2 progress: completed=%d total=%d (%.1f%%)", new_total, task_count, + "progress: completed=%d total=%d (%.1f%%)", new_total, task_count, 100.0 * new_total / task_count ); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index 132e298a2d..8d41f3b657 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -186,7 +186,7 @@ void SharedMemoryHandle::print_layout() { SharedMemoryHeader *h = header; - LOG_DEBUG("=== PTO2 Shared Memory Layout ==="); + LOG_DEBUG("=== Shared Memory Layout ==="); LOG_DEBUG("Base address: %p", sm_base); LOG_DEBUG("Total size: %" PRIu64 " bytes", h->total_size); LOG_DEBUG("Ring depth: %d", CHIP_MAX_RING_DEPTH); diff --git a/src/a5/docs/runtimes.md b/src/a5/docs/runtimes.md index e2d7e295ac..986de54a08 100644 --- a/src/a5/docs/runtimes.md +++ b/src/a5/docs/runtimes.md @@ -17,7 +17,7 @@ See [host_build_graph/docs/RUNTIME_LOGIC.md](../runtime/host_build_graph/docs/RU Graph recording and replay are documented in [GRAPH_EXECUTION.md](../runtime/host_build_graph/docs/GRAPH_EXECUTION.md). -## tensormap_and_ringbuffer (PTO2) +## tensormap_and_ringbuffer See [tensormap_and_ringbuffer/docs/](../runtime/tensormap_and_ringbuffer/docs/): diff --git a/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp b/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp index 968b97640b..066d724db0 100644 --- a/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp +++ b/src/a5/runtime/host_build_graph/aicore/aicore_executor.cpp @@ -273,7 +273,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // Performance profiling: record task execution. // Two identity fields go into the record (different roles): - // - task_token_raw (PTO2 ring/local) is pulled from the dispatch + // - task_token_raw (ring/local) is pulled from the dispatch // payload's LocalContext.async_ctx — already in AICore cache // from the just-completed task, no extra GM load. Host uses // it as the canonical task identity for JSON output / ring diff --git a/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp b/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp index 6685adcb0f..e9d98e6e86 100644 --- a/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp +++ b/src/a5/runtime/host_build_graph/aicpu/aicpu_executor.cpp @@ -60,7 +60,7 @@ // here (cleared on teardown before runtime_destroy). extern "C" void framework_bind_runtime(RuntimeContext *rt); -static int32_t read_pto2_runtime_status(Runtime *runtime) { +static int32_t read_runtime_status(Runtime *runtime) { if (runtime == nullptr) { return 0; } @@ -484,7 +484,7 @@ extern "C" int32_t aicpu_execute(Runtime *runtime) { LOG_ERROR("aicpu_execute: Thread execution failed with rc=%d", rc); } - int32_t runtime_rc = read_pto2_runtime_status(runtime); + int32_t runtime_rc = read_runtime_status(runtime); // The finalizer publishes cleanup eligibility only after runtime destruction. if (g_aicpu_executor.completion_gate_.claim_cleanup()) { diff --git a/src/a5/runtime/host_build_graph/docs/profiling_levels.md b/src/a5/runtime/host_build_graph/docs/profiling_levels.md index 238594e542..7534a297af 100644 --- a/src/a5/runtime/host_build_graph/docs/profiling_levels.md +++ b/src/a5/runtime/host_build_graph/docs/profiling_levels.md @@ -59,7 +59,7 @@ Each sub-level macro requires `SIMPLER_DFX=1`: **What's compiled:** - Debug/diagnostic logs (always present) -- Progress tracking (`PTO2 progress: completed=...`) +- Progress tracking (`progress: completed=...`) - Stall detection and dump (triggered after the `SCHEDULER_TIMEOUT_MS` wall-clock no-progress budget) - Deadlock/livelock detection (`diagnose_stuck_state`, called on stall) @@ -74,8 +74,8 @@ Each sub-level macro requires `SIMPLER_DFX=1`: - No `sched_start/sched_end/sched_cost` timestamps - No `orch_start/orch_end/orch_cost` timestamps - No `Scheduler summary: total_time=...` -- No `PTO2 total submitted tasks` log -- `PTO2 progress: completed=... total=...` may appear (thread 0 only, at task completion milestones) +- No `total submitted tasks` log +- `progress: completed=... total=...` may appear (thread 0 only, at task completion milestones) --- @@ -83,7 +83,7 @@ Each sub-level macro requires `SIMPLER_DFX=1`: host_build_graph boots **scheduler-only** — the orchestrator runs on the host, so the device log carries no `orch_start`/`orch_end`/`orch_cost` lines and no -`PTO2 total submitted tasks` line (see the note at the top of this file). Every +`total submitted tasks` line (see the note at the top of this file). Every AICPU thread schedules its own core slice, so `N_sched == aicpu_thread_num`. **What's compiled:** @@ -393,7 +393,7 @@ header just like on onboard. | 3 | + Scheduler phases (`SCHED_*`) | | 4 | + Orchestrator phases (full) | -At level 1 the AICore record carries the full PTO2 `task_token_raw` +At level 1 the AICore record carries the full `task_token_raw` (`(ring_id << 32) | local_id`), read straight from `LocalContext.async_ctx.task_token.raw` inside the AICore helper — already in cache from the dispatch payload, so no extra GM load. diff --git a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp index efcfeb179a..1d8139e717 100644 --- a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp @@ -312,14 +312,14 @@ static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedM return 0; } - void *pto2_sm = runtime->get_gm_sm_ptr(); - if (pto2_sm == nullptr) { + void *device_sm = runtime->get_gm_sm_ptr(); + if (device_sm == nullptr) { return 0; } - int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(SharedMemoryHeader)); + int hdr_rc = api->copy_from_device(host_header, device_sm, sizeof(SharedMemoryHeader)); if (hdr_rc != 0) { - LOG_WARN("Failed to copy PTO2 header from device"); + LOG_WARN("Failed to copy the shared-memory header from device"); return 0; } @@ -373,7 +373,7 @@ bool create_orch_so_tempfile(const uint8_t *data, size_t size, std::string *out_ return true; } -// The orchestration .so exports these (PTO2 submit_task form). +// The orchestration .so exports these (submit_task form). typedef void (*OrchestrationEntryFunc)(const ChipTaskArgs &); typedef void (*OrchestrationBindFunc)(RuntimeContext *); typedef void (*OrchestrationPrewarmFunc)(); @@ -1013,7 +1013,7 @@ extern "C" int register_callable_impl(const ChipCallable *callable, const HostAp /** * Per-run binding: build device-side argument storage (tensor copy-out, GM - * heap, PTO2 shared memory) and publish it to the runtime. Assumes the + * heap, shared memory) and publish it to the runtime. Assumes the * callable-side state (kernel binaries, orch SO bytes, func/config names) * is already populated by register_callable_impl. * diff --git a/src/a5/runtime/host_build_graph/runtime/runtime.h b/src/a5/runtime/host_build_graph/runtime/runtime.h index 6449c56591..0fbf4b6158 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime.h @@ -163,7 +163,7 @@ class Runtime { int32_t aicpu_allowed_cpu_count; int32_t aicpu_launch_count; - // PTO2 integration: kernel_id -> GM function_bin_addr mapping + // kernel binary resolution: kernel_id -> GM function_bin_addr mapping // NOTE: Made public for direct access from aicore code uint64_t func_id_to_addr_[RUNTIME_MAX_FUNC_ID]; @@ -183,7 +183,7 @@ class Runtime { private: // Kernel binary tracking for cleanup - void *gm_sm_ptr_; // GM pointer to PTO2 shared memory (device) + void *gm_sm_ptr_; // GM pointer to shared memory (device) void *slot_states_ptr_; // Pointer to ChipTaskSlotState array (scheduler-private, for profiling) ChipStorageTaskArgs orch_args_storage_; // Copy of args for device @@ -292,7 +292,7 @@ class Runtime { // Task graph is now managed by RuntimeContext, not Runtime // ========================================================================= - /** @deprecated Task count is now in PTO2 shared memory */ + /** @deprecated Task count is now in shared memory */ int get_task_count() const { return 0; } /** @deprecated RT2 uses DispatchPayload, not Task. Always returns nullptr. */ diff --git a/src/a5/runtime/host_build_graph/runtime/runtime_types.h b/src/a5/runtime/host_build_graph/runtime/runtime_types.h index 9b32e39a67..6dc0662341 100644 --- a/src/a5/runtime/host_build_graph/runtime/runtime_types.h +++ b/src/a5/runtime/host_build_graph/runtime/runtime_types.h @@ -71,7 +71,7 @@ // Task management // Actual window size is passed at runtime to runtime_create_from_sm(). -// Use pto2_task_slot(sched, task_id) for slot calculation. +// The slot is local_id masked by the window size, which is why the window is a power of two. #define CHIP_TASK_WINDOW_SIZE 16384 // Default task window size (power of 2) // host_build_graph has one ring, and carries no per-ring dimension anywhere: diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index efbd4e2fc1..703b18b835 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -83,8 +83,7 @@ LoopAction SchedulerContext::handle_orchestrator_exit( if (task_count > 0 && completed_tasks_.load(std::memory_order_relaxed) >= task_count) { completed_.store(true, std::memory_order_release); LOG_INFO( - "Thread %d: PTO2 completed tasks %d/%d", thread_idx, completed_tasks_.load(std::memory_order_relaxed), - task_count + "Thread %d: completed tasks %d/%d", thread_idx, completed_tasks_.load(std::memory_order_relaxed), task_count ); return LoopAction::BREAK_LOOP; } @@ -905,7 +904,7 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { } #endif - // Initialize task counters. Task count comes from PTO2 shared memory. + // Initialize task counters. Task count comes from shared memory. if (runtime->get_gm_sm_ptr()) { auto *header = static_cast(runtime->get_gm_sm_ptr()); // Read at one-time boot init, before the SM is reset for the run, so a ring @@ -914,10 +913,10 @@ int32_t SchedulerContext::post_handshake_init(Runtime *runtime) { // CHIP_TASK_WINDOW_SIZE], since the ring cannot hold more than its task // window — so any garbage pattern, negative or positive, leaves the count // at 0, which is the correct value at boot. - int32_t pto2_count = 0; + int32_t window_tasks = 0; int32_t ring_tasks = header->ring.fc.current_task_index.load(std::memory_order_acquire); - if (ring_tasks > 0 && ring_tasks <= CHIP_TASK_WINDOW_SIZE) pto2_count = ring_tasks; - total_tasks_ = pto2_count; + if (ring_tasks > 0 && ring_tasks <= CHIP_TASK_WINDOW_SIZE) window_tasks = ring_tasks; + total_tasks_ = window_tasks; } else { total_tasks_ = 0; } diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index 99cecc119d..7f9537c399 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -227,7 +227,7 @@ void SchedulerContext::complete_slot_task( // The slot key is the 32-bit register token AICore wrote into // dual_issue_slots[task_id & 1].task_id (the DATA_MAIN_BASE value), not // task_id.raw — the latter carries the (ring_id << 32 | local_id) - // encoding and would never match. The PTO2 id travels separately for + // encoding and would never match. The task identity travels separately for // the PmuRecord. pmu_aicpu_complete_record( core_id, thread_idx, static_cast(expected_reg_task_id), slot_state.task->task_id.raw, diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index de8c5fdf41..6b69d1766d 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -154,7 +154,7 @@ class SchedulerContext { // Run all post-orchestration scheduler bookkeeping: // - publishes core assignments to the perf collector (SIMPLER_DFX) - // - latches submitted task count from PTO2 shared memory + // - latches submitted task count from shared memory // - folds inline_completed_tasks into completed_tasks_ // (skipped on fatal error — emergency_shutdown runs instead) // Callers must invoke rt_orchestration_done(rt) before this — that diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 61f5ca99dc..11ae17c929 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -892,7 +892,7 @@ int32_t SchedulerContext::run_resolution_thread(Runtime *runtime, int32_t thread always_assert(sched_ != nullptr); SharedMemoryHeader *header = sched_->sm_header; if (!header) { - LOG_ERROR("PTO2 resolution: header is null"); + LOG_ERROR("resolution: header is null"); return -1; } LOG_INFO("Thread %d: resolution (P) thread starting, serving %d schedulers", thread_idx, active_sched_threads_); @@ -1064,13 +1064,13 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ SharedMemoryHeader *header = sched_->sm_header; if (!header) { - LOG_ERROR("PTO2 dispatch: header is null"); + LOG_ERROR("dispatch: header is null"); return -1; } Handshake *hank = static_cast(runtime->workers); - LOG_INFO("Thread %d: PTO2 dispatch starting with %d cores", thread_idx, core_trackers_[thread_idx].core_num()); + LOG_INFO("Thread %d: dispatch starting with %d cores", thread_idx, core_trackers_[thread_idx].core_num()); int32_t cur_thread_completed = 0; // Non-zero once a scheduler-hang timeout latches; returned in place of the // completed count so the caller still sees the negative error rc while the @@ -1224,7 +1224,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ if (new_total <= PROGRESS_VERBOSE_THRESHOLD || new_total / PROGRESS_LOG_INTERVAL != prev / PROGRESS_LOG_INTERVAL || new_total >= task_count) { LOG_INFO( - "PTO2 progress: completed=%d total=%d (%.1f%%)", new_total, task_count, + "progress: completed=%d total=%d (%.1f%%)", new_total, task_count, 100.0 * new_total / task_count ); } diff --git a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp index e9f7689253..a2acc0738f 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/runtime_init.cpp @@ -203,7 +203,7 @@ bool OrchestratorState::init( auto *orch = this; *orch = OrchestratorState{}; - // A power-of-two window lets pto2_task_slot() mask instead of dividing. + // A power-of-two window lets the slot index mask instead of dividing. always_assert(task_window_size > 0 && (task_window_size & (task_window_size - 1)) == 0); orch->sm_header = reinterpret_cast(sm_base); diff --git a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp index 70350a5603..35e3e044fe 100644 --- a/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/host_build_graph/runtime/shared/shared_memory.cpp @@ -181,7 +181,7 @@ void SharedMemoryHandle::print_layout() { SharedMemoryHeader *h = header; - LOG_DEBUG("=== PTO2 Shared Memory Layout ==="); + LOG_DEBUG("=== Shared Memory Layout ==="); LOG_DEBUG("Base address: %p", sm_base); LOG_DEBUG("Total size: %" PRIu64 " bytes", h->total_size); LOG_DEBUG("Ring:"); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp b/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp index 803227de2a..b621488e3d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/aicore/aicore_executor.cpp @@ -239,7 +239,7 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in } // Performance profiling: record task execution. task_token_raw is - // the PTO2 identity (already in AICore cache from the dispatch + // the task identity (already in AICore cache from the dispatch // payload); reg_task_id is the per-core dispatch token AICore just // read. Host uses reg_task_id as join key vs the AICPU stream. if (chip_swimlane_enabled) { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index 1d105fd88e..bdb95b524c 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -803,7 +803,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { #endif #endif // SIMPLER_ORCH_PROFILING - // Latch task count from PTO2 shared memory to hand off to the + // Latch task count from shared memory to hand off to the // scheduler. The orchestrator's run window (start_time / end_time / // submit_count) is no longer published to shared memory — the // device LOG_INFO "orch_start=… orch_end=… orch_cost=…" line @@ -842,7 +842,7 @@ int32_t AicpuExecutor::run(Runtime *runtime) { ); if (submitted_tasks >= 0) { LOG_INFO( - "PTO2 total submitted tasks = %d, already executed %d tasks", submitted_tasks, + "total submitted tasks = %d, already executed %d tasks", submitted_tasks, sched_ctx_.completed_tasks_count() ); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 04d4e904e8..cbfca9b274 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -1,6 +1,6 @@ # Multi-Ring Buffer Architecture -> Extension to the PTO2 runtime. For the base architecture, see [RUNTIME_LOGIC.md](RUNTIME_LOGIC.md). +> Extension to `tensormap_and_ringbuffer`. For the base architecture, see [RUNTIME_LOGIC.md](RUNTIME_LOGIC.md). ## 1. Problem diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 82f8a1e4bb..2cf89e6993 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -841,7 +841,7 @@ capacity because no task execution/reclaim happens during graph build. --- -## 11. PTO2 Orchestration API +## 11. Orchestration API The orchestration API is defined in `orchestration_api.h`. Orchestration code depends only on this header. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md index 6ac0f2f027..867b4ec87b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md @@ -2,7 +2,7 @@ ## 1. Goal -Define a single, main-branch-aligned specification for PTO2 cluster submission that combines: +Define a single, main-branch-aligned specification for cluster submission that combines: 1. Product requirements (what must be true). 2. Runtime design (how it is implemented on current main baseline). @@ -82,7 +82,7 @@ Rules: 4. `aiv0_kernel_id` and `aiv1_kernel_id` are semantically equivalent. 5. Wrappers are orchestration sugar only (inline in orchestration API); no dedicated runtime ops entries. 6. Submit-contract types are defined once in a shared header-only submit-types surface consumed by orchestration and runtime headers. -7. Invalid submits follow existing PTO2 behavior (`always_assert`), not a new recoverable return-code API. +7. Invalid submits follow existing behavior (`always_assert`), not a new recoverable return-code API. ## 7. Data Model (Requirements + Design) diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/device_log_profiling.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/device_log_profiling.md index df592a6bef..aa9ba62cda 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/device_log_profiling.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/device_log_profiling.md @@ -1,4 +1,4 @@ -# PTO2 Device Log Profiling Guide +# Device Log Profiling Guide ## How to Find Device Logs @@ -21,7 +21,7 @@ A single run produces two profiling blocks in the device log: | Block | Emitted by | Function | Content | | ----- | ---------- | -------- | ------- | | **Orchestrator Profiling** | Thread 3 (orchestrator) | `aicpu_orchestration_entry` | Time breakdown of graph construction on device | -| **PTO2 Scheduler Summary** | Threads 0/1/2 (schedulers) | `SchedulerContext::resolve_and_dispatch` | Per-thread scheduling statistics, phase timing, and lock contention | +| **Scheduler Summary** | Threads 0/1/2 (schedulers) | `SchedulerContext::resolve_and_dispatch` | Per-thread scheduling statistics, phase timing, and lock contention | All timing values are in microseconds (us), converted from AICPU cycle counters. @@ -47,7 +47,7 @@ Thread 3: fanin+ready : 1207.400us (8.3%) Thread 3: finalize+SM : 697.500us (4.8%) Thread 3: scope_end : 364.080us Thread 3: avg/task : 0.874us -Thread 3: PTO2 total submitted tasks = 16704 +Thread 3: total submitted tasks = 16704 ``` ### Field Reference @@ -75,7 +75,7 @@ Thread 3: PTO2 total submitted tasks = 16704 --- -## Block 2: PTO2 Scheduler Summary +## Block 2: Scheduler Summary Each of the 3 scheduler threads (Thread 0, 1, 2) prints its own summary after completing all tasks. The output has two sub-sections: **summary** and **phase breakdown**. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/profiling_levels.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/profiling_levels.md index 862e22d753..4db296a856 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/profiling_levels.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/profiling_levels.md @@ -47,7 +47,7 @@ Each sub-level macro requires `SIMPLER_DFX=1`: **What's compiled:** - Debug/diagnostic logs (always present) -- Progress tracking (`PTO2 progress: completed=...`) +- Progress tracking (`progress: completed=...`) - Stall detection and dump (triggered after the `SCHEDULER_TIMEOUT_MS` wall-clock no-progress budget) - Deadlock/livelock detection (`diagnose_stuck_state`, called on stall) @@ -62,8 +62,8 @@ Each sub-level macro requires `SIMPLER_DFX=1`: - No `sched_start/sched_end/sched_cost` timestamps - No `orch_start/orch_end/orch_cost` timestamps - No `Scheduler summary: total_time=...` -- No `PTO2 total submitted tasks` log -- `PTO2 progress: completed=... total=...` may appear (thread 0 only, at task completion milestones) +- No `total submitted tasks` log +- `progress: completed=... total=...` may appear (thread 0 only, at task completion milestones) --- @@ -161,7 +161,7 @@ captured at chip_swimlane_level >= 3) and `deps.json`; consume them via **Log output (per orchestrator thread, normal run):** the orchestrator phase breakdown, followed by the `orch_start/orch_end/orch_cost` line and the -`PTO2 total submitted tasks` line — all gated by `SIMPLER_ORCH_PROFILING`. This +`total submitted tasks` line — all gated by `SIMPLER_ORCH_PROFILING`. This level adds orchestrator-side logs only; the scheduler side is unchanged from Level 1 (add `SIMPLER_SCHED_PROFILING` / Level 2 for scheduler detail). @@ -180,7 +180,7 @@ Thread X: finalize+SM : XXXus (XX.X%) work=XXXus wait=XXXus atomics=XXX Thread X: scope_end : XXXus atomics=XXX Thread X: avg/task : XXXus Thread X: orch_start=XXX orch_end=XXX orch_cost=XXXus -PTO2 total submitted tasks = XXX, already executed XXX tasks +total submitted tasks = XXX, already executed XXX tasks ``` **Note:** Orchestrator logs always print when `SIMPLER_ORCH_PROFILING=1`, regardless of `enable_chip_swimlane` flag. @@ -240,7 +240,7 @@ header just like on onboard. | 3 | + Scheduler phases (`SCHED_*`) | | 4 | + Orchestrator phases (full) | -At level 1 the AICore record carries the full PTO2 `task_token_raw` +At level 1 the AICore record carries the full `task_token_raw` (`(ring_id << 32) | local_id`), read straight from `LocalContext.async_ctx.task_token.raw` inside the AICore helper — already in cache from the dispatch payload, so no extra GM load. @@ -405,7 +405,7 @@ definitions to runtime headers. | 0 | `SIMPLER_DFX=0` | 0 | No timing output | | 1 | `SIMPLER_DFX=1` | 0 | Host-side `Orch`/`Sched` `[STRACE]` windows only; no device logs | | 2 | `+SIMPLER_SCHED_PROFILING=1` | per sched thread | `sched_start` + phase breakdown + `Scheduler summary` | -| 3 | `+SIMPLER_ORCH_PROFILING=1` | per orch thread | Orchestrator phase breakdown + `orch_start` + `PTO2 total` | +| 3 | `+SIMPLER_ORCH_PROFILING=1` | per orch thread | Orchestrator phase breakdown + `orch_start` + `total submitted tasks` | | 4 | `+SIMPLER_TENSORMAP_PROFILING=1` | per orch thread | + TensorMap lookup stats (4 lines) | --- diff --git a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index 384028ccbf..16d9004458 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -289,14 +289,14 @@ static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedM return 0; } - void *pto2_sm = runtime->get_gm_sm_ptr(); - if (pto2_sm == nullptr) { + void *device_sm = runtime->get_gm_sm_ptr(); + if (device_sm == nullptr) { return 0; } - int hdr_rc = api->copy_from_device(host_header, pto2_sm, sizeof(SharedMemoryHeader)); + int hdr_rc = api->copy_from_device(host_header, device_sm, sizeof(SharedMemoryHeader)); if (hdr_rc != 0) { - LOG_WARN("Failed to copy PTO2 header from device"); + LOG_WARN("Failed to copy the shared-memory header from device"); return 0; } @@ -673,7 +673,7 @@ static void apply_orch_sched_env_flags(Runtime *runtime) { ); } -// per-(cid,config): reserve and acquire the static device pools. GM heap, PTO2 +// per-(cid,config): reserve and acquire the static device pools. GM heap, shared memory // shared memory, and the prebuilt runtime arena all live in one backing // allocation; setup_static_arena reserves the three regions and commits in one // shot. The runtime-arena size is recovered by replaying the (pure, cheap) @@ -705,7 +705,7 @@ static bool ensure_static_arenas( out->gm_sm = api->acquire_pooled_gm_sm(); int64_t t_sm_end = _now_ms(); if (out->gm_sm == nullptr) { - LOG_ERROR("Failed to acquire pooled PTO2 shared memory"); + LOG_ERROR("Failed to acquire pooled shared memory"); return false; } @@ -844,7 +844,7 @@ static bool build_and_cache_prebuilt_arena( /** * Per-run binding: build device-side argument storage (tensor copy-out, GM - * heap, PTO2 shared memory) and publish it to the runtime. Assumes the + * heap, shared memory) and publish it to the runtime. Assumes the * callable-side state (kernel binaries, orch SO bytes, func/config names) * is already populated by register_callable_impl. * @@ -1033,7 +1033,7 @@ extern "C" int validate_runtime_impl(Runtime *runtime, const HostApi *api, int e if (sched_error_code == SIMPLER_ERROR_SCHEDULER_TIMEOUT) { int32_t detail = host_header.sched_stall_detail.load(std::memory_order_acquire); LOG_ERROR( - "PTO2 scheduler timeout sub_class=%s (detail=%d) completed=%d/%d running=%d ready=%d waiting=%d " + "scheduler timeout sub_class=%s (detail=%d) completed=%d/%d running=%d ready=%d waiting=%d " "orch_done=%d stuck_task_id=%" PRId64 " stuck_core=%d", stall_detail_name(detail), detail, host_header.sched_stall_completed.load(std::memory_order_relaxed), host_header.sched_stall_total.load(std::memory_order_relaxed), diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime.h index ac78ded0ae..eb92d7e7d4 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime.h @@ -192,7 +192,7 @@ struct alignas(64) DeviceRuntimeLaunchDesc { // src/a5/runtime/host_build_graph/runtime/runtime.h for rationale. int32_t aicpu_launch_count; - // PTO2 integration: kernel_id -> GM function_bin_addr mapping + // kernel binary resolution: kernel_id -> GM function_bin_addr mapping uint64_t func_id_to_addr_[RUNTIME_MAX_FUNC_ID]; // Serial orchestrator -> scheduler start control. @@ -201,7 +201,7 @@ struct alignas(64) DeviceRuntimeLaunchDesc { // Controlled via SIMPLER_TMR_SERIAL_ORCH_SCHED_ENABLE environment variable. bool serial_orch_sched; - void *gm_sm_ptr_; // GM pointer to PTO2 shared memory (device) + void *gm_sm_ptr_; // GM pointer to shared memory (device) ChipStorageTaskArgs orch_args_storage_; // Copy of args for device // Prebuilt-arena fast path (trb only). Set by the host before rtMemcpy'ing @@ -319,7 +319,7 @@ class Runtime { // Task graph is now managed by RuntimeContext, not Runtime // ========================================================================= - /** @deprecated Task count is now in PTO2 shared memory */ + /** @deprecated Task count is now in shared memory */ int get_task_count() const { return 0; } /** @deprecated RT2 uses DispatchPayload, not Task. Always returns nullptr. */ diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h index 71ac588a28..0d61599e31 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/runtime_types.h @@ -61,7 +61,7 @@ // Task management // NOTE: CHIP_TASK_WINDOW_SIZE is now a per-ring default value. // Actual window size is passed at runtime to runtime_reserve_layout(). -// Use pto2_task_slot(sched, task_id) for slot calculation. +// The slot is local_id masked by the window size, which is why the window is a power of two. #define CHIP_TASK_WINDOW_SIZE 16384 // Default per-ring task window size (power of 2) // Multi-ring: number of independent ring layers (HeapRing + TaskRing + DepPool per layer) diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 6388461101..8b2b613b0a 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -86,8 +86,7 @@ LoopAction SchedulerContext::handle_orchestrator_exit( if (task_count > 0 && completed_tasks_.load(std::memory_order_relaxed) >= task_count) { completed_.store(true, std::memory_order_release); LOG_INFO( - "Thread %d: PTO2 completed tasks %d/%d", thread_idx, completed_tasks_.load(std::memory_order_relaxed), - task_count + "Thread %d: completed tasks %d/%d", thread_idx, completed_tasks_.load(std::memory_order_relaxed), task_count ); return LoopAction::BREAK_LOOP; } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index bb93f1a4fc..f7a054f818 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -242,8 +242,8 @@ void SchedulerContext::complete_slot_task( if (is_pmu_enabled()) { // Slot key must be the 32-bit register token AICore wrote into // dual_issue_slots[task_id & 1].task_id (= DATA_MAIN_BASE value). - // task_id.raw is the full PTO2 (ring_id<<32|local_id) encoding — - // matching on that would never hit. Pass the PTO2 id separately + // task_id.raw is the full (ring_id<<32|local_id) encoding — + // matching on that would never hit. Pass the task identity separately // for the PmuRecord. pmu_aicpu_complete_record( core_id, thread_idx, static_cast(expected_reg_task_id), slot_state.task->task_id.raw, diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 32349f4797..a6c7106edd 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -109,7 +109,7 @@ class SchedulerContext { // Run all post-orchestration scheduler bookkeeping: // - publishes core assignments to the perf collector (SIMPLER_DFX) - // - latches submitted task count from PTO2 shared memory + // - latches submitted task count from shared memory // - folds inline_completed_tasks into completed_tasks_ // - flips orchestrator_done_ and triggers core transition // (skipped on fatal error — emergency_shutdown runs instead) diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 43ca085a19..e9a0e3cc6a 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -845,13 +845,13 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ SharedMemoryHeader *header = sched_->sm_header; if (!header) { - LOG_ERROR("PTO2 dispatch: header is null"); + LOG_ERROR("dispatch: header is null"); return -1; } Handshake *hank = static_cast(runtime->dev.workers); - LOG_INFO("Thread %d: PTO2 dispatch starting with %d cores", thread_idx, tracker.core_num()); + LOG_INFO("Thread %d: dispatch starting with %d cores", thread_idx, tracker.core_num()); int32_t cur_thread_completed = 0; // Non-zero once a scheduler-hang timeout latches; returned in place of the // completed count so the caller still sees the negative error rc while the @@ -1004,7 +1004,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ if (new_total <= PROGRESS_VERBOSE_THRESHOLD || new_total / PROGRESS_LOG_INTERVAL != prev / PROGRESS_LOG_INTERVAL || new_total >= task_count) { LOG_INFO( - "PTO2 progress: completed=%d total=%d (%.1f%%)", new_total, task_count, + "progress: completed=%d total=%d (%.1f%%)", new_total, task_count, 100.0 * new_total / task_count ); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp index 132e298a2d..8d41f3b657 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/shared_memory.cpp @@ -186,7 +186,7 @@ void SharedMemoryHandle::print_layout() { SharedMemoryHeader *h = header; - LOG_DEBUG("=== PTO2 Shared Memory Layout ==="); + LOG_DEBUG("=== Shared Memory Layout ==="); LOG_DEBUG("Base address: %p", sm_base); LOG_DEBUG("Total size: %" PRIu64 " bytes", h->total_size); LOG_DEBUG("Ring depth: %d", CHIP_MAX_RING_DEPTH); diff --git a/src/common/hierarchical/orchestrator.h b/src/common/hierarchical/orchestrator.h index f04da7bca0..219cc4930f 100644 --- a/src/common/hierarchical/orchestrator.h +++ b/src/common/hierarchical/orchestrator.h @@ -178,7 +178,7 @@ class Orchestrator { // Non-blocking: `scope_end` walks the scope's tasks and releases one // ref per task, returning immediately. Actual CONSUMED transitions // happen asynchronously as each task's consumer count reaches - // threshold (mirrors L2's `pto2_scope_end`). The owning run fence + // threshold (mirrors the chip runtime's `rt_scope_end`). The owning run fence // provides the synchronous completion boundary. void scope_begin(); void scope_end(); diff --git a/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h b/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h index 4a3bebeb01..1f4cb64485 100644 --- a/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h +++ b/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h @@ -217,7 +217,7 @@ void chip_swimlane_aicpu_record_sched_phase( * @param thread_idx Scheduler thread that completed the task * @param complete_time Timestamp sampled immediately before completion propagation * @param loop_iter Current scheduler-loop iteration number - * @param task_id Full PTO2 task identity + * @param task_id Full task identity */ void chip_swimlane_aicpu_record_dummy_task( int thread_idx, uint64_t complete_time, uint32_t loop_iter, uint64_t task_id @@ -229,7 +229,7 @@ void chip_swimlane_aicpu_record_dummy_task( * @param thread_idx Scheduler thread that completed the task * @param complete_time Timestamp sampled immediately before completion propagation * @param loop_iter Current scheduler-loop iteration number - * @param task_id Full PTO2 task identity + * @param task_id Full task identity */ void chip_swimlane_aicpu_record_predicated_skip( int thread_idx, uint64_t complete_time, uint32_t loop_iter, uint64_t task_id @@ -260,7 +260,7 @@ void chip_swimlane_aicpu_set_orch_thread_idx(int thread_idx); * * @param start_time Submit start timestamp * @param end_time Submit end timestamp - * @param task_id Task identifier. For tensormap_and_ringbuffer, full PTO2 + * @param task_id Task identifier. For tensormap_and_ringbuffer, full * encoding: (ring_id << 32) | local_id, enabling * cross-view correlation between orchestrator and * scheduler swimlanes. diff --git a/src/common/platform/include/common/args_dump.h b/src/common/platform/include/common/args_dump.h index 5729cd9500..c19dec0964 100644 --- a/src/common/platform/include/common/args_dump.h +++ b/src/common/platform/include/common/args_dump.h @@ -94,7 +94,7 @@ constexpr uint8_t ARGS_DUMP_MAX_FUNC_IDS = 3; */ struct alignas(64) ArgsDumpRecord { // === Cache line 1 (64B) === - uint64_t task_id; // PTO2 encoding or plain task index + uint64_t task_id; // TaskId encoding or plain task index uint8_t role; // ArgsDumpRole (formal callable signature) uint8_t stage; // ArgsDumpStage (before/after execution) uint8_t ndims; // Number of dimensions diff --git a/src/common/platform/include/common/chip_swimlane_profiling.h b/src/common/platform/include/common/chip_swimlane_profiling.h index 0773d2920c..5b6ccc4534 100644 --- a/src/common/platform/include/common/chip_swimlane_profiling.h +++ b/src/common/platform/include/common/chip_swimlane_profiling.h @@ -138,7 +138,7 @@ static_assert(sizeof(ChipSwimlaneAicpuTaskRecord) == 32, "ChipSwimlaneAicpuTaskR * * Two identity fields with different roles: * - * - `task_token_raw` — the PTO2 task identity `(ring_id << 32) | local_id`. + * - `task_token_raw` — the task identity `(ring_id << 32) | local_id`. * Per-task unique. AICore reads it from * `LocalContext.async_ctx.task_token.raw` (already in the dispatch * payload's cache line). The host pulls it from here as the canonical @@ -522,7 +522,7 @@ enum class ChipSwimlaneSchedPhaseKind : uint32_t { // early-dispatch hits. tasks_processed = # consumers visited. // Separate-lane (Worker View pid=4 AICPU_N) DummyTask = 7, // Per-dummy identity marker (zero-width). phase_data.dummy_task - // carries the local/ring components of the full PTO2 identity. + // carries the local/ring components of the full task identity. Drain = 8, // handle_drain_mode outer: the sync_start stop-the-world drain // (ack barrier + availability + parallel stage + finalize). // One bar per dispatch-loop iteration that enters the drain, @@ -608,7 +608,7 @@ static_assert(sizeof(ChipSwimlaneAicpuSchedPhaseRecord) == 64, "ChipSwimlaneAicp struct ChipSwimlaneAicpuOrchPhaseRecord { uint64_t start_time; // Submit start timestamp uint64_t end_time; // Submit end timestamp - uint64_t task_id; // Full PTO2 encoding (ring_id << 32) | local_id + uint64_t task_id; // Full TaskId encoding (ring_id << 32) | local_id uint32_t submit_idx; // Monotonic submit counter uint32_t _pad; // 32B alignment padding }; diff --git a/src/common/platform/include/common/dep_gen.h b/src/common/platform/include/common/dep_gen.h index a9aaaa0f1d..fa9be0a035 100644 --- a/src/common/platform/include/common/dep_gen.h +++ b/src/common/platform/include/common/dep_gen.h @@ -104,7 +104,7 @@ enum DepGenRecordFlags : uint32_t { * blob covers exactly two cache lines instead of straddling three. */ struct DepGenRecord { - uint64_t task_id; // PTO2 encoding (ring_id << 32) | local_id + uint64_t task_id; // TaskId encoding (ring_id << 32) | local_id uint32_t flags; // DepGenRecordFlags bitmask uint16_t tensor_count; // number of valid ChipTensor slots uint16_t explicit_dep_count; // number of valid explicit_dep slots diff --git a/src/common/platform/onboard/host/device_runner_base.cpp b/src/common/platform/onboard/host/device_runner_base.cpp index ebb2c57199..df85546b27 100644 --- a/src/common/platform/onboard/host/device_runner_base.cpp +++ b/src/common/platform/onboard/host/device_runner_base.cpp @@ -107,7 +107,7 @@ HostRuntimeTimeoutConfig resolve_onboard_timeout_config() { 0; if (host_timeout_env_set && order_status != RuntimeTimeoutOrderStatus::OK) { LOG_WARN( - "Ignoring PTO2 timeout env overrides: %s (scheduler=%d ms, op_execute=%llu us, stream_sync=%d ms)", + "Ignoring timeout env overrides: %s (scheduler=%d ms, op_execute=%llu us, stream_sync=%d ms)", runtime_timeout_order_status_name(order_status), cfg.scheduler_timeout_ms, (unsigned long long)cfg.op_execute_timeout_us, cfg.stream_sync_timeout_ms ); @@ -323,7 +323,7 @@ int DeviceRunnerBase::setup_static_arena( LOG_ERROR("arena bank %u is outside [0, %zu)", arena_bank, arena_banks_.size()); return PTO_RUNTIME_ERR_INTERNAL; } - // Three independent device_malloc'd buffers: GM heap, PTO2 SM, prebuilt + // Three independent device_malloc'd buffers: GM heap, shared memory, prebuilt // runtime arena. Split out from a single large allocation because the // combined size can exceed the device allocator's largest contiguous // block. Each arena commits exactly one region, so its base() is the @@ -1431,7 +1431,7 @@ int DeviceRunnerBase::finalize_common_impl(bool abandon_device_resources) { aicpu_seen_callable_ids_.clear(); aicpu_dlopen_total_ = 0; - // Release the three per-Worker pooled arenas (GM heap, PTO2 SM, optional + // Release the three per-Worker pooled arenas (GM heap, shared memory, optional // trb prebuilt runtime arena — each its own device_malloc). Must precede // mem_alloc_.finalize() so the arenas free through the still-live // allocator, not after it. diff --git a/src/common/platform/onboard/host/device_runner_base.h b/src/common/platform/onboard/host/device_runner_base.h index 74091fe68c..66dbba736c 100644 --- a/src/common/platform/onboard/host/device_runner_base.h +++ b/src/common/platform/onboard/host/device_runner_base.h @@ -14,7 +14,7 @@ * * This module owns the host-side state and methods that are identical * between the two onboard arches today: - * - The `MemoryAllocator` and the three `DeviceArena`s (gm heap, PTO2 + * - The `MemoryAllocator` and the three `DeviceArena`s (gm heap, shared memory * SM, runtime arena) backing the per-Worker pooled regions. * - The trivial tensor-memory wrappers (`allocate_tensor`, * `free_tensor`, `copy_*_device`). @@ -167,7 +167,7 @@ class DeviceRunnerBase { virtual void unregister_device_memory_from_host(void *dev_ptr) { (void)dev_ptr; } /** - * Commit the three per-Worker pooled regions (PTO2 GM heap, PTO2 + * Commit the three per-Worker pooled regions (GM heap, shared * shared memory, trb prebuilt runtime arena) as three independent * device allocations. Must be called before any `acquire_pooled_*`. * Idempotent on identical (or smaller) sizes; an equal-or-smaller @@ -188,7 +188,7 @@ class DeviceRunnerBase { int setup_static_arena(uint32_t arena_bank, size_t gm_heap_size, size_t gm_sm_size, size_t runtime_arena_size); /** - * Return the pooled GM heap / PTO2 SM / runtime arena base pointer of the + * Return the pooled GM heap / shared memory / runtime arena base pointer of the * selected arena bank. `setup_static_arena` (arch subclass) must have * already committed the relevant region on that bank; otherwise returns * nullptr. The runtime arena accessor is trb-only — hbg's diff --git a/src/common/platform/shared/aicpu/dep_gen_collector_aicpu.cpp b/src/common/platform/shared/aicpu/dep_gen_collector_aicpu.cpp index 4ad38d1e5a..f2efd3f591 100644 --- a/src/common/platform/shared/aicpu/dep_gen_collector_aicpu.cpp +++ b/src/common/platform/shared/aicpu/dep_gen_collector_aicpu.cpp @@ -317,7 +317,7 @@ void dep_gen_aicpu_record_submit( } // Per-subslot kernel ids (AIC, AIV0, AIV1). The orchestrator owns the - // identity-side of the swimlane join: with task_id (PTO2 raw) + kernel_id + // identity-side of the swimlane join: with task_id (TaskId raw) + kernel_id // captured here, the host post-processor can name every AICore record. // Inactive subslots stay at INVALID_KERNEL_ID (-1); the caller is expected // to pass that sentinel rather than 0. diff --git a/src/common/platform/sim/host/device_runner_base.cpp b/src/common/platform/sim/host/device_runner_base.cpp index a440a54b2e..8653d0a717 100644 --- a/src/common/platform/sim/host/device_runner_base.cpp +++ b/src/common/platform/sim/host/device_runner_base.cpp @@ -128,7 +128,7 @@ int SimDeviceRunnerBase::setup_static_arena( ) { if (arena_bank >= arena_banks_.size()) return PTO_RUNTIME_ERR_INTERNAL; ArenaBank &bank = this->arena_bank(arena_bank); - // Three independent device_malloc'd buffers: GM heap, PTO2 SM, prebuilt + // Three independent device_malloc'd buffers: GM heap, shared memory, prebuilt // runtime arena. Split out from a single large allocation because the // combined size can exceed the device allocator's largest contiguous // block. Each arena commits exactly one region, so its base() is the diff --git a/src/common/platform/sim/host/device_runner_base.h b/src/common/platform/sim/host/device_runner_base.h index 3341351687..d48e623044 100644 --- a/src/common/platform/sim/host/device_runner_base.h +++ b/src/common/platform/sim/host/device_runner_base.h @@ -354,7 +354,7 @@ class SimDeviceRunnerBase { }; std::array graph_definition_blocks_{}; - // Each arena bank backs the three pooled regions (PTO2 GM heap / PTO2 + // Each arena bank backs the three pooled regions (GM heap / shared // shared memory / trb prebuilt runtime arena) for one pipeline slot. They // are separate allocations because the combined size can exceed the device // allocator's largest contiguous block. Released explicitly in finalize() diff --git a/src/common/task_interface/task_args.h b/src/common/task_interface/task_args.h index 2276c7fbb7..3fdc3a801b 100644 --- a/src/common/task_interface/task_args.h +++ b/src/common/task_interface/task_args.h @@ -181,7 +181,7 @@ using TaskArgs = TaskArgsTpl; // L2 runtime ABI: fixed POD matching runtime.so byte-for-byte, and the sole ChipTensor-typed args // container — the materialized form a chip child decodes the L3->L2 Tensor blob into, just before -// pto2_run_runtime. +// simpler_run. using ChipStorageTaskArgs = TaskArgsTpl; // ============================================================================ diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 57df448556..bf7d0fc0e9 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -842,7 +842,7 @@ add_a2a3_test(test_a2a3_fatal a2a3/test_a2a3_fatal.cpp) add_a2a3_test(test_task_timing_slots a2a3/test_task_timing_slots.cpp) add_a2a3_test(test_acl_hal_device common/test_acl_hal_device.cpp) -# PTO2 runtime-linked tests +# runtime-linked tests add_a2a3_runtime_test(test_task_allocator a2a3/test_task_allocator.cpp) add_a2a3_runtime_test(test_scope_deadlock_detection common/test_scope_deadlock_detection.cpp) set(HBG_RUNTIME_DIR ${CMAKE_SOURCE_DIR}/../../../src/a2a3/runtime/host_build_graph/runtime) diff --git a/tests/ut/cpp/a2a3/test_a2a3_fatal.cpp b/tests/ut/cpp/a2a3/test_a2a3_fatal.cpp index 3d7ddd392a..047f01be54 100644 --- a/tests/ut/cpp/a2a3/test_a2a3_fatal.cpp +++ b/tests/ut/cpp/a2a3/test_a2a3_fatal.cpp @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * Unit tests for PTO2 A2A3 fatal error handling. + * Unit tests for a2a3 fatal error handling. * * Tests API short-circuit after fatal state, explicit fatal routing, * and allocation with invalid arguments. diff --git a/tests/ut/cpp/common/test_trb_runtime_temp_buffer.cpp b/tests/ut/cpp/common/test_trb_runtime_temp_buffer.cpp index e7af56e0e3..307e40150b 100644 --- a/tests/ut/cpp/common/test_trb_runtime_temp_buffer.cpp +++ b/tests/ut/cpp/common/test_trb_runtime_temp_buffer.cpp @@ -318,7 +318,7 @@ TEST_F(TrbRuntimeTempBufferTest, FailedExecutionWithoutDeviceStatusSkipsTensorCo ASSERT_EQ(runtime.tensor_leases_.size(), 1u); std::memset(runtime.tensor_leases_[0].dev_ptr, 0x2a, output.size()); - // A stream/bind failure may happen before the device publishes a PTO2 + // A stream/bind failure may happen before the device publishes a // status. The one D2H is the diagnostic header; tensor data stays untouched. EXPECT_EQ(validate_runtime_impl(&runtime, &api_, -1), 0); EXPECT_EQ(fake_.copy_from_count, 1); diff --git a/tests/ut/py/test_resource_failure_summary.py b/tests/ut/py/test_resource_failure_summary.py index 0fb5f9f08a..f38187169d 100644 --- a/tests/ut/py/test_resource_failure_summary.py +++ b/tests/ut/py/test_resource_failure_summary.py @@ -45,7 +45,7 @@ def test_emit_resource_failure_summary_prints_nodeid_and_annotation(capsys): "line1\n" "E RuntimeError: run_prepared failed with code 507018\n" "simpler runtime failed: orch_error_code=0 sched_error_code=100 runtime_status=-100\n" - "PTO2 scheduler timeout sub_class=S1:running-stalled\n" + "scheduler timeout sub_class=S1:running-stalled\n" ), duration_s=12.34, nodeid="tests/st/runtime_fatal_codes/test_probe.py::test_bad[param]", @@ -69,7 +69,7 @@ def test_emit_resource_failure_summary_prints_nodeid_and_annotation(capsys): assert "line1" not in out assert "RuntimeError: run_prepared failed with code 507018" not in out assert "simpler runtime failed: orch_error_code=0 sched_error_code=100 runtime_status=-100" not in out - assert "PTO2 scheduler timeout sub_class=S1:running-stalled" not in out + assert "scheduler timeout sub_class=S1:running-stalled" not in out assert "standalone pass" not in out diff --git a/tests/ut/py/test_worker/test_host_worker.py b/tests/ut/py/test_worker/test_host_worker.py index 11bc91c9a1..4bccb69c77 100644 --- a/tests/ut/py/test_worker/test_host_worker.py +++ b/tests/ut/py/test_worker/test_host_worker.py @@ -7330,7 +7330,7 @@ def test_user_nested_scope_binding_is_exposed(self): hw.init() def orch(o, args, cfg): - # Raw calls — match L2's pto2_scope_begin / pto2_scope_end. + # Raw calls — match the chip runtime's rt_scope_begin / rt_scope_end. o.scope_begin() o.scope_end() # Context-manager form. From 295217ff25ae03e36f46c1d251dea021c8f1fc9b Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:54:01 -0700 Subject: [PATCH 20/22] Docs: record that the PTO2 retirement is down to its Tier-C knobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rule 10 still described a ~6.6k-occurrence backlog across 367 files. There are 129 occurrences left and every one is deliberate: the three `PTO2_RING_*` environment knobs, `PTO2_MANUAL_MAX_SEQ` (pypto-lib's, documented here only), and the rule's own illustrations of the banned spelling. Left as it was, the rule would send the next contributor looking for a sweep that no longer exists — and, worse, would let a genuine regression look like leftover backlog. A `PTO2` match now means a Tier-C knob or a defect. Two additions come from doing the work rather than from restating it: - Why an environment variable is the worst case in Tier C: an unrecognised name is ignored, so the rename fails silently in production instead of at compile time. Two knobs had already drifted that way and were dead when found. - What to check a target name against. The bare name may be held by the host orchestrator under `src/common/hierarchical/` (rule 13 decides who keeps it), by an external header the same translation units include (CANN defines `ALIGN_UP`), or — for an object-like macro — by any declaration at all, since a macro rewrites its identifier textually instead of colliding where the compiler can report it. Checking only the first of the three is how `#define HEAP_SIZE` came to rewrite a unit test's `static constexpr uint64_t HEAP_SIZE = 4096`. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/rules/codestyle.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/.claude/rules/codestyle.md b/.claude/rules/codestyle.md index db7f75b9c1..3583cea2e1 100644 --- a/.claude/rules/codestyle.md +++ b/.claude/rules/codestyle.md @@ -125,10 +125,15 @@ The project is **`simpler`** ("Simple Runtime"); its two runtimes are `host_build_graph` and `tensormap_and_ringbuffer`. **"PTO Runtime", "PTO Runtime2", and a bare "Runtime2" are not names of anything** — the - `2` never denoted a version any reader can identify. The legacy surface - is far too large to fix at once (snapshot 2026-07-26: ~6.6k `PTO2` - occurrences across 367 files, plus 17 prose "PTO Runtime[2]" mentions), - so it is retired in graded steps. Find the current surface with: + `2` never denoted a version any reader can identify. + + **Tiers A and B are done** (snapshot 2026-08-24: 129 occurrences, down + from ~6.6k across 367 files on 2026-07-26). What remains is Tier C and + nothing else: `PTO2_RING_TASK_WINDOW` / `PTO2_RING_HEAP` / + `PTO2_RING_DEP_POOL`, plus `PTO2_MANUAL_MAX_SEQ`, which pypto-lib owns + and this repo only documents. **So a `PTO2` match now means either a + Tier-C knob or a defect** — there is no longer a backlog to work + through. Find the current surface with: ```bash grep -rIn -E "PTO Runtime2?|PTO2|pto2_|pto_runtime2" --include=* . | grep -v '^\./\.git/' @@ -142,7 +147,15 @@ | **B — internal identifiers** | `PTO2Foo` types, `pto2_*` functions, internal `PTO2_*` macros and enumerators, `pto_*.h` / `pto_runtime2*.cpp` file names | Rename **only when you are already modifying that code**, per rule 9, and finish the identifier you started (below). | | **C — external contracts** | `PTO2_RING_*` and other `runtime_env` knobs (documented in `MULTI_RING.md`, set by pypto-lib / pypto-serving), `extern "C"` symbols in `runtime_c_api.h`, on-wire / serialized names | **Exempt until a migration ships.** Renaming these breaks callers in other repos. Ask the user before touching one; land it only with a compatibility alias or a coordinated cross-repo change. | - Two constraints make the difference between progress and churn: + An environment variable is the worst case in Tier C and the reason it + exists: an unrecognised name is *ignored*, so the rename fails silently + in production instead of at compile time. Two knobs that had already + drifted this way were found dead during the Tier-B work — a benchmark + setting `PTO2_SERIAL_ORCH_SCHED` against a runtime reading + `SIMPLER_TMR_SERIAL_ORCH_SCHED_ENABLE`, and an error hint naming + `PTO2_SCHEDULER_TIMEOUT_MS` instead of `SIMPLER_SCHEDULER_TIMEOUT_MS`. + + Three constraints make the difference between progress and churn: - **One identifier, all its occurrences, one commit.** A rename that leaves both spellings alive is worse than no rename — the reader now @@ -153,6 +166,15 @@ - **The count only goes down.** Introducing a new `PTO2`/`PTO Runtime` spelling anywhere — including by copying an existing `pto_*.h` to a new arch — is a defect, not neutral. Rule 9 makes this unconditional. + - **Check the target name against three things, not one.** The bare name + may be taken by the host orchestrator under `src/common/hierarchical/` + (`TensorMap`, `ReadyQueue`, `TaskState`, `HeapRing`, `MAX_SCOPE_DEPTH` + — rule 13 decides who keeps it, and the chip runtime takes `Chip` / + `CHIP_`); by an external header the same translation units include + (CANN defines `ALIGN_UP`); or, for an **object-like macro**, by any + declaration anywhere, because a macro rewrites its identifier textually + rather than colliding where the compiler can say so. Macros therefore + keep a prefix even when the bare name looks free. Do **not** open a repo-wide mechanical rename PR: it collides with every in-flight branch, and a 6k-line diff cannot be reviewed for the handful From 7f16161652d4bec97fdb7ae26414577d6497a192 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:40:27 -0700 Subject: [PATCH 21/22] Refactor: retire the process-wide ring sizing environment variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PTO2_RING_TASK_WINDOW`, `PTO2_RING_HEAP` and `PTO2_RING_DEP_POOL` are no longer read. `CallConfig.runtime_env` carries the same three values per task, and is strictly more expressive: two tasks in one process can hold different ring sizes, which a process-wide export never allowed. So the env layer was not a fallback worth keeping — it was a second way to say the same thing, in a weaker form, and the one spelling of it that could not be renamed. Removing it deletes the whole parsing path from all four `host/runtime_maker.cpp` copies — `trim_copy`, `parse_uint_token`, `apply_env_ring_value[s]` and the scalar / comma-separated grammar they implemented — leaving per-task entries over the compile-time default with nothing in between. `` and (in host_build_graph) `` were there only for that grammar and go with it. Net -283 lines. What replaces it is one warning. An unrecognised environment variable is *ignored*, so a caller that still exports one of these would otherwise run on the 256 MB default with the sizing it asked for nowhere in evidence — and the observable symptom is a heap-exhaustion deadlock that looks like a runtime regression rather than a configuration change. `warn_on_retired_ring_env()` reports each retired name that is set, once per bind, and points at `CallConfig.runtime_env`. It is a migration aid with no behavioral effect; delete it once no caller sets them. **Every diagnostic that advertised these variables had to move with them**, or the removal would have manufactured the same defect this branch already fixed twice (`PTO2_SERIAL_ORCH_SCHED`, `PTO2_SCHEDULER_TIMEOUT_MS`): a hint naming a variable nothing reads. So the device-side ring-buffer and orchestrator deadlock reports, the fanin-spill capacity report, the five `error_hint()` strings, the two capacity tables in `docs/troubleshooting/`, the `running-onboard` triage table, and the runnable command in `docs/dfx/scope-stats.md` now name `runtime_env.ring_task_window` / `ring_heap` / `ring_dep_pool`. Two unit tests assert on that hint text and move with it. `MULTI_RING.md` loses the two env tiers from its precedence ladder and its env-fallback section, which was the only place the comma-separated per-ring grammar was documented. The cross-repo edge is real and named where it matters. The deepseek example's own comment records that pypto-lib's daily CI sizes this network the same way, and `.claude/skills/multi-repo-qwen-setup` documented exporting `PTO2_RING_HEAP=4294967296` for batch-16 qwen prefill — 4 GiB, against a 256 MB default. That skill now gives the values as `CallConfig.runtime_env` fields and states outright that a runner still exporting the old names deadlocks exactly as its own gotcha section describes. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/rules/codestyle.md | 35 ++++-- .claude/rules/running-onboard.md | 2 +- .claude/skills/multi-repo-qwen-setup/SKILL.md | 13 +- docs/dfx/profiling-config-naming.md | 9 +- docs/dfx/scope-stats.md | 17 ++- docs/troubleshooting/device-error-codes.md | 2 +- .../device-error-codes/capacity.md | 10 +- .../test_deepseek_v4_flash_decode.py | 4 +- .../paged_attention_ringbuffer/README.md | 2 +- .../test_paged_attention_ringbuffer.py | 2 +- .../paged_attention_ringbuffer/README.md | 2 +- .../test_paged_attention_ringbuffer.py | 2 +- .../workers/l2/per_task_runtime_env/main.py | 4 +- .../workers/l3/per_task_runtime_env/README.md | 2 +- .../workers/l3/per_task_runtime_env/main.py | 2 +- python/bindings/task_interface.cpp | 2 +- python/simpler/worker.py | 4 +- simpler_setup/scene_test.py | 4 +- .../host_build_graph/host/runtime_maker.cpp | 101 ++++----------- .../orchestrator_core/orchestrator.cpp | 2 +- .../host_build_graph/runtime/ring_buffer.h | 4 +- .../docs/MULTI_RING.md | 24 ++-- .../docs/RUNTIME_LOGIC.md | 4 +- .../host/runtime_maker.cpp | 116 +++--------------- .../runtime/orchestrator.cpp | 2 +- .../runtime/ring_buffer.cpp | 4 +- .../runtime/ring_buffer.h | 12 +- .../host_build_graph/host/runtime_maker.cpp | 101 ++++----------- .../orchestrator_core/orchestrator.cpp | 2 +- .../host_build_graph/runtime/ring_buffer.h | 4 +- .../docs/MULTI_RING.md | 24 ++-- .../docs/RUNTIME_LOGIC.md | 4 +- .../host/runtime_maker.cpp | 116 +++--------------- .../runtime/orchestrator.cpp | 2 +- .../runtime/ring_buffer.cpp | 4 +- .../runtime/ring_buffer.h | 12 +- src/common/runtime_status/error_names.h | 11 +- src/common/task_interface/call_config.h | 6 +- src/common/worker/runtime_c_api.h | 2 +- .../paged_attention/test_paged_attention.py | 2 +- .../paged_attention/test_paged_attention.py | 2 +- tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp | 2 +- tests/ut/cpp/a5/test_orchestrator_fanin.cpp | 2 +- 43 files changed, 203 insertions(+), 480 deletions(-) diff --git a/.claude/rules/codestyle.md b/.claude/rules/codestyle.md index 3583cea2e1..25684a8ca1 100644 --- a/.claude/rules/codestyle.md +++ b/.claude/rules/codestyle.md @@ -127,13 +127,16 @@ "PTO Runtime2", and a bare "Runtime2" are not names of anything** — the `2` never denoted a version any reader can identify. - **Tiers A and B are done** (snapshot 2026-08-24: 129 occurrences, down - from ~6.6k across 367 files on 2026-07-26). What remains is Tier C and - nothing else: `PTO2_RING_TASK_WINDOW` / `PTO2_RING_HEAP` / - `PTO2_RING_DEP_POOL`, plus `PTO2_MANUAL_MAX_SEQ`, which pypto-lib owns - and this repo only documents. **So a `PTO2` match now means either a - Tier-C knob or a defect** — there is no longer a backlog to work - through. Find the current surface with: + **The retirement is complete** (snapshot 2026-08-24: 6.6k occurrences across + 367 files on 2026-07-26 → 0 that this repo reads). Nothing here reads a + `PTO2` name any more: the last of them were the `PTO2_RING_*` ring-sizing + environment variables, retired in favour of per-task + `CallConfig.runtime_env`. What is left in the tree is this rule's own + examples, the warning that fires if a caller still exports a retired ring + variable, prose recording that these names are gone, and + `PTO2_MANUAL_MAX_SEQ` — a pypto-lib knob two example READMEs document and + this repo never reads. **So a new `PTO2` match is a defect** — there is no + backlog to work through. Find the surface with: ```bash grep -rIn -E "PTO Runtime2?|PTO2|pto2_|pto_runtime2" --include=* . | grep -v '^\./\.git/' @@ -145,15 +148,21 @@ | ---- | ---- | ------ | | **A — prose** | Brand mentions in docs, headings, skill descriptions, issue templates, and code comments (`# PTO Runtime2 Profiling Levels`, "the PTO Runtime consists of…") | **Fix on sight, unconditionally.** No compile risk, no contract. Say `simpler`, or name the actual component ("the AICPU orchestrator", "the `tensormap_and_ringbuffer` runtime") when that is what the sentence means. | | **B — internal identifiers** | `PTO2Foo` types, `pto2_*` functions, internal `PTO2_*` macros and enumerators, `pto_*.h` / `pto_runtime2*.cpp` file names | Rename **only when you are already modifying that code**, per rule 9, and finish the identifier you started (below). | - | **C — external contracts** | `PTO2_RING_*` and other `runtime_env` knobs (documented in `MULTI_RING.md`, set by pypto-lib / pypto-serving), `extern "C"` symbols in `runtime_c_api.h`, on-wire / serialized names | **Exempt until a migration ships.** Renaming these breaks callers in other repos. Ask the user before touching one; land it only with a compatibility alias or a coordinated cross-repo change. | + | **C — external contracts** | `runtime_env` knobs, `extern "C"` symbols in `runtime_c_api.h`, on-wire / serialized names | **Exempt until a migration ships.** Renaming these breaks callers in other repos. Ask the user before touching one; land it only with a compatibility alias or a coordinated cross-repo change. | An environment variable is the worst case in Tier C and the reason it exists: an unrecognised name is *ignored*, so the rename fails silently - in production instead of at compile time. Two knobs that had already - drifted this way were found dead during the Tier-B work — a benchmark - setting `PTO2_SERIAL_ORCH_SCHED` against a runtime reading - `SIMPLER_TMR_SERIAL_ORCH_SCHED_ENABLE`, and an error hint naming - `PTO2_SCHEDULER_TIMEOUT_MS` instead of `SIMPLER_SCHEDULER_TIMEOUT_MS`. + in production instead of at compile time. Three knobs proved the point. + Two were found already dead — a benchmark setting `PTO2_SERIAL_ORCH_SCHED` + against a runtime reading `SIMPLER_TMR_SERIAL_ORCH_SCHED_ENABLE`, and an + error hint naming `PTO2_SCHEDULER_TIMEOUT_MS` instead of + `SIMPLER_SCHEDULER_TIMEOUT_MS`. The third case, `PTO2_RING_*`, is the one + to copy: the knob was not renamed but **removed**, because + `CallConfig.runtime_env` already carried the same sizing per task and was + strictly more expressive. Retiring an env knob outright beats renaming it — + and when you do, the runtime should say so on sight rather than silently + fall back, which is what `warn_on_retired_ring_env()` in each + `host/runtime_maker.cpp` is for. Three constraints make the difference between progress and churn: diff --git a/.claude/rules/running-onboard.md b/.claude/rules/running-onboard.md index d1fc1aceab..78fee2ffea 100644 --- a/.claude/rules/running-onboard.md +++ b/.claude/rules/running-onboard.md @@ -175,7 +175,7 @@ for the signature that actually fired:** | `FATAL: Task Allocator Deadlock` / `FATAL: Dependency Pool Deadlock` / `FATAL: Fanin Spill Pool Deadlock` | an allocator or pool could not reclaim enough space | This header identifies the blocked resource, not the root cause. Classify it using the structural/timeout line that follows. | | `Provable head-of-line deadlock` | **proven open-scope structural deadlock** | TRB only: the reclaim head is the oldest task owned by an open scope on that ring. The blocked orchestrator cannot end that scope, so the head cannot become consumed. On A5, classification waits for at least 10 ms without reclaim progress and an exact-watermark publication acknowledgment. | | `No reclaim progress for ~500 ms` / `cannot reclaim space after ~500 ms` | allocator/pool **reclaim timeout** | TRB only — the 500ms backstop (`ALLOC_DEADLOCK_TIMEOUT_CYCLES`) lives in its reclaiming allocators. It proves prolonged lack of reclaim progress, not why progress stopped; check capacity, the dumped head, consumers and scheduler state. | -| `Task Window Exhausted` / `Fanin Capacity Exhausted` / `TensorMap Entry Pool Exhausted` | **capacity**, HBG only | HBG is whole-graph-resident and reclaims nothing mid-run, so this is a sizing verdict reached immediately, not a stall. The line carries used/capacity and the requested amount; raise `PTO2_RING_TASK_WINDOW` / `CHIP_TENSORMAP_POOL_SIZE` or shrink the graph. Inline fanin is hard-capped at `CHIP_MAX_FANIN=128`; HBG has no `PTO2_RING_DEP_POOL`. HBG's graph heap has no knob and cannot exhaust: it is committed after orchestration at the measured size, so a graph too large for the device fails host-side at that commit. | +| `Task Window Exhausted` / `Fanin Capacity Exhausted` / `TensorMap Entry Pool Exhausted` | **capacity**, HBG only | HBG is whole-graph-resident and reclaims nothing mid-run, so this is a sizing verdict reached immediately, not a stall. The line carries used/capacity and the requested amount; raise `runtime_env.ring_task_window` / `CHIP_TENSORMAP_POOL_SIZE` or shrink the graph. Inline fanin is hard-capped at `CHIP_MAX_FANIN=128`; HBG has no dependency spill pool. HBG's graph heap has no knob and cannot exhaust: it is committed after orchestration at the measured size, so a graph too large for the device fails host-side at that commit. | | `Timeout (N cycles): producer/consumers ...` | **SPIN** wait on a specific producer/consumer | `runtime_core.cpp`. | | `HandleTaskTimeout` / `kill aicpu-sd` | **OS op-execute timeout** | STARS/tsdaemon, default 45s (`PLATFORM_OP_EXECUTE_TIMEOUT_US`). **A 45s kill ≠ deadlock** — the op was merely long or stalled. Raise this constant to measure true on-device duration. | | `log_stall_diagnostics` (cores idle + tasks `state=WAIT fanin 0/N` + `completed` frozen) | **forward-progress stall** | No dedicated detector — intermittent races (often contention-triggered) land here and are reaped only by the op-timeout above. | diff --git a/.claude/skills/multi-repo-qwen-setup/SKILL.md b/.claude/skills/multi-repo-qwen-setup/SKILL.md index 2d17db80c7..dc4504fde0 100644 --- a/.claude/skills/multi-repo-qwen-setup/SKILL.md +++ b/.claude/skills/multi-repo-qwen-setup/SKILL.md @@ -161,10 +161,15 @@ log to tell them apart**: 1. **Heap-exhausted deadlock** (default 256 MB ring): the open prefill scope's live-set exceeds one ring → device log prints `FATAL: Task Allocator Deadlock - Heap Exhausted! ... Provable head-of-line`. - Fix: raise the ring. batch-1 needs just over 256 MB; batch-16 needs the - prefill config `PTO2_RING_HEAP=4294967296 PTO2_RING_TASK_WINDOW=131072 - PTO2_RING_DEP_POOL=131072` (heap, task-window, and dep-pool all scale with - batch; default `task_window=16384` also overflows at batch-16). + Fix: raise the ring. batch-1 needs just over 256 MB; batch-16 needs + `ring_heap=4294967296`, `ring_task_window=131072`, `ring_dep_pool=131072` + (heap, task-window, and dep-pool all scale with batch; the default + `task_window=16384` also overflows at batch-16). These are per-task values on + `CallConfig.runtime_env` — the runner has to pass them with the prefill call. + The `PTO2_RING_*` environment variables that used to carry them are no longer + read by simpler, and a stale export is silently ignored (simpler warns), so a + runner still exporting them runs on the 256 MB default and deadlocks exactly + as described above. 2. **Op-execute timeout** (slow/contended prefill): device log prints `HandleTaskTimeout ... timeOut:` and kills `aicpu-sd`. The compiled defaults (`PLATFORM_OP_EXECUTE_TIMEOUT_US`=45 s, diff --git a/docs/dfx/profiling-config-naming.md b/docs/dfx/profiling-config-naming.md index 93155d8bff..460327f31f 100644 --- a/docs/dfx/profiling-config-naming.md +++ b/docs/dfx/profiling-config-naming.md @@ -72,11 +72,10 @@ translation unit (`CHIP_HEAP_SIZE`, `CHIP_ALIGN_UP`). Configuration surface expo to users / CI / external consumers is unified under `SIMPLER_`, and status codes under `SIMPLER_ERROR_*`. -The exception is `PTO2_RING_TASK_WINDOW` / `PTO2_RING_HEAP` / -`PTO2_RING_DEP_POOL`. Those three are read from the environment by name, and other -repositories and shell scripts set them, so renaming them would be ignored -silently rather than failing to compile. They keep the legacy spelling until a -cross-repo migration retires them. +There is no `PTO2_` configuration surface left. `PTO2_RING_TASK_WINDOW` / +`PTO2_RING_HEAP` / `PTO2_RING_DEP_POOL` were the last of it; ring sizing is per +task now, on `CallConfig.runtime_env`, and the runtime warns if one of those names +is still exported so a stale export cannot quietly become the compile-time default. ## Compile-time gates vs runtime emission diff --git a/docs/dfx/scope-stats.md b/docs/dfx/scope-stats.md index 8c3e36b161..91865f32d2 100644 --- a/docs/dfx/scope-stats.md +++ b/docs/dfx/scope-stats.md @@ -43,12 +43,19 @@ schema. The metadata line is also the quickest way to verify per-ring runtime sizing. For example, after running with: +```python +# In the scene test's CASES entry -- ring sizing is per task, so it travels with +# the case rather than with the process: +"config": { + "runtime_env": { + "ring_task_window": [8192, 16384, 131072, 524288], + "ring_heap": [134217728, 268435456, 402653184, 536870912], + "ring_dep_pool": [4096, 8192, 16384, 32768], + } +} +``` + ```bash -CASE=... -NAME=... -PTO2_RING_TASK_WINDOW=8192,16384,131072,524288 \ -PTO2_RING_HEAP=134217728,268435456,402653184,536870912 \ -PTO2_RING_DEP_POOL=4096,8192,16384,32768 \ python "tests/st/${CASE}/test_${NAME}.py" -p a2a3 -d 0 --enable-scope-stats ``` diff --git a/docs/troubleshooting/device-error-codes.md b/docs/troubleshooting/device-error-codes.md index a16ae593a3..de94cd0a0f 100644 --- a/docs/troubleshooting/device-error-codes.md +++ b/docs/troubleshooting/device-error-codes.md @@ -11,7 +11,7 @@ The `error hint:` line that sent you here is the second of a pair: ```text [ERROR] simpler runtime failed: orch_error_code=1 sched_error_code=0 runtime_status=-1 [ERROR] error detail: orch_error_code=1 SCOPE_DEADLOCK - tasks submitted in one scope ... -[ERROR] error hint: raise ring_task_window (PTO2_RING_TASK_WINDOW) or split the scope ... +[ERROR] error hint: raise runtime_env.ring_task_window or split the scope ... ``` `error detail:` gives the **name** and a full description of the mechanism. diff --git a/docs/troubleshooting/device-error-codes/capacity.md b/docs/troubleshooting/device-error-codes/capacity.md index 8955e71005..834c9af9c9 100644 --- a/docs/troubleshooting/device-error-codes/capacity.md +++ b/docs/troubleshooting/device-error-codes/capacity.md @@ -42,12 +42,12 @@ that trips the code. | Runtime | Bottleneck resource | Code | Fix | | ------- | ------------------- | ---- | --- | -| HBG | task window | 3 | raise `ring_task_window` (`PTO2_RING_TASK_WINDOW`), or shrink the graph | -| HBG | inline fanin | 4 | reduce distinct producers to `CHIP_MAX_FANIN` (currently 128) or less; HBG has no `PTO2_RING_DEP_POOL` | +| HBG | task window | 3 | raise `runtime_env.ring_task_window`, or shrink the graph | +| HBG | inline fanin | 4 | reduce distinct producers to `CHIP_MAX_FANIN` (currently 128) or less; HBG has no dependency spill pool | | HBG | TensorMap entries | 11 | increase `CHIP_TENSORMAP_POOL_SIZE`, or reduce registered outputs | -| TRB | open-scope task window | 1 or 3 | raise `PTO2_RING_TASK_WINDOW`, split the scope, or diagnose stalled reclaim | -| TRB | heap | 2 | raise `PTO2_RING_HEAP`, shrink allocations, or diagnose stalled reclaim | -| TRB | dependency pool | 4 | raise `PTO2_RING_DEP_POOL`, cut fanin, or diagnose stalled reclaim | +| TRB | open-scope task window | 1 or 3 | raise `runtime_env.ring_task_window`, split the scope, or diagnose stalled reclaim | +| TRB | heap | 2 | raise `runtime_env.ring_heap`, shrink allocations, or diagnose stalled reclaim | +| TRB | dependency pool | 4 | raise `runtime_env.ring_dep_pool`, cut fanin, or diagnose stalled reclaim | HBG's graph heap is absent from the table because it has no knob and cannot latch code 2: orchestration allocates it out of a virtual window, and the device region diff --git a/examples/a2a3/tensormap_and_ringbuffer/deepseek_v4_flash_decode/test_deepseek_v4_flash_decode.py b/examples/a2a3/tensormap_and_ringbuffer/deepseek_v4_flash_decode/test_deepseek_v4_flash_decode.py index 5e4ae5c0c8..1318cbb341 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/deepseek_v4_flash_decode/test_deepseek_v4_flash_decode.py +++ b/examples/a2a3/tensormap_and_ringbuffer/deepseek_v4_flash_decode/test_deepseek_v4_flash_decode.py @@ -615,8 +615,8 @@ class TestDeepseekV4FlashDecode(SceneTestCase): "device_count": N_RANKS, "num_sub_workers": 0, # Ring sizing for the 43-layer graph. The task window and dep - # pool match pypto-lib's daily CI env for this network - # (PTO2_RING_* 16384 / 16384). The heap is twice that env's + # pool match pypto-lib's daily CI sizing for this network + # (16384 / 16384). The heap is twice the compile-time default's # 1 GiB: the MoE per-expert tile grid is static, so tile scratch # is allocated for all 32 experts of every MoE layer whatever the # routing turns out to be. diff --git a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md index d16cedf0d1..82911af6aa 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md +++ b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md @@ -30,7 +30,7 @@ referenced, the golden check fails or the orchestrator wedges. Two things this example is the reference for: - **Per-case ring sizing through `config.runtime_env`**, not the process-global - `PTO2_RING_*` environment variables. The rings are sized per task, so one + now-retired `PTO2_RING_*` environment variables. The rings are sized per task, so one suite can mix a stress case with normally-sized ones. - **Non-power-of-2 sizes are accepted.** 4 MiB is chosen to keep the stress intent compact, not because the runtime requires a round number. diff --git a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/test_paged_attention_ringbuffer.py b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/test_paged_attention_ringbuffer.py index 2809b08d71..e66f6e2e49 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/test_paged_attention_ringbuffer.py +++ b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_ringbuffer/test_paged_attention_ringbuffer.py @@ -10,7 +10,7 @@ """Paged attention with small ring buffer sizes — stress test for ring rotation/reclamation. Drives per-case ring sizing through ``config.runtime_env`` (ring_task_window / -ring_heap / ring_dep_pool) rather than the process-global PTO2_RING_* env, plus +ring_heap / ring_dep_pool) rather than the process-global PTO2_RING_* env it replaced, plus INOUT tensors, bfloat16, and AIC+AIV mixed execution. """ diff --git a/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md b/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md index 8cce365ba1..05deeb2be2 100644 --- a/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md +++ b/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/README.md @@ -30,7 +30,7 @@ referenced, the golden check fails or the orchestrator wedges. Two things this example is the reference for: - **Per-case ring sizing through `config.runtime_env`**, not the process-global - `PTO2_RING_*` environment variables. The rings are sized per task, so one + now-retired `PTO2_RING_*` environment variables. The rings are sized per task, so one suite can mix a stress case with normally-sized ones. - **Non-power-of-2 sizes are accepted.** 4 MiB is chosen to keep the stress intent compact, not because the runtime requires a round number. diff --git a/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/test_paged_attention_ringbuffer.py b/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/test_paged_attention_ringbuffer.py index c1e16bab24..e734e97048 100644 --- a/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/test_paged_attention_ringbuffer.py +++ b/examples/a5/tensormap_and_ringbuffer/paged_attention_ringbuffer/test_paged_attention_ringbuffer.py @@ -10,7 +10,7 @@ """Paged attention with small ring buffer sizes — stress test for ring rotation/reclamation. Drives per-case ring sizing through ``config.runtime_env`` (ring_task_window / -ring_heap / ring_dep_pool) rather than the process-global PTO2_RING_* env, plus +ring_heap / ring_dep_pool) rather than the process-global PTO2_RING_* env it replaced, plus INOUT tensors, bfloat16, and AIC+AIV mixed execution. """ diff --git a/examples/workers/l2/per_task_runtime_env/main.py b/examples/workers/l2/per_task_runtime_env/main.py index ab23b488a0..23f84e1470 100644 --- a/examples/workers/l2/per_task_runtime_env/main.py +++ b/examples/workers/l2/per_task_runtime_env/main.py @@ -13,7 +13,7 @@ a different ``CallConfig.runtime_env`` (ring buffer sizing) — covering both the scalar form (one value broadcast to every ring) and the per-ring form (each scope-depth ring sized independently). Ring sizing is a per-run knob carried on -``CallConfig`` — no process-wide ``PTO2_RING_*`` env export needed, and each +``CallConfig`` — the only place ring sizing lives, and each ``worker.run`` binds its ring buffers from the config it was handed. Each runtime_env field takes EITHER a scalar (broadcast to every ring) OR a @@ -68,7 +68,7 @@ RING_FIELDS = ("ring_task_window", "ring_heap", "ring_dep_pool") # (label, runtime_env dict or None). None => no override; falls back to the -# PTO2_RING_* env var / compile-time default. Same kernel + same inputs run +# compile-time default. Same kernel + same inputs run # under every sizing, so all of them produce identical (correct) output. RING_CONFIGS = [ # Scalar form: one value broadcast to every ring. diff --git a/examples/workers/l3/per_task_runtime_env/README.md b/examples/workers/l3/per_task_runtime_env/README.md index a7bfcdb06b..5c5bd64631 100644 --- a/examples/workers/l3/per_task_runtime_env/README.md +++ b/examples/workers/l3/per_task_runtime_env/README.md @@ -8,7 +8,7 @@ each need a different ring footprint. ## What it shows Before this knob, every L2 dispatched from one L3 shared the process-wide -`PTO2_RING_*` env and could not be sized independently. Now each +process-wide `PTO2_RING_*` env, since retired, and could not be sized independently. Now each `submit_next_level` gets its own `CallConfig`: Each spec sets `ring_task_window` / `ring_heap` / `ring_dep_pool` to a scalar diff --git a/examples/workers/l3/per_task_runtime_env/main.py b/examples/workers/l3/per_task_runtime_env/main.py index 1babcae327..ae268d7528 100644 --- a/examples/workers/l3/per_task_runtime_env/main.py +++ b/examples/workers/l3/per_task_runtime_env/main.py @@ -14,7 +14,7 @@ several heterogeneous L2 tasks in one launch, and each L2 needs a different ring footprint (a heavy task wants a big heap / wide window; a light one is fine with the default). Before this knob, all L2 tasks in one L3 launch shared -the process-wide ``PTO2_RING_*`` env and could not be sized independently. +the process-wide ``PTO2_RING_*`` env, since retired, and could not be sized independently. The demo dispatches three L2 tasks: two use the scalar form (one ring value broadcast to every ring), and a third passes 4-entry lists for diff --git a/python/bindings/task_interface.cpp b/python/bindings/task_interface.cpp index 484c817640..d5655460c7 100644 --- a/python/bindings/task_interface.cpp +++ b/python/bindings/task_interface.cpp @@ -2538,7 +2538,7 @@ NB_MODULE(_task_interface, m) { return os.str(); }); - // --- RuntimeEnv (per-task PTO2_RING_* overrides; nested under CallConfig.runtime_env) --- + // --- RuntimeEnv (per-task ring sizing; nested under CallConfig.runtime_env) --- // Each ring resource is exposed as ONE property that accepts either an int // (broadcast to every ring) or a list of RUNTIME_ENV_RING_COUNT ints // (per-ring). The value always reads back as a list — the wire layout is the diff --git a/python/simpler/worker.py b/python/simpler/worker.py index 45777523b1..9a648a9575 100644 --- a/python/simpler/worker.py +++ b/python/simpler/worker.py @@ -458,7 +458,7 @@ def _shm_name(token: str, suffix: str): # Startup readiness bound. A child that neither reports INIT_READY/INIT_FAILED # nor exits within this window is treated as hung and startup is aborted. # Generous by default so a legitimately slow device/runtime init (large -# PTO2_RING_HEAP, cold arena build) is never falsely reaped; override per Worker +# runtime_env.ring_heap, cold arena build) is never falsely reaped; override per Worker # via the `startup_timeout_s` config kwarg. The point is to bound *hangs*, not # to police slow-but-progressing init. _STARTUP_TIMEOUT_S = 300.0 @@ -7838,7 +7838,7 @@ def _setup(): self._startup_group_leader_pids.add(pid) # Cross-chip init barrier. ChipWorker.init can have a long right tail - # (e.g. PTO2_RING_HEAP=4 GiB pushes per-rank device_malloc beyond the + # (e.g. a 4 GiB runtime_env.ring_heap pushes per-rank device_malloc beyond the # host stream sync budget); without this barrier a fast-init chip # starts its aclrtSyncStream window N seconds before a slow peer # reaches the same point, and any cross-rank wait inside the op (HCCL diff --git a/simpler_setup/scene_test.py b/simpler_setup/scene_test.py index ba5425edb1..c49b9493c4 100644 --- a/simpler_setup/scene_test.py +++ b/simpler_setup/scene_test.py @@ -1541,8 +1541,8 @@ def _build_config( # 0 = auto: DeviceRunner uses the architecture default. config.aicpu_thread_num = config_dict.get("aicpu_thread_num", 0) # Per-task ring sizing (tensormap_and_ringbuffer only; 0 = unset), - # nested under the "runtime_env" key. Takes precedence over the - # PTO2_RING_* env vars / RUNTIME_ENV. Each value is either a scalar + # nested under the "runtime_env" key. This is the only way to size the + # rings -- there is no process-wide env. Each value is either a scalar # (broadcast to every ring) or a list of RUNTIME_ENV_RING_COUNT ints # (per-ring); the binding accepts both forms. runtime_env = config_dict.get("runtime_env", {}) diff --git a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp index a52b693d87..0fca680514 100644 --- a/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp @@ -183,73 +183,18 @@ static void record_bind_phase(HostPhaseKind kind, int64_t start_ns, const char * host_phase_record_bind(static_cast(kind), static_cast(start_ns), with_counters, payload); } -static std::string trim_copy(const std::string &input) { - size_t begin = 0; - while (begin < input.size() && std::isspace(static_cast(input[begin]))) { - ++begin; - } - size_t end = input.size(); - while (end > begin && std::isspace(static_cast(input[end - 1]))) { - --end; - } - return input.substr(begin, end - begin); -} - -static bool parse_uint_token( - const char *name, const std::string &raw, uint64_t min_val, uint64_t max_val, bool require_power_of_2, uint64_t *out -) { - std::string token = trim_copy(raw); - if (token.empty()) { - LOG_WARN("%s has an empty value in '%s', ignored", name, raw.c_str()); - return false; - } - - if (token[0] == '-') { - LOG_WARN("%s=%s invalid (must be a non-negative integer), ignored", name, token.c_str()); - return false; - } - char *endptr = nullptr; - errno = 0; - unsigned long long parsed = std::strtoull(token.c_str(), &endptr, 10); - if (errno == ERANGE || endptr == token.c_str() || *endptr != '\0') { - LOG_WARN("%s=%s invalid (must be a non-negative integer), ignored", name, token.c_str()); - return false; - } - uint64_t val = static_cast(parsed); - - if (val < min_val || val > max_val) { - LOG_WARN( - "%s=%s invalid (must be in [%" PRIu64 ", %" PRIu64 "]), ignored", name, token.c_str(), min_val, max_val - ); - return false; - } - if (require_power_of_2 && !is_power_of_2_u64(val)) { - LOG_WARN("%s=%s invalid (must be a power of 2), ignored", name, token.c_str()); - return false; - } - *out = val; - return true; -} - -// The PTO2_RING_* knobs are shared with tensormap_and_ringbuffer, where a value -// may be a comma-separated list, one entry per ring. hbg has one ring, so it -// accepts the single-value spelling and rejects a list — which is what the -// multi-ring parser did here too, since it required exactly one entry. -static void -apply_env_ring_value(const char *name, uint64_t min_val, uint64_t max_val, bool require_power_of_2, uint64_t *out) { - const char *env = std::getenv(name); - if (!env) return; - - std::string text(env); - if (text.find(',') != std::string::npos) { - LOG_WARN("%s=%s invalid (this runtime has one ring; expected a single value), ignored", name, env); - return; - } - uint64_t value = 0; - if (!parse_uint_token(name, text, min_val, max_val, require_power_of_2, &value)) { - return; +// The ring sizes were once settable process-wide through PTO2_RING_TASK_WINDOW / +// PTO2_RING_HEAP. They are per task now, through CallConfig.runtime_env, and +// nothing reads those names. Exporting one is therefore a silent +// misconfiguration -- the run takes the compile-time default and the requested +// sizing appears nowhere -- so it is reported once per bind. +static void warn_on_retired_ring_env() { + static constexpr const char *kRetired[] = {"PTO2_RING_TASK_WINDOW", "PTO2_RING_HEAP"}; + for (const char *name : kRetired) { + if (std::getenv(name) != nullptr) { + LOG_WARN("%s is no longer read; size the rings per task via CallConfig.runtime_env", name); + } } - *out = value; } // ring_task_window points into the #pragma pack(1) RuntimeEnv wire struct @@ -268,18 +213,18 @@ static uint64_t read_ring_override(const uint64_t *base, int idx) { } // ring_task_window points at the first slot of a per-ring array in the RuntimeEnv -// wire struct (0 = unset); hbg has one ring and reads slot 0. Precedence: -// per-task entry > PTO2_RING_TASK_WINDOW env value > compile-time default. +// wire struct (0 = unset); hbg has one ring and reads slot 0. A per-task entry +// wins over the compile-time default, and there is nothing between them. // // The heap takes no configuration: its device region is committed after // orchestration, sized to what the graph turned out to need, so there is nothing -// to resolve up front. RuntimeEnv::ring_heap and PTO2_RING_HEAP remain the -// reclaiming runtime's knobs; this function does not resolve them, and the caller -// reads them only to warn that they reach nothing here. +// to resolve up front. RuntimeEnv::ring_heap stays the reclaiming runtime's knob; +// this function does not resolve it, and the caller reads it only to warn that it +// reaches nothing here. static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t *eff_task_window_size) { *eff_task_window_size = CHIP_TASK_WINDOW_SIZE; - apply_env_ring_value("PTO2_RING_TASK_WINDOW", 4, static_cast(INT32_MAX), true, eff_task_window_size); + warn_on_retired_ring_env(); const uint64_t task_window_override = read_ring_override(ring_task_window, 0); if (task_window_override != 0) { @@ -1048,13 +993,13 @@ extern "C" int bind_callable_to_runtime_impl( if (!resolve_task_window_size(ring_task_window, &eff_task_window_size)) { return PTO_RUNTIME_ERR_INTERNAL; } - // The heap takes no configuration, so a set ring_heap / PTO2_RING_HEAP reaches - // nothing in this runtime — most often it is a config written for the reclaiming - // one, whose knob it still is. - if (read_ring_override(ring_heap, 0) != 0 || std::getenv("PTO2_RING_HEAP") != nullptr) { + // The heap takes no configuration, so a set ring_heap reaches nothing in this + // runtime — most often it is a config written for the reclaiming one, whose knob + // it still is. + if (read_ring_override(ring_heap, 0) != 0) { LOG_WARN( - "%s", "host_build_graph ignores ring_heap / PTO2_RING_HEAP: its graph heap is committed after " - "orchestration at the size the graph turned out to need" + "%s", "host_build_graph ignores ring_heap: its graph heap is committed after orchestration at the " + "size the graph turned out to need" ); } LOG_INFO("Ring task window: %" PRIu64, eff_task_window_size); diff --git a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 9b27e5f806..8fdf44233a 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1201,7 +1201,7 @@ static bool append_fanin_or_fail( LOG_ERROR(" Requested: at least %d distinct producer dependencies", fanin_builder->count + 1); LOG_ERROR("Solution:"); LOG_ERROR(" Reduce the task fanin to at most CHIP_MAX_FANIN=%d.", CHIP_MAX_FANIN); - LOG_ERROR(" HBG has no dependency spill pool; PTO2_RING_DEP_POOL does not apply."); + LOG_ERROR(" HBG has no dependency spill pool; runtime_env.ring_dep_pool does not apply."); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; diff --git a/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h b/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h index 9d2dc1eac6..6a2fe3f0ac 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h +++ b/src/a2a3/runtime/host_build_graph/runtime/ring_buffer.h @@ -238,8 +238,8 @@ class TaskAllocator { LOG_ERROR(" Shrink the graph's intermediate tensors; this heap has no configuration knob"); } else { LOG_ERROR( - " Increase task window (current: %d); env PTO2_RING_TASK_WINDOW= (e.g. %d)", window_size_, - window_size_ * 2 + " Increase task window (current: %d); CallConfig.runtime_env.ring_task_window= (e.g. %d)", + window_size_, window_size_ * 2 ); } LOG_ERROR("========================================"); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 2c9802b507..333164dd31 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -245,8 +245,6 @@ independently for each resource and ring: ```text per-ring CallConfig entry (a scalar is broadcast to every entry) - > per-ring PTO2_RING_* env value - > scalar PTO2_RING_* env value > compile-time default ``` @@ -302,21 +300,13 @@ per-case `config` dict — each value is a scalar or a four-entry list: } ``` -Process-wide env fallback accepts either one scalar value or exactly four -comma-separated per-ring values. Invalid env values are logged and ignored, then -fall through to defaults. `PTO2_RING_HEAP` values are integer bytes: - -```bash -# Uniform, old behavior: -PTO2_RING_TASK_WINDOW=1024 -PTO2_RING_HEAP=1048576 -PTO2_RING_DEP_POOL=1024 - -# Per-ring, indexed by ring_id 0..3: -PTO2_RING_TASK_WINDOW=8192,16384,131072,524288 -PTO2_RING_HEAP=134217728,268435456,402653184,536870912 -PTO2_RING_DEP_POOL=4096,8192,16384,32768 -``` +There is no process-wide fallback. `PTO2_RING_TASK_WINDOW` / `PTO2_RING_HEAP` / +`PTO2_RING_DEP_POOL` used to size every ring in the process; they are not read any +more, and the runtime logs a warning if one is still exported, because otherwise +the requested sizing would vanish into the compile-time default. Size the rings +per task instead — that is what the `CallConfig` block above does, and it is +strictly more expressive: two tasks in one process can hold different ring sizes, +which the env never allowed. Use `--enable-scope-stats` to confirm the effective values for a real run. The first line of `scope_stats/scope_stats.jsonl` includes `task_window_max`, diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index e653a32b67..af0b3dfd22 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -210,7 +210,7 @@ else: **Reclamation**: Scheduler threads advance `last_task_alive` via lock-free CAS when the oldest task reaches state CONSUMED (4). This frees slots for reuse. -**Flow control**: When the ring is full, the orchestrator blocks until the scheduler advances `last_task_alive`. With `PTO2_RING_TASK_WINDOW=16` and 208 tasks, slots are recycled ~13 times each. +**Flow control**: When the ring is full, the orchestrator blocks until the scheduler advances `last_task_alive`. With a 16-slot `runtime_env.ring_task_window` and 208 tasks, slots are recycled ~13 times each. ### 4.2 Heap Ring @@ -250,7 +250,7 @@ The ring buffer mechanism provides **flow control** between the orchestrator (pr **TensorMap pool back-pressure**: Before STEP 4 registers a task's outputs, the orchestrator's `ensure_tensormap_capacity` reserves pool space for the inserts. When the shared entry pool is exhausted, it reclaims retired entries across all rings and spin-waits until reclaim actually frees entries, with a 500 ms wall-clock deadlock backstop (see Section 5.4). -This back-pressure is essential for correctness with small ring sizes — for example, with `PTO2_RING_TASK_WINDOW=16` and 208 tasks, the orchestrator blocks ~192 times, each time waiting for the scheduler to drain completed tasks before continuing. +This back-pressure is essential for correctness with small ring sizes — for example, with a 16-slot `runtime_env.ring_task_window` and 208 tasks, the orchestrator blocks ~192 times, each time waiting for the scheduler to drain completed tasks before continuing. ### 4.5 Deadlock Detection diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index 4367aecc8f..74987b1f84 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -118,100 +117,18 @@ static std::string format_ring_array(const T (&values)[CHIP_MAX_RING_DEPTH]) { return out; } -static std::string trim_copy(const std::string &input) { - size_t begin = 0; - while (begin < input.size() && std::isspace(static_cast(input[begin]))) { - ++begin; - } - size_t end = input.size(); - while (end > begin && std::isspace(static_cast(input[end - 1]))) { - --end; - } - return input.substr(begin, end - begin); -} - -static bool parse_uint_token( - const char *name, const std::string &raw, uint64_t min_val, uint64_t max_val, bool require_power_of_2, uint64_t *out -) { - std::string token = trim_copy(raw); - if (token.empty()) { - LOG_WARN("%s has an empty value in '%s', ignored", name, raw.c_str()); - return false; - } - - if (token[0] == '-') { - LOG_WARN("%s=%s invalid (must be a non-negative integer), ignored", name, token.c_str()); - return false; - } - char *endptr = nullptr; - errno = 0; - unsigned long long parsed = std::strtoull(token.c_str(), &endptr, 10); - if (errno == ERANGE || endptr == token.c_str() || *endptr != '\0') { - LOG_WARN("%s=%s invalid (must be a non-negative integer), ignored", name, token.c_str()); - return false; - } - uint64_t val = static_cast(parsed); - - if (val < min_val || val > max_val) { - LOG_WARN( - "%s=%s invalid (must be in [%" PRIu64 ", %" PRIu64 "]), ignored", name, token.c_str(), min_val, max_val - ); - return false; - } - if (require_power_of_2 && !is_power_of_2_u64(val)) { - LOG_WARN("%s=%s invalid (must be a power of 2), ignored", name, token.c_str()); - return false; - } - *out = val; - return true; -} - -static void apply_env_ring_values( - const char *name, uint64_t min_val, uint64_t max_val, bool require_power_of_2, uint64_t out[CHIP_MAX_RING_DEPTH] -) { - const char *env = std::getenv(name); - if (!env) return; - - std::string text(env); - if (text.find(',') == std::string::npos) { - uint64_t value = 0; - if (!parse_uint_token(name, text, min_val, max_val, require_power_of_2, &value)) { - return; - } - for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - out[r] = value; - } - return; - } - - uint64_t parsed[CHIP_MAX_RING_DEPTH]{}; - size_t pos = 0; - for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - size_t comma = text.find(',', pos); - std::string token = text.substr(pos, comma == std::string::npos ? std::string::npos : comma - pos); - if (!parse_uint_token(name, token, min_val, max_val, require_power_of_2, &parsed[r])) { - return; - } - if (comma == std::string::npos) { - if (r != CHIP_MAX_RING_DEPTH - 1) { - LOG_WARN( - "%s=%s invalid (expected exactly %d comma-separated values), ignored", name, env, - CHIP_MAX_RING_DEPTH - ); - return; - } - pos = text.size(); - } else { - pos = comma + 1; +// The ring sizes were once settable process-wide through PTO2_RING_TASK_WINDOW / +// PTO2_RING_HEAP / PTO2_RING_DEP_POOL. They are per task now, through +// CallConfig.runtime_env, and nothing reads those names. Exporting one is +// therefore a silent misconfiguration -- the run takes the compile-time default +// and the requested sizing appears nowhere -- so it is reported once per bind. +static void warn_on_retired_ring_env() { + static constexpr const char *kRetired[] = {"PTO2_RING_TASK_WINDOW", "PTO2_RING_HEAP", "PTO2_RING_DEP_POOL"}; + for (const char *name : kRetired) { + if (std::getenv(name) != nullptr) { + LOG_WARN("%s is no longer read; size the rings per task via CallConfig.runtime_env", name); } } - if (pos < text.size() || (!text.empty() && text.back() == ',')) { - LOG_WARN("%s=%s invalid (expected exactly %d comma-separated values), ignored", name, env, CHIP_MAX_RING_DEPTH); - return; - } - for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - out[r] = parsed[r]; - } } // ring_task_window / ring_heap / ring_dep_pool point into the #pragma pack(1) @@ -230,9 +147,10 @@ static uint64_t read_ring_override(const uint64_t *base, int idx) { } // Each of ring_task_window / ring_heap / ring_dep_pool is a per-ring array of -// CHIP_MAX_RING_DEPTH entries (0 = unset). Precedence per ring: per-task entry > -// PTO2_RING_* env value > compile-time default. A "size all rings the same" -// request arrives already broadcast to every entry by the caller. +// CHIP_MAX_RING_DEPTH entries (0 = unset). A per-ring entry wins over the +// compile-time default, and there is nothing between them: sizing is per task. +// A "size all rings the same" request arrives already broadcast to every entry +// by the caller. static bool resolve_ring_config( const uint64_t *ring_task_window, const uint64_t *ring_heap, const uint64_t *ring_dep_pool, uint64_t eff_task_window_sizes[CHIP_MAX_RING_DEPTH], uint64_t eff_heap_sizes[CHIP_MAX_RING_DEPTH], @@ -245,9 +163,7 @@ static bool resolve_ring_config( dep_pool_values[r] = CHIP_DEP_LIST_POOL_SIZE; } - apply_env_ring_values("PTO2_RING_TASK_WINDOW", 4, static_cast(INT32_MAX), true, eff_task_window_sizes); - apply_env_ring_values("PTO2_RING_HEAP", 1024, std::numeric_limits::max(), false, eff_heap_sizes); - apply_env_ring_values("PTO2_RING_DEP_POOL", 4, static_cast(INT32_MAX), false, dep_pool_values); + warn_on_retired_ring_env(); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { const uint64_t task_window_override = read_ring_override(ring_task_window, r); @@ -539,7 +455,7 @@ extern "C" int prepared_run_config_compatible_impl( } // per-(cid,config): resolve the cache-key sizing knobs. Pure host parsing over -// per-task overrides, PTO2_RING_* env, and compile-time defaults. Derived +// per-task overrides and compile-time defaults. Derived // allocation sizes are computed only on cache miss. static bool resolve_arena_sizing( const uint64_t *ring_task_window, const uint64_t *ring_heap, const uint64_t *ring_dep_pool, ArenaSizingConfig *out diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 4b041f7394..648abbeca9 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -584,7 +584,7 @@ static bool check_scope_can_accept_task(OrchestratorState *orch, TaskAllocator & LOG_ERROR(" 1. Reduce tasks per scope (use batching/unroll)"); LOG_ERROR(" 2. Increase task window (current: %d)", allocator.window_size()); LOG_ERROR(" Compile-time: CHIP_TASK_WINDOW_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_TASK_WINDOW="); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_task_window="); LOG_ERROR(" 3. Split work across multiple scopes"); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_SCOPE_DEADLOCK); diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index 16ae4b83b7..f76747cc3f 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -110,7 +110,7 @@ bool FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { " Increase fanin spill pool capacity (current: %d, recommended: %d)", capacity, high_water * 2 ); LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", high_water * 2); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_dep_pool=%d", high_water * 2); LOG_ERROR("========================================"); latch_pool_error(error_code_ptr, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; @@ -170,7 +170,7 @@ void DepListPool::report_deadlock(SharedMemoryRingHeader &ring, int32_t needed, LOG_ERROR("Solution:"); LOG_ERROR(" Increase dep pool capacity (current: %d, recommended: %d)", capacity, high_water * 2); LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", high_water * 2); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_dep_pool=%d", high_water * 2); LOG_ERROR("========================================"); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index b98cfddc9a..73e2ada70a 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -467,16 +467,16 @@ class TaskAllocator { LOG_ERROR(" 2. Size the ring for the peak live-set retained behind the oldest open-scope task."); } else if (heap_blocked) { LOG_ERROR( - " Increase heap (current: %" PRIu64 "); env PTO2_RING_HEAP= (e.g. %" PRIu64 ")", heap_size_, - heap_size_ * 2 + " Increase heap (current: %" PRIu64 "); CallConfig.runtime_env.ring_heap= (e.g. %" PRIu64 ")", + heap_size_, heap_size_ * 2 ); LOG_ERROR( " If one increase completes, it was under-provisioned; otherwise debug the stuck head consumer." ); } else { LOG_ERROR( - " Increase task window (current: %d); env PTO2_RING_TASK_WINDOW= (e.g. %d)", window_size_, - active_tasks * 2 + " Increase task window (current: %d); CallConfig.runtime_env.ring_task_window= (e.g. %d)", + window_size_, active_tasks * 2 ); LOG_ERROR( " If one increase completes, it was under-provisioned; otherwise debug the stuck head consumer." @@ -550,7 +550,7 @@ struct FaninPool { LOG_ERROR("Solution:"); LOG_ERROR(" Increase fanin spill pool capacity (current: %d, recommended: %d).", capacity, capacity * 2); LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", capacity * 2); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_dep_pool=%d", capacity * 2); LOG_ERROR("========================================"); if (error_code_ptr) { error_code_ptr->store(SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED, std::memory_order_release); @@ -755,7 +755,7 @@ struct DepListPool { LOG_ERROR("Solution:"); LOG_ERROR(" Increase dep pool capacity (current: %d, recommended: %d).", capacity, capacity * 2); LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", capacity * 2); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_dep_pool=%d", capacity * 2); LOG_ERROR("========================================"); if (error_code_ptr) { error_code_ptr->store(SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED, std::memory_order_release); diff --git a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp index 1d8139e717..133c70e995 100644 --- a/src/a5/runtime/host_build_graph/host/runtime_maker.cpp +++ b/src/a5/runtime/host_build_graph/host/runtime_maker.cpp @@ -183,73 +183,18 @@ static void record_bind_phase(HostPhaseKind kind, int64_t start_ns, const char * host_phase_record_bind(static_cast(kind), static_cast(start_ns), with_counters, payload); } -static std::string trim_copy(const std::string &input) { - size_t begin = 0; - while (begin < input.size() && std::isspace(static_cast(input[begin]))) { - ++begin; - } - size_t end = input.size(); - while (end > begin && std::isspace(static_cast(input[end - 1]))) { - --end; - } - return input.substr(begin, end - begin); -} - -static bool parse_uint_token( - const char *name, const std::string &raw, uint64_t min_val, uint64_t max_val, bool require_power_of_2, uint64_t *out -) { - std::string token = trim_copy(raw); - if (token.empty()) { - LOG_WARN("%s has an empty value in '%s', ignored", name, raw.c_str()); - return false; - } - - if (token[0] == '-') { - LOG_WARN("%s=%s invalid (must be a non-negative integer), ignored", name, token.c_str()); - return false; - } - char *endptr = nullptr; - errno = 0; - unsigned long long parsed = std::strtoull(token.c_str(), &endptr, 10); - if (errno == ERANGE || endptr == token.c_str() || *endptr != '\0') { - LOG_WARN("%s=%s invalid (must be a non-negative integer), ignored", name, token.c_str()); - return false; - } - uint64_t val = static_cast(parsed); - - if (val < min_val || val > max_val) { - LOG_WARN( - "%s=%s invalid (must be in [%" PRIu64 ", %" PRIu64 "]), ignored", name, token.c_str(), min_val, max_val - ); - return false; - } - if (require_power_of_2 && !is_power_of_2_u64(val)) { - LOG_WARN("%s=%s invalid (must be a power of 2), ignored", name, token.c_str()); - return false; - } - *out = val; - return true; -} - -// The PTO2_RING_* knobs are shared with tensormap_and_ringbuffer, where a value -// may be a comma-separated list, one entry per ring. hbg has one ring, so it -// accepts the single-value spelling and rejects a list — which is what the -// multi-ring parser did here too, since it required exactly one entry. -static void -apply_env_ring_value(const char *name, uint64_t min_val, uint64_t max_val, bool require_power_of_2, uint64_t *out) { - const char *env = std::getenv(name); - if (!env) return; - - std::string text(env); - if (text.find(',') != std::string::npos) { - LOG_WARN("%s=%s invalid (this runtime has one ring; expected a single value), ignored", name, env); - return; - } - uint64_t value = 0; - if (!parse_uint_token(name, text, min_val, max_val, require_power_of_2, &value)) { - return; +// The ring sizes were once settable process-wide through PTO2_RING_TASK_WINDOW / +// PTO2_RING_HEAP. They are per task now, through CallConfig.runtime_env, and +// nothing reads those names. Exporting one is therefore a silent +// misconfiguration -- the run takes the compile-time default and the requested +// sizing appears nowhere -- so it is reported once per bind. +static void warn_on_retired_ring_env() { + static constexpr const char *kRetired[] = {"PTO2_RING_TASK_WINDOW", "PTO2_RING_HEAP"}; + for (const char *name : kRetired) { + if (std::getenv(name) != nullptr) { + LOG_WARN("%s is no longer read; size the rings per task via CallConfig.runtime_env", name); + } } - *out = value; } // ring_task_window points into the #pragma pack(1) RuntimeEnv wire struct @@ -268,18 +213,18 @@ static uint64_t read_ring_override(const uint64_t *base, int idx) { } // ring_task_window points at the first slot of a per-ring array in the RuntimeEnv -// wire struct (0 = unset); hbg has one ring and reads slot 0. Precedence: -// per-task entry > PTO2_RING_TASK_WINDOW env value > compile-time default. +// wire struct (0 = unset); hbg has one ring and reads slot 0. A per-task entry +// wins over the compile-time default, and there is nothing between them. // // The heap takes no configuration: its device region is committed after // orchestration, sized to what the graph turned out to need, so there is nothing -// to resolve up front. RuntimeEnv::ring_heap and PTO2_RING_HEAP remain the -// reclaiming runtime's knobs; this function does not resolve them, and the caller -// reads them only to warn that they reach nothing here. +// to resolve up front. RuntimeEnv::ring_heap stays the reclaiming runtime's knob; +// this function does not resolve it, and the caller reads it only to warn that it +// reaches nothing here. static bool resolve_task_window_size(const uint64_t *ring_task_window, uint64_t *eff_task_window_size) { *eff_task_window_size = CHIP_TASK_WINDOW_SIZE; - apply_env_ring_value("PTO2_RING_TASK_WINDOW", 4, static_cast(INT32_MAX), true, eff_task_window_size); + warn_on_retired_ring_env(); const uint64_t task_window_override = read_ring_override(ring_task_window, 0); if (task_window_override != 0) { @@ -1064,13 +1009,13 @@ extern "C" int bind_callable_to_runtime_impl( if (!resolve_task_window_size(ring_task_window, &eff_task_window_size)) { return PTO_RUNTIME_ERR_INTERNAL; } - // The heap takes no configuration, so a set ring_heap / PTO2_RING_HEAP reaches - // nothing in this runtime — most often it is a config written for the reclaiming - // one, whose knob it still is. - if (read_ring_override(ring_heap, 0) != 0 || std::getenv("PTO2_RING_HEAP") != nullptr) { + // The heap takes no configuration, so a set ring_heap reaches nothing in this + // runtime — most often it is a config written for the reclaiming one, whose knob + // it still is. + if (read_ring_override(ring_heap, 0) != 0) { LOG_WARN( - "%s", "host_build_graph ignores ring_heap / PTO2_RING_HEAP: its graph heap is committed after " - "orchestration at the size the graph turned out to need" + "%s", "host_build_graph ignores ring_heap: its graph heap is committed after orchestration at the " + "size the graph turned out to need" ); } LOG_INFO("Ring task window: %" PRIu64, eff_task_window_size); diff --git a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp index 9b27e5f806..8fdf44233a 100644 --- a/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp +++ b/src/a5/runtime/host_build_graph/runtime/orchestrator_core/orchestrator.cpp @@ -1201,7 +1201,7 @@ static bool append_fanin_or_fail( LOG_ERROR(" Requested: at least %d distinct producer dependencies", fanin_builder->count + 1); LOG_ERROR("Solution:"); LOG_ERROR(" Reduce the task fanin to at most CHIP_MAX_FANIN=%d.", CHIP_MAX_FANIN); - LOG_ERROR(" HBG has no dependency spill pool; PTO2_RING_DEP_POOL does not apply."); + LOG_ERROR(" HBG has no dependency spill pool; runtime_env.ring_dep_pool does not apply."); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; diff --git a/src/a5/runtime/host_build_graph/runtime/ring_buffer.h b/src/a5/runtime/host_build_graph/runtime/ring_buffer.h index 9d2dc1eac6..6a2fe3f0ac 100644 --- a/src/a5/runtime/host_build_graph/runtime/ring_buffer.h +++ b/src/a5/runtime/host_build_graph/runtime/ring_buffer.h @@ -238,8 +238,8 @@ class TaskAllocator { LOG_ERROR(" Shrink the graph's intermediate tensors; this heap has no configuration knob"); } else { LOG_ERROR( - " Increase task window (current: %d); env PTO2_RING_TASK_WINDOW= (e.g. %d)", window_size_, - window_size_ * 2 + " Increase task window (current: %d); CallConfig.runtime_env.ring_task_window= (e.g. %d)", + window_size_, window_size_ * 2 ); } LOG_ERROR("========================================"); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index cbfca9b274..f3ef8661aa 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -247,8 +247,6 @@ independently for each resource and ring: ```text per-ring CallConfig entry (a scalar is broadcast to every entry) - > per-ring PTO2_RING_* env value - > scalar PTO2_RING_* env value > compile-time default ``` @@ -304,21 +302,13 @@ per-case `config` dict — each value is a scalar or a four-entry list: } ``` -Process-wide env fallback accepts either one scalar value or exactly four -comma-separated per-ring values. Invalid env values are logged and ignored, then -fall through to defaults. `PTO2_RING_HEAP` values are integer bytes: - -```bash -# Uniform, old behavior: -PTO2_RING_TASK_WINDOW=1024 -PTO2_RING_HEAP=1048576 -PTO2_RING_DEP_POOL=1024 - -# Per-ring, indexed by ring_id 0..3: -PTO2_RING_TASK_WINDOW=8192,16384,131072,524288 -PTO2_RING_HEAP=134217728,268435456,402653184,536870912 -PTO2_RING_DEP_POOL=4096,8192,16384,32768 -``` +There is no process-wide fallback. `PTO2_RING_TASK_WINDOW` / `PTO2_RING_HEAP` / +`PTO2_RING_DEP_POOL` used to size every ring in the process; they are not read any +more, and the runtime logs a warning if one is still exported, because otherwise +the requested sizing would vanish into the compile-time default. Size the rings +per task instead — that is what the `CallConfig` block above does, and it is +strictly more expressive: two tasks in one process can hold different ring sizes, +which the env never allowed. Use `--enable-scope-stats` to confirm the effective values for a real run. The first line of `scope_stats/scope_stats.jsonl` includes `task_window_max`, diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 2cf89e6993..efe332a6ae 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -211,7 +211,7 @@ else: **Reclamation**: Scheduler threads advance `last_task_alive` via lock-free CAS when the oldest task reaches state CONSUMED (4). This frees slots for reuse. -**Flow control**: When the ring is full, the orchestrator blocks until the scheduler advances `last_task_alive`. With `PTO2_RING_TASK_WINDOW=16` and 208 tasks, slots are recycled ~13 times each. +**Flow control**: When the ring is full, the orchestrator blocks until the scheduler advances `last_task_alive`. With a 16-slot `runtime_env.ring_task_window` and 208 tasks, slots are recycled ~13 times each. ### 4.2 Heap Ring @@ -260,7 +260,7 @@ share one entry pool. K=16 batching is enabled only after every reclaim consumer is wired to the current request/ack masks; incomplete wiring keeps per-advance publication. -This back-pressure is essential for correctness with small ring sizes — for example, with `PTO2_RING_TASK_WINDOW=16` and 208 tasks, the orchestrator blocks ~192 times, each time waiting for the scheduler to drain completed tasks before continuing. +This back-pressure is essential for correctness with small ring sizes — for example, with a 16-slot `runtime_env.ring_task_window` and 208 tasks, the orchestrator blocks ~192 times, each time waiting for the scheduler to drain completed tasks before continuing. ### 4.5 Deadlock Detection diff --git a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp index 16d9004458..ff02a39bb2 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -118,100 +117,18 @@ static std::string format_ring_array(const T (&values)[CHIP_MAX_RING_DEPTH]) { return out; } -static std::string trim_copy(const std::string &input) { - size_t begin = 0; - while (begin < input.size() && std::isspace(static_cast(input[begin]))) { - ++begin; - } - size_t end = input.size(); - while (end > begin && std::isspace(static_cast(input[end - 1]))) { - --end; - } - return input.substr(begin, end - begin); -} - -static bool parse_uint_token( - const char *name, const std::string &raw, uint64_t min_val, uint64_t max_val, bool require_power_of_2, uint64_t *out -) { - std::string token = trim_copy(raw); - if (token.empty()) { - LOG_WARN("%s has an empty value in '%s', ignored", name, raw.c_str()); - return false; - } - - if (token[0] == '-') { - LOG_WARN("%s=%s invalid (must be a non-negative integer), ignored", name, token.c_str()); - return false; - } - char *endptr = nullptr; - errno = 0; - unsigned long long parsed = std::strtoull(token.c_str(), &endptr, 10); - if (errno == ERANGE || endptr == token.c_str() || *endptr != '\0') { - LOG_WARN("%s=%s invalid (must be a non-negative integer), ignored", name, token.c_str()); - return false; - } - uint64_t val = static_cast(parsed); - - if (val < min_val || val > max_val) { - LOG_WARN( - "%s=%s invalid (must be in [%" PRIu64 ", %" PRIu64 "]), ignored", name, token.c_str(), min_val, max_val - ); - return false; - } - if (require_power_of_2 && !is_power_of_2_u64(val)) { - LOG_WARN("%s=%s invalid (must be a power of 2), ignored", name, token.c_str()); - return false; - } - *out = val; - return true; -} - -static void apply_env_ring_values( - const char *name, uint64_t min_val, uint64_t max_val, bool require_power_of_2, uint64_t out[CHIP_MAX_RING_DEPTH] -) { - const char *env = std::getenv(name); - if (!env) return; - - std::string text(env); - if (text.find(',') == std::string::npos) { - uint64_t value = 0; - if (!parse_uint_token(name, text, min_val, max_val, require_power_of_2, &value)) { - return; - } - for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - out[r] = value; - } - return; - } - - uint64_t parsed[CHIP_MAX_RING_DEPTH]{}; - size_t pos = 0; - for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - size_t comma = text.find(',', pos); - std::string token = text.substr(pos, comma == std::string::npos ? std::string::npos : comma - pos); - if (!parse_uint_token(name, token, min_val, max_val, require_power_of_2, &parsed[r])) { - return; - } - if (comma == std::string::npos) { - if (r != CHIP_MAX_RING_DEPTH - 1) { - LOG_WARN( - "%s=%s invalid (expected exactly %d comma-separated values), ignored", name, env, - CHIP_MAX_RING_DEPTH - ); - return; - } - pos = text.size(); - } else { - pos = comma + 1; +// The ring sizes were once settable process-wide through PTO2_RING_TASK_WINDOW / +// PTO2_RING_HEAP / PTO2_RING_DEP_POOL. They are per task now, through +// CallConfig.runtime_env, and nothing reads those names. Exporting one is +// therefore a silent misconfiguration -- the run takes the compile-time default +// and the requested sizing appears nowhere -- so it is reported once per bind. +static void warn_on_retired_ring_env() { + static constexpr const char *kRetired[] = {"PTO2_RING_TASK_WINDOW", "PTO2_RING_HEAP", "PTO2_RING_DEP_POOL"}; + for (const char *name : kRetired) { + if (std::getenv(name) != nullptr) { + LOG_WARN("%s is no longer read; size the rings per task via CallConfig.runtime_env", name); } } - if (pos < text.size() || (!text.empty() && text.back() == ',')) { - LOG_WARN("%s=%s invalid (expected exactly %d comma-separated values), ignored", name, env, CHIP_MAX_RING_DEPTH); - return; - } - for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { - out[r] = parsed[r]; - } } // ring_task_window / ring_heap / ring_dep_pool point into the #pragma pack(1) @@ -230,9 +147,10 @@ static uint64_t read_ring_override(const uint64_t *base, int idx) { } // Each of ring_task_window / ring_heap / ring_dep_pool is a per-ring array of -// CHIP_MAX_RING_DEPTH entries (0 = unset). Precedence per ring: per-task entry > -// PTO2_RING_* env value > compile-time default. A "size all rings the same" -// request arrives already broadcast to every entry by the caller. +// CHIP_MAX_RING_DEPTH entries (0 = unset). A per-ring entry wins over the +// compile-time default, and there is nothing between them: sizing is per task. +// A "size all rings the same" request arrives already broadcast to every entry +// by the caller. static bool resolve_ring_config( const uint64_t *ring_task_window, const uint64_t *ring_heap, const uint64_t *ring_dep_pool, uint64_t eff_task_window_sizes[CHIP_MAX_RING_DEPTH], uint64_t eff_heap_sizes[CHIP_MAX_RING_DEPTH], @@ -245,9 +163,7 @@ static bool resolve_ring_config( dep_pool_values[r] = CHIP_DEP_LIST_POOL_SIZE; } - apply_env_ring_values("PTO2_RING_TASK_WINDOW", 4, static_cast(INT32_MAX), true, eff_task_window_sizes); - apply_env_ring_values("PTO2_RING_HEAP", 1024, std::numeric_limits::max(), false, eff_heap_sizes); - apply_env_ring_values("PTO2_RING_DEP_POOL", 4, static_cast(INT32_MAX), false, dep_pool_values); + warn_on_retired_ring_env(); for (int r = 0; r < CHIP_MAX_RING_DEPTH; r++) { const uint64_t task_window_override = read_ring_override(ring_task_window, r); @@ -539,7 +455,7 @@ extern "C" int prepared_run_config_compatible_impl( } // per-(cid,config): resolve the cache-key sizing knobs. Pure host parsing over -// per-task overrides, PTO2_RING_* env, and compile-time defaults. Derived +// per-task overrides and compile-time defaults. Derived // allocation sizes are computed only on cache miss. static bool resolve_arena_sizing( const uint64_t *ring_task_window, const uint64_t *ring_heap, const uint64_t *ring_dep_pool, ArenaSizingConfig *out diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp index 52c78653e2..841e9d0862 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/orchestrator.cpp @@ -576,7 +576,7 @@ static bool check_scope_can_accept_task(OrchestratorState *orch, TaskAllocator & LOG_ERROR(" 1. Reduce tasks per scope (use batching/unroll)"); LOG_ERROR(" 2. Increase task window (current: %d)", allocator.window_size()); LOG_ERROR(" Compile-time: CHIP_TASK_WINDOW_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_TASK_WINDOW="); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_task_window="); LOG_ERROR(" 3. Split work across multiple scopes"); LOG_ERROR("========================================"); orch_mark_fatal(orch, SIMPLER_ERROR_SCOPE_DEADLOCK); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp index d0778b67d5..4e959a83b1 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.cpp @@ -122,7 +122,7 @@ bool FaninPool::ensure_space(SharedMemoryRingHeader &ring, int32_t needed) { " Increase fanin spill pool capacity (current: %d, recommended: %d)", capacity, high_water * 2 ); LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", high_water * 2); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_dep_pool=%d", high_water * 2); LOG_ERROR("========================================"); latch_pool_error(error_code_ptr, SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED); return false; @@ -183,7 +183,7 @@ void DepListPool::report_deadlock(SharedMemoryRingHeader &ring, int32_t needed, LOG_ERROR("Solution:"); LOG_ERROR(" Increase dep pool capacity (current: %d, recommended: %d)", capacity, high_water * 2); LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", high_water * 2); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_dep_pool=%d", high_water * 2); LOG_ERROR("========================================"); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h index 9d545df03f..6894e004f1 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/ring_buffer.h @@ -540,16 +540,16 @@ class TaskAllocator { LOG_ERROR(" 2. Size the ring for the peak live-set retained behind the oldest open-scope task."); } else if (heap_blocked) { LOG_ERROR( - " Increase heap (current: %" PRIu64 "); env PTO2_RING_HEAP= (e.g. %" PRIu64 ")", heap_size_, - heap_size_ * 2 + " Increase heap (current: %" PRIu64 "); CallConfig.runtime_env.ring_heap= (e.g. %" PRIu64 ")", + heap_size_, heap_size_ * 2 ); LOG_ERROR( " If one increase completes, it was under-provisioned; otherwise debug the stuck head consumer." ); } else { LOG_ERROR( - " Increase task window (current: %d); env PTO2_RING_TASK_WINDOW= (e.g. %d)", window_size_, - active_tasks * 2 + " Increase task window (current: %d); CallConfig.runtime_env.ring_task_window= (e.g. %d)", + window_size_, active_tasks * 2 ); LOG_ERROR( " If one increase completes, it was under-provisioned; otherwise debug the stuck head consumer." @@ -643,7 +643,7 @@ struct FaninPool { LOG_ERROR("Solution:"); LOG_ERROR(" Increase fanin spill pool capacity (current: %d, recommended: %d).", capacity, capacity * 2); LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", capacity * 2); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_dep_pool=%d", capacity * 2); LOG_ERROR("========================================"); if (error_code_ptr) { error_code_ptr->store(SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED, std::memory_order_release); @@ -869,7 +869,7 @@ struct DepListPool { LOG_ERROR("Solution:"); LOG_ERROR(" Increase dep pool capacity (current: %d, recommended: %d).", capacity, capacity * 2); LOG_ERROR(" Compile-time: CHIP_DEP_LIST_POOL_SIZE in runtime_types.h"); - LOG_ERROR(" Runtime env: PTO2_RING_DEP_POOL=%d", capacity * 2); + LOG_ERROR(" Per task: CallConfig.runtime_env.ring_dep_pool=%d", capacity * 2); LOG_ERROR("========================================"); if (error_code_ptr) { error_code_ptr->store(SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED, std::memory_order_release); diff --git a/src/common/runtime_status/error_names.h b/src/common/runtime_status/error_names.h index f2cf4ed4f1..fb7b0e51b5 100644 --- a/src/common/runtime_status/error_names.h +++ b/src/common/runtime_status/error_names.h @@ -134,17 +134,17 @@ static inline const char *error_hint(int32_t code) { case SIMPLER_ERROR_NONE: return ""; case SIMPLER_ERROR_SCOPE_DEADLOCK: - return "raise ring_task_window (PTO2_RING_TASK_WINDOW) or split the scope so slots are " + return "raise runtime_env.ring_task_window or split the scope so slots are " "reclaimed sooner; enable CallConfig.enable_scope_stats to see which scope peaked"; case SIMPLER_ERROR_HEAP_RING_DEADLOCK: - return "raise ring_heap (PTO2_RING_HEAP) or shrink per-task args / intermediate tensors; the " + return "raise runtime_env.ring_heap or shrink per-task args / intermediate tensors; the " "'Ring buffer sizes:' line above reports the configured capacities"; case SIMPLER_ERROR_FLOW_CONTROL_DEADLOCK: - return "raise ring_task_window (PTO2_RING_TASK_WINDOW) or shrink the graph; in a reclaiming " + return "raise runtime_env.ring_task_window or shrink the graph; in a reclaiming " "runtime also check for nested submission or a stalled consumer"; case SIMPLER_ERROR_FANIN_CAPACITY_EXCEEDED: return "follow the preceding resource diagnostic: HBG has no spill pool, so reduce fanin to " - "CHIP_MAX_FANIN or less; for a reclaiming runtime raise PTO2_RING_DEP_POOL or cut fanin"; + "CHIP_MAX_FANIN or less; for a reclaiming runtime raise runtime_env.ring_dep_pool or cut fanin"; case SIMPLER_ERROR_INVALID_ARGS: return "an orchestration bug, not a capacity problem -- resizing the rings will not help; " "recheck the arguments of the orchestration API calls listed above"; @@ -175,7 +175,8 @@ static inline const char *error_hint(int32_t code) { "actually polls and retires them"; #ifdef SIMPLER_ERROR_READY_QUEUE_OVERFLOW case SIMPLER_ERROR_READY_QUEUE_OVERFLOW: - return "ensure PTO2_RING_TASK_WINDOW does not exceed the ready-queue capacity; otherwise keep the device " + return "ensure runtime_env.ring_task_window does not exceed the ready-queue capacity; otherwise keep the " + "device " "log and report the scheduler queue-accounting bug"; #endif default: diff --git a/src/common/task_interface/call_config.h b/src/common/task_interface/call_config.h index bfbfbeb3df..7905addd5f 100644 --- a/src/common/task_interface/call_config.h +++ b/src/common/task_interface/call_config.h @@ -55,14 +55,14 @@ inline constexpr int RUNTIME_ENV_UINT64_FIELD_COUNT = RUNTIME_ENV_FIELD_GROUPS * #pragma pack(push, 1) // Per-task runtime-environment overrides — the programmatic equivalent of the -// `PTO2_RING_*` env vars, grouped under their own sub-struct so they read as a +// ring sizes, grouped under their own sub-struct so they read as a // distinct configuration tier from the top-level `aicpu_thread_num` knob. // Consumed by tensormap_and_ringbuffer only; other runtimes // ignore them. // // Each resource is a per-scope-depth-ring array (index 0..3). A 0 entry is // unset and falls through to the next precedence tier: per-ring entry > -// PTO2_RING_* env var > compile-time default. ring_heap is bytes per ring. +// compile-time default. ring_heap is bytes per ring. // The Python/scene-test layer accepts a scalar and broadcasts it to every ring // before populating these arrays, so a one-value "size all rings the same" // request arrives here as [v, v, v, v]. @@ -117,7 +117,7 @@ struct CallConfig { int32_t enable_pmu = 0; // 0 = disabled; >0 = enabled, value selects event type int32_t enable_dep_gen = 0; int32_t enable_scope_stats = 0; // writes /scope_stats/scope_stats.jsonl - RuntimeEnv runtime_env; // per-task PTO2_RING_* overrides + RuntimeEnv runtime_env; // per-task ring sizing char output_prefix[1024] = {}; bool diagnostics_any() const noexcept { diff --git a/src/common/worker/runtime_c_api.h b/src/common/worker/runtime_c_api.h index e0dfd44ed4..0ee61ab1a0 100644 --- a/src/common/worker/runtime_c_api.h +++ b/src/common/worker/runtime_c_api.h @@ -359,7 +359,7 @@ int simpler_register_callable(DeviceContextHandle ctx, int32_t callable_id, cons * enables + output_prefix, and the per-task ring sizing overrides * (`runtime_env.ring_task_window` / `.ring_heap` / `.ring_dep_pool`, each a * per-scope-depth-ring array of RUNTIME_ENV_RING_COUNT entries; 0 = unset, - * precedence per ring: per-ring entry > PTO2_RING_* env var > compile-time + * precedence per ring: per-ring entry > compile-time * default). Ring overrides are consumed by tensormap_and_ringbuffer only; other * runtime variants accept and ignore them. Wire-compatible POD; prepare copies * it into the native-run context before returning. diff --git a/tests/st/a2a3/host_build_graph/paged_attention/test_paged_attention.py b/tests/st/a2a3/host_build_graph/paged_attention/test_paged_attention.py index 23f1e02be4..3a527988a5 100644 --- a/tests/st/a2a3/host_build_graph/paged_attention/test_paged_attention.py +++ b/tests/st/a2a3/host_build_graph/paged_attention/test_paged_attention.py @@ -73,7 +73,7 @@ class TestPagedAttentionHostBuildGraph(SceneTestCase): # before the device schedules — so the ring cannot reclaim # mid-orchestration and must hold the entire graph at once. That # exceeds the default ring window. Run it explicitly with a large - # PTO2_RING_TASK_WINDOW if needed; the GM heap needs no knob, since + # runtime_env.ring_task_window if needed; the GM heap needs no knob, since # it is committed to the size orchestration measured. "name": "Case1", "platforms": ["a2a3"], diff --git a/tests/st/a5/host_build_graph/paged_attention/test_paged_attention.py b/tests/st/a5/host_build_graph/paged_attention/test_paged_attention.py index 2baa7117f1..73c86bfc8f 100644 --- a/tests/st/a5/host_build_graph/paged_attention/test_paged_attention.py +++ b/tests/st/a5/host_build_graph/paged_attention/test_paged_attention.py @@ -73,7 +73,7 @@ class TestPagedAttentionHostBuildGraphA5(SceneTestCase): # before the device schedules — so the ring cannot reclaim # mid-orchestration and must hold the entire graph at once. That # exceeds the default ring window (16384 slots). Run it explicitly - # with a large PTO2_RING_TASK_WINDOW if needed; the GM heap needs no + # with a large runtime_env.ring_task_window if needed; the GM heap needs no # knob, since it is committed to the size orchestration measured. "name": "Case1", "platforms": ["a5"], diff --git a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp index cfdb3543a6..a3aaa6b205 100644 --- a/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a2a3/test_orchestrator_fanin.cpp @@ -270,7 +270,7 @@ TEST_F(OrchestratorFaninTest, SubmitPathHeapDeadlockLogReportsRingAndRealHeapSta EXPECT_NE(log.find("Heap ring 1:"), std::string::npos); EXPECT_NE(log.find("used=3072"), std::string::npos); EXPECT_NE(log.find("available=1024"), std::string::npos); - EXPECT_EQ(log.find("PTO2_RING_HEAP="), std::string::npos); + EXPECT_EQ(log.find("runtime_env.ring_heap="), std::string::npos); } TEST_F(OrchestratorFaninTest, StructuralCheckRejectsOpenAncestorWhenNestedScopesShareRing) { diff --git a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp index 0f94bf3ebe..6ea756e409 100644 --- a/tests/ut/cpp/a5/test_orchestrator_fanin.cpp +++ b/tests/ut/cpp/a5/test_orchestrator_fanin.cpp @@ -283,7 +283,7 @@ TEST_F(OrchestratorFaninTest, SubmitPathHeapDeadlockLogReportsRingAndRealHeapSta EXPECT_NE(log.find("Heap ring 1:"), std::string::npos); EXPECT_NE(log.find("used=3072"), std::string::npos); EXPECT_NE(log.find("available=1024"), std::string::npos); - EXPECT_EQ(log.find("PTO2_RING_HEAP="), std::string::npos); + EXPECT_EQ(log.find("runtime_env.ring_heap="), std::string::npos); } TEST_F(OrchestratorFaninTest, StructuralCheckRejectsOpenAncestorWhenNestedScopesShareRing) { From 08589fb7db8556950d8fb145d9dfa3b7de296a24 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Mon, 24 Aug 2026 18:40:22 -0700 Subject: [PATCH 22/22] Fix: ship the cmake includes the platform builds need in the wheel `simpler_setup/runtime_compiler.py` sets `SIMPLER_CMAKE_DIR` to `PROJECT_ROOT / "cmake"`, which is `simpler_setup/_assets/cmake` in an installed package. The `SKBUILD_MODE` install block never created it, so every runtime cmake configure from a wheel install failed on the first `include()`: CMake Error at CMakeLists.txt:21 (include): include could not find requested file: .../site-packages/simpler_setup/_assets/cmake/host_log_sources.cmake which then surfaces as `ValueError: empty compile database` from the clang-tidy pre-commit hook, and because every gated CI job declares `pre-commit` in `needs:`, it skips the entire test matrix rather than failing one job. All four files under `cmake/` are reachable from a platform configure, so the whole directory ships: `host_log_sources.cmake` and `profiling_config.cmake` are included unconditionally by the platform `CMakeLists.txt`, `profiling_config.cmake` configures `profiling_build_config.h.in`, and `sanitizers.cmake` is included for a sanitizer build. Installing only the file named in the error would have failed at the next `include()` instead. Dates from #1845, which added `include("${SIMPLER_CMAKE_DIR}/host_log_sources.cmake")` to six platform `CMakeLists.txt` without an install rule to match. It stayed hidden because the hook only configures on a build-cache miss, and `tools/verify_packaging.sh` checks the entry points rather than the asset tree. Verified by building the wheel both ways: `pip wheel --no-build-isolation --no-deps .` produces zero `simpler_setup/_assets/cmake/*` entries before this change and all four after. Co-Authored-By: Claude Opus 5 (1M context) --- CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 842be96b0b..90f0495fbe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,15 @@ add_custom_target(build_package_sim if(SKBUILD_MODE) install(DIRECTORY ${CMAKE_SOURCE_DIR}/src/ DESTINATION simpler_setup/_assets/src) + # The platform CMakeLists include these through SIMPLER_CMAKE_DIR, which + # runtime_compiler.py resolves to PROJECT_ROOT/cmake — the installed + # simpler_setup/_assets/cmake in a wheel. All four are reachable from a + # platform configure: host_log_sources.cmake and profiling_config.cmake are + # included unconditionally, profiling_config.cmake configures + # profiling_build_config.h.in, and sanitizers.cmake is included when a + # sanitizer build is requested. + install(DIRECTORY ${CMAKE_SOURCE_DIR}/cmake/ + DESTINATION simpler_setup/_assets/cmake) install(DIRECTORY ${CMAKE_SOURCE_DIR}/build/lib/ DESTINATION simpler_setup/_assets/build/lib OPTIONAL