From b7fe1911fb86db9c9bacdc7a74e88a69d783d176 Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Thu, 6 Aug 2026 19:51:57 -0700 Subject: [PATCH 01/11] CI: update pinned PTO-ISA to 0cefc9a5 Qualify the dispatch DCCI cache-line arguments for the updated interface. --- .../workers/l3/ep_dispatch_combine/kernels/aiv/dispatch.cpp | 2 +- pto_isa.pin | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/workers/l3/ep_dispatch_combine/kernels/aiv/dispatch.cpp b/examples/workers/l3/ep_dispatch_combine/kernels/aiv/dispatch.cpp index 1201572110..37bb295be9 100644 --- a/examples/workers/l3/ep_dispatch_combine/kernels/aiv/dispatch.cpp +++ b/examples/workers/l3/ep_dispatch_combine/kernels/aiv/dispatch.cpp @@ -308,7 +308,7 @@ extern "C" __aicore__ __attribute__((always_inline)) void kernel_entry(__gm__ in // its row loop. Flush the one cache line ([L=16,1] INT32 = 64 B) so its HBM // value is visible regardless of dispatch/consumer timing. (recv_x/recv_w/ // recv_idx go out via TSTORE and need no dcci.) - dcci((__gm__ void *)recv_count_out, SINGLE_CACHE_LINE, CACHELINE_OUT); + dcci((__gm__ void *)recv_count_out, cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_OUT); // ------------------------------------------------------------------ // payload_push: push x / weight / idx payloads via TPUT. diff --git a/pto_isa.pin b/pto_isa.pin index c57af28923..614447802a 100644 --- a/pto_isa.pin +++ b/pto_isa.pin @@ -1 +1 @@ -83d01313d9bfc247c4b7c8bcf969d1019f0d106f +0cefc9a5a1c24c62655cc345d408559595a8af32 From df9bf962ded111e7f85341fc8b17b19fb200bd58 Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Fri, 7 Aug 2026 02:16:30 -0700 Subject: [PATCH 02/11] fix(pr): resolve issues for #1644 - migrate SDMA completion polling to post-done IDs - carry A2/A3 post IDs through deferred completion state - cover monotonic completion semantics with mailbox tests --- .../runtime/aicore_completion_mailbox.h | 18 +++++-- .../runtime/aicore_completion_mailbox_types.h | 3 +- .../backend/sdma/sdma_completion_kernel.h | 31 ++++++------ .../backend/sdma/sdma_completion_scheduler.h | 49 +++++-------------- .../runtime/pto_async_kernel_api.h | 1 + .../runtime/pto_async_wait.h | 16 +++--- .../runtime/pto_completion_token.h | 1 + .../scheduler/scheduler_completion.cpp | 4 +- .../backend/sdma/sdma_completion_kernel.h | 31 ++++++------ .../backend/sdma/sdma_completion_scheduler.h | 46 +++++------------ .../runtime/pto_async_wait.h | 6 ++- .../a2a3/test_aicore_completion_mailbox.cpp | 27 ++++++++-- .../cpp/a5/test_aicore_completion_mailbox.cpp | 31 ++++++++++-- 13 files changed, 142 insertions(+), 122 deletions(-) diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h index 0f73a043a2..f0d8a5a506 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h @@ -40,7 +40,7 @@ static_assert( // carries the slot_state pointer in `addr` so the consumer can finalize the // AsyncWaitEntry.slot_state binding for tasks whose conditions arrived // before the FIN thread saw task_complete. New kinds may be added in future -// without growing the message — the `_pad[5]` slack is reserved for +// without growing the message — the `_pad[3]` slack is reserved for // kind-specific payload extension. #define MSG_KIND_CONDITION 0u #define MSG_KIND_TASK_NORMAL_DONE 1u @@ -52,15 +52,16 @@ struct AICoreCompletionMailboxMessage { // other fields below as a side effect, so they stay plain. std::atomic seq; PTO2TaskId task_token; - // CONDITION: completion observation addr (counter / SDMA event record). + // CONDITION: completion observation addr (counter / backend record). // TASK_NORMAL_DONE: PTO2TaskSlotState pointer carried over to the consumer // so it can finalize the AsyncWaitEntry.slot_state binding. uint64_t addr; + uint64_t backend_cookie; uint32_t expected_value; uint32_t engine; int32_t completion_type; uint32_t kind; - uint32_t _pad[5]; + uint32_t _pad[3]; }; static_assert(sizeof(AICoreCompletionMailboxMessage) == PTO2_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); @@ -79,6 +80,7 @@ static_assert( struct AICoreCompletionMsgView { PTO2TaskId task_token{PTO2TaskId::invalid()}; uint64_t addr{0}; + uint64_t backend_cookie{0}; uint32_t expected_value{0}; uint32_t engine{0}; int32_t completion_type{0}; @@ -115,6 +117,13 @@ struct AICoreCompletionMailbox { // independent of the AsyncWaitList::busy lock. bool try_push_condition( PTO2TaskId task_token, uint64_t addr, uint32_t expected_value, uint32_t engine, int32_t completion_type + ) { + return try_push_condition(task_token, addr, 0, expected_value, engine, completion_type); + } + + bool try_push_condition( + PTO2TaskId task_token, uint64_t addr, uint64_t backend_cookie, uint32_t expected_value, uint32_t engine, + int32_t completion_type ) { while (true) { uint64_t h = head.load(std::memory_order_relaxed); @@ -125,6 +134,7 @@ struct AICoreCompletionMailbox { AICoreCompletionMailboxMessage *slot = &entries[h & AICORE_COMPLETION_MAILBOX_MASK]; slot->task_token.raw = task_token.raw; slot->addr = addr; + slot->backend_cookie = backend_cookie; slot->expected_value = expected_value; slot->engine = engine; slot->completion_type = completion_type; @@ -149,6 +159,7 @@ struct AICoreCompletionMailbox { AICoreCompletionMailboxMessage *slot = &entries[h & AICORE_COMPLETION_MAILBOX_MASK]; slot->task_token.raw = task_token.raw; slot->addr = slot_state_addr; + slot->backend_cookie = 0; slot->expected_value = 0; slot->engine = 0; slot->completion_type = 0; @@ -173,6 +184,7 @@ struct AICoreCompletionMailbox { if (slot->seq.load(std::memory_order_acquire) != t + 1) return false; out.task_token.raw = slot->task_token.raw; out.addr = slot->addr; + out.backend_cookie = slot->backend_cookie; out.expected_value = slot->expected_value; out.engine = slot->engine; out.completion_type = slot->completion_type; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h index da0d89ad7e..b687b45309 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox_types.h @@ -45,13 +45,14 @@ inline constexpr int32_t MAX_COMPLETIONS_PER_TASK = 64; // pin the compiler against caching / reordering on either side. struct DeferredCompletionEntry { uint64_t addr; + uint64_t backend_cookie; uint32_t expected_value; uint32_t engine; int32_t completion_type; uint32_t _pad; }; -static_assert(sizeof(DeferredCompletionEntry) == 24, "DeferredCompletionEntry layout drift"); +static_assert(sizeof(DeferredCompletionEntry) == 32, "DeferredCompletionEntry layout drift"); struct alignas(PTO2_ALIGN_SIZE) DeferredCompletionSlab { volatile uint32_t count; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_kernel.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_kernel.h index 8a24b48f2b..560d05b50c 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_kernel.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_kernel.h @@ -28,7 +28,7 @@ #endif // Re-exposed PTO-ISA constant so examples / callers don't need to include -// just to spell their scratch tile. +// just to spell their scratch tile. inline constexpr uint32_t SDMA_SCRATCH_ALIGNMENT = pto::comm::sdma::UB_ALIGN_SIZE; enum class SdmaOp : uint8_t { @@ -42,11 +42,9 @@ enum class SdmaOp : uint8_t { // time; the SdmaTget() / SdmaTput() helpers below let callers skip the // template arguments. // -// sync_id selects which event-record slot inside the workspace the engine -// writes into. Concurrent dispatches must use distinct sync_ids; today every -// caller submits one request per kernel invocation so passing 0 is safe. -// Future work -// will fold sync_id allocation into the adapter. +// sync_id is forwarded to PTO-ISA's session builder for its internal pipe +// synchronization. Today every caller submits one request per kernel +// invocation, so passing 0 is safe. template struct SdmaRequestDescriptor { SdmaOp op; @@ -77,9 +75,10 @@ inline __aicore__ SdmaRequestDescriptor Sdma namespace pto2::detail { -inline __aicore__ void register_sdma_event_record(AsyncCtx &ctx, volatile __gm__ void *record_addr) { +inline __aicore__ void +register_sdma_post_done_record(AsyncCtx &ctx, volatile __gm__ void *record_addr, uint64_t post_id) { CompletionToken token{ - reinterpret_cast(record_addr), 0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD, 0 + reinterpret_cast(record_addr), 0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD, post_id }; (void)register_completion_condition(ctx, token); } @@ -101,25 +100,25 @@ register_pto_async_event(AsyncCtx &ctx, const PtoAsyncEvent &event, const PtoAsy return; } - ::pto::comm::sdma::detail::UbTmpBuf tmp_buf; - uint32_t sync_id = 0; - __gm__ uint8_t *recv_workspace = nullptr; + uint64_t post_id = 0; uint32_t queue_num = 0; - if (!::pto::comm::sdma::detail::PrepareEventCheck( - session.sdmaSession, tmp_buf, sync_id, recv_workspace, queue_num - )) { + __gm__ uint8_t *post_done_base = session.sdmaRuntimeCtx.postDoneBase; + if (!::pto::comm::sdma::detail::DecodeSdmaEventHandle(event.handle, post_id, queue_num) || + post_done_base == nullptr) { defer_error(ctx, PTO2_ERROR_ASYNC_COMPLETION_INVALID); return; } for (uint32_t queue_id = 0; queue_id < queue_num; ++queue_id) { - register_sdma_event_record(ctx, ::pto::comm::sdma::detail::GetEventRecord(recv_workspace, queue_id)); + register_sdma_post_done_record( + ctx, ::pto::comm::sdma::detail::GetPostDoneRecordAddr(post_done_base, queue_id), post_id + ); } } } // namespace pto2::detail // SDMA overload of the runtime's send_request_entry. Submits the descriptor -// to PTO-ISA, then registers the resulting AsyncEvent's GM flag(s) into the +// to PTO-ISA, then registers the resulting AsyncEvent's GM post-done records into the // AsyncCtx deferred-wait slab and flushes. Returns false on submit/session // failure (also records the error in ctx.completion_error_code). template diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h index 689219c355..fdbfbc8f13 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h @@ -19,48 +19,25 @@ #include "pto_completion_token.h" #include "pto_runtime_status.h" -// runtime-side mirror of the PTO-ISA SdmaEventRecord. SDMA backend is the only -// allowed holder of this ABI knowledge; the generic scheduler dispatches into -// the helpers below through the completion ops table. -struct SdmaEventRecord { - uint32_t flag; - uint32_t sq_tail; - uint64_t channel_info; -}; - -static_assert(sizeof(SdmaEventRecord) == 16, "SDMA event record ABI drift"); -static_assert(offsetof(SdmaEventRecord, sq_tail) == 4, "SDMA event record ABI drift"); - inline uintptr_t sdma_completion_cache_line(const volatile void *addr) { return reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); } -inline CompletionPollResult poll_sdma_event_record(uint64_t record_addr) { - if (record_addr == 0) { +// PTO-ISA stores the latest completed post ID as a monotonic uint64_t in one +// cache-line-sized record per queue. The completion type keeps its historical +// name, but the scheduler must compare the record with the post ID carried in +// backend_cookie and must not clear or otherwise retire the shared record. +inline CompletionPollResult poll_sdma_post_done_record(uint64_t record_addr, uint64_t expected_post_id) { + if (record_addr == 0 || expected_post_id == 0) { return {CompletionPollState::FAILED, PTO2_ERROR_ASYNC_COMPLETION_INVALID}; } - volatile SdmaEventRecord *record = - reinterpret_cast(static_cast(record_addr)); + volatile uint64_t *record = reinterpret_cast(static_cast(record_addr)); cache_invalidate_range(reinterpret_cast(sdma_completion_cache_line(record)), PTO2_ALIGN_SIZE); - uint32_t flag = __atomic_load_n(&record->flag, __ATOMIC_ACQUIRE); - return {flag != 0 ? CompletionPollState::READY : CompletionPollState::PENDING, PTO2_ERROR_NONE}; + uint64_t completed_post_id = __atomic_load_n(record, __ATOMIC_ACQUIRE); + return { + completed_post_id >= expected_post_id ? CompletionPollState::READY : CompletionPollState::PENDING, + PTO2_ERROR_NONE + }; } -inline void retire_sdma_event_record(uint64_t record_addr) { - if (record_addr == 0) return; - volatile SdmaEventRecord *record = - reinterpret_cast(static_cast(record_addr)); - cache_invalidate_range(reinterpret_cast(sdma_completion_cache_line(record)), PTO2_ALIGN_SIZE); - uint32_t completed_tail = __atomic_load_n(&record->sq_tail, __ATOMIC_ACQUIRE); - uint64_t channel_info_addr = __atomic_load_n(&record->channel_info, __ATOMIC_ACQUIRE); - - volatile uint64_t *record_head = reinterpret_cast(record); - __atomic_store_n(record_head, 0ULL, __ATOMIC_RELEASE); - cache_flush_range(const_cast(reinterpret_cast(record_head)), sizeof(uint64_t)); - - if (channel_info_addr == 0) return; - uint64_t packed = (static_cast(completed_tail) << 32) | static_cast(completed_tail); - volatile uint64_t *channel_info = reinterpret_cast(static_cast(channel_info_addr)); - __atomic_store_n(channel_info, packed, __ATOMIC_RELEASE); - cache_flush_range(const_cast(reinterpret_cast(channel_info)), sizeof(uint64_t)); -} +inline void retire_sdma_post_done_record(uint64_t /*record_addr*/, uint64_t /*expected_post_id*/) {} diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_async_kernel_api.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_async_kernel_api.h index cf6eb47901..f350d7c3e3 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_async_kernel_api.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_async_kernel_api.h @@ -130,6 +130,7 @@ inline __aicore__ bool register_completion_condition(AsyncCtx &ctx, const Comple volatile __gm__ DeferredCompletionEntry *slot = &ctx.completion_entries[idx]; slot->addr = token.addr; + slot->backend_cookie = token.backend_cookie; slot->expected_value = token.expected_value; slot->engine = token.engine; slot->completion_type = token.completion_type; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_async_wait.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_async_wait.h index 2f8f718fb5..564dd1bd42 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_async_wait.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_async_wait.h @@ -54,6 +54,7 @@ struct CompletionCondition { bool retired{false}; volatile uint32_t *counter_addr{nullptr}; uint64_t addr{0}; + uint64_t backend_cookie{0}; uint32_t expected_value{0}; CompletionPollResult test() const; @@ -76,10 +77,12 @@ inline CompletionPollResult counter_poll_op(const CompletionCondition &cond) { inline void counter_retire_op(CompletionCondition & /*cond*/) {} inline CompletionPollResult sdma_event_record_poll_op(const CompletionCondition &cond) { - return poll_sdma_event_record(cond.addr); + return poll_sdma_post_done_record(cond.addr, cond.backend_cookie); } -inline void sdma_event_record_retire_op(CompletionCondition &cond) { retire_sdma_event_record(cond.addr); } +inline void sdma_event_record_retire_op(CompletionCondition &cond) { + retire_sdma_post_done_record(cond.addr, cond.backend_cookie); +} inline const CompletionBackendOps *completion_backend_ops_for(int completion_type) { static const CompletionBackendOps kOps[] = { @@ -226,8 +229,8 @@ struct AsyncWaitList { entry->normal_done = false; } if (!append_condition_locked( - *entry, msg.addr, msg.expected_value, static_cast(msg.engine), msg.completion_type, - error_code + *entry, msg.addr, msg.backend_cookie, msg.expected_value, static_cast(msg.engine), + msg.completion_type, error_code )) { return drained; } @@ -271,8 +274,8 @@ struct AsyncWaitList { } bool append_condition_locked( - AsyncWaitEntry &entry, uint64_t addr, uint32_t expected_value, AsyncEngine engine, int32_t completion_type, - int32_t &error_code + AsyncWaitEntry &entry, uint64_t addr, uint64_t backend_cookie, uint32_t expected_value, AsyncEngine engine, + int32_t completion_type, int32_t &error_code ) { if (entry.condition_count >= MAX_COMPLETIONS_PER_TASK) { error_code = PTO2_ERROR_ASYNC_REGISTRATION_FAILED; @@ -284,6 +287,7 @@ struct AsyncWaitList { cond.satisfied = false; cond.retired = false; cond.addr = addr; + cond.backend_cookie = backend_cookie; cond.counter_addr = completion_type == COMPLETION_TYPE_COUNTER ? reinterpret_cast(static_cast(addr)) : nullptr; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_completion_token.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_completion_token.h index c5a8c345f3..3c165b3879 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_completion_token.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_completion_token.h @@ -28,6 +28,7 @@ struct CompletionToken { uint32_t expected_value; uint32_t engine; int32_t completion_type; + // Backend-specific 64-bit metadata (for example, an SDMA post ID). uint64_t backend_cookie; }; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index 165974d881..c3842dbf14 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -140,7 +140,9 @@ void SchedulerContext::complete_slot_task( const PTO2TaskId token = slot_state.task->task_id; for (uint32_t i = 0; i < cond_count; ++i) { volatile DeferredCompletionEntry *e = &deferred_slab->entries[i]; - while (!mailbox->try_push_condition(token, e->addr, e->expected_value, e->engine, e->completion_type)) { + while (!mailbox->try_push_condition( + token, e->addr, e->backend_cookie, e->expected_value, e->engine, e->completion_type + )) { sched_->async_wait_list.mpsc_skipped_count.fetch_add(1, std::memory_order_relaxed); SPIN_WAIT_HINT(); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_kernel.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_kernel.h index 1de677520c..eb37ad2f13 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_kernel.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_kernel.h @@ -29,7 +29,7 @@ #endif // Re-exposed PTO-ISA constant so examples / callers don't need to include -// just to spell their scratch tile. +// just to spell their scratch tile. inline constexpr uint32_t SDMA_SCRATCH_ALIGNMENT = pto::comm::sdma::UB_ALIGN_SIZE; enum class SdmaOp : uint8_t { @@ -43,11 +43,9 @@ enum class SdmaOp : uint8_t { // time; the SdmaTget() / SdmaTput() helpers below let callers skip the // template arguments. // -// sync_id selects which event-record slot inside the workspace the engine -// writes into. Concurrent dispatches must use distinct sync_ids; today every -// caller submits one request per kernel invocation so passing 0 is safe. -// Future work -// will fold sync_id allocation into the adapter. +// sync_id is forwarded to PTO-ISA's session builder for its internal pipe +// synchronization. Today every caller submits one request per kernel +// invocation, so passing 0 is safe. template struct SdmaRequestDescriptor { SdmaOp op; @@ -78,9 +76,10 @@ inline __aicore__ SdmaRequestDescriptor Sdma namespace pto2::detail { -inline __aicore__ void register_sdma_event_record(AsyncCtx &ctx, volatile __gm__ void *record_addr) { +inline __aicore__ void +register_sdma_post_done_record(AsyncCtx &ctx, volatile __gm__ void *record_addr, uint64_t post_id) { CompletionToken token{ - reinterpret_cast(record_addr), 0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD, 0 + reinterpret_cast(record_addr), 0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD, post_id }; (void)register_completion_condition(ctx, token); } @@ -102,25 +101,25 @@ register_pto_async_event(AsyncCtx &ctx, const PtoAsyncEvent &event, const PtoAsy return; } - ::pto::comm::sdma::detail::UbTmpBuf tmp_buf; - uint32_t sync_id = 0; - __gm__ uint8_t *recv_workspace = nullptr; + uint64_t post_id = 0; uint32_t queue_num = 0; - if (!::pto::comm::sdma::detail::PrepareEventCheck( - session.sdmaSession, tmp_buf, sync_id, recv_workspace, queue_num - )) { + __gm__ uint8_t *post_done_base = session.sdmaRuntimeCtx.postDoneBase; + if (!::pto::comm::sdma::detail::DecodeSdmaEventHandle(event.handle, post_id, queue_num) || + post_done_base == nullptr) { defer_error(ctx, PTO2_ERROR_ASYNC_COMPLETION_INVALID); return; } for (uint32_t queue_id = 0; queue_id < queue_num; ++queue_id) { - register_sdma_event_record(ctx, ::pto::comm::sdma::detail::GetEventRecord(recv_workspace, queue_id)); + register_sdma_post_done_record( + ctx, ::pto::comm::sdma::detail::GetPostDoneRecordAddr(post_done_base, queue_id), post_id + ); } } } // namespace pto2::detail // SDMA overload of the runtime's send_request_entry. Submits the descriptor -// to PTO-ISA, then registers the resulting AsyncEvent's GM flag(s) into the +// to PTO-ISA, then registers the resulting AsyncEvent's GM post-done records into the // AsyncCtx deferred-wait slab and flushes. Returns false on submit/session // failure (also records the error in ctx.completion_error_code). template diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h index c3ea2eecf9..3bca00ea1e 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/backend/sdma/sdma_completion_scheduler.h @@ -20,42 +20,22 @@ #include "pto_completion_token.h" #include "pto_runtime_status.h" -// runtime-side mirror of the PTO-ISA SdmaEventRecord. SDMA backend is the only -// allowed holder of this ABI knowledge; the generic scheduler dispatches into -// the helpers below through the completion ops table. -struct SdmaEventRecord { - uint32_t flag; - uint32_t sq_tail; - uint64_t channel_info; -}; - -static_assert(sizeof(SdmaEventRecord) == 16, "SDMA event record ABI drift"); -static_assert(offsetof(SdmaEventRecord, sq_tail) == 4, "SDMA event record ABI drift"); - -inline CompletionPollResult poll_sdma_event_record(uint64_t record_addr) { - if (record_addr == 0) { +// PTO-ISA stores the latest completed post ID as a monotonic uint64_t in one +// cache-line-sized record per queue. The completion type keeps its historical +// name, but the scheduler must compare the record with the post ID carried in +// backend_cookie and must not clear or otherwise retire the shared record. +inline CompletionPollResult poll_sdma_post_done_record(uint64_t record_addr, uint64_t expected_post_id) { + if (record_addr == 0 || expected_post_id == 0) { return {CompletionPollState::FAILED, PTO2_ERROR_ASYNC_COMPLETION_INVALID}; } - volatile SdmaEventRecord *record = - reinterpret_cast(static_cast(record_addr)); - uint32_t flag = __atomic_load_n(&record->flag, __ATOMIC_ACQUIRE); - return {flag != 0 ? CompletionPollState::READY : CompletionPollState::PENDING, PTO2_ERROR_NONE}; + volatile uint64_t *record = reinterpret_cast(static_cast(record_addr)); + uint64_t completed_post_id = __atomic_load_n(record, __ATOMIC_ACQUIRE); + return { + completed_post_id >= expected_post_id ? CompletionPollState::READY : CompletionPollState::PENDING, + PTO2_ERROR_NONE + }; } -inline void retire_sdma_event_record(uint64_t record_addr) { - if (record_addr == 0) return; - volatile SdmaEventRecord *record = - reinterpret_cast(static_cast(record_addr)); - uint32_t completed_tail = __atomic_load_n(&record->sq_tail, __ATOMIC_ACQUIRE); - uint64_t channel_info_addr = __atomic_load_n(&record->channel_info, __ATOMIC_ACQUIRE); - - volatile uint64_t *record_head = reinterpret_cast(record); - __atomic_store_n(record_head, 0ULL, __ATOMIC_RELEASE); - - if (channel_info_addr == 0) return; - uint64_t packed = (static_cast(completed_tail) << 32) | static_cast(completed_tail); - volatile uint64_t *channel_info = reinterpret_cast(static_cast(channel_info_addr)); - __atomic_store_n(channel_info, packed, __ATOMIC_RELEASE); -} +inline void retire_sdma_post_done_record(uint64_t /*record_addr*/, uint64_t /*expected_post_id*/) {} #endif // SRC_A5_RUNTIME_TENSORMAP_AND_RINGBUFFER_RUNTIME_BACKEND_SDMA_SDMA_COMPLETION_SCHEDULER_H_ diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_async_wait.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_async_wait.h index cf6383519f..3a6d39afa8 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_async_wait.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_async_wait.h @@ -74,10 +74,12 @@ inline CompletionPollResult counter_poll_op(const CompletionCondition &cond) { inline void counter_retire_op(CompletionCondition & /*cond*/) {} inline CompletionPollResult sdma_event_record_poll_op(const CompletionCondition &cond) { - return poll_sdma_event_record(cond.addr); + return poll_sdma_post_done_record(cond.addr, cond.backend_cookie); } -inline void sdma_event_record_retire_op(CompletionCondition &cond) { retire_sdma_event_record(cond.addr); } +inline void sdma_event_record_retire_op(CompletionCondition &cond) { + retire_sdma_post_done_record(cond.addr, cond.backend_cookie); +} inline CompletionPollResult urma_event_handle_poll_op(const CompletionCondition &cond) { return pto2::urma_backend::poll_urma_event_handle(cond.addr, cond.backend_cookie); diff --git a/tests/ut/cpp/a2a3/test_aicore_completion_mailbox.cpp b/tests/ut/cpp/a2a3/test_aicore_completion_mailbox.cpp index 5fbdb21284..fd43918776 100644 --- a/tests/ut/cpp/a2a3/test_aicore_completion_mailbox.cpp +++ b/tests/ut/cpp/a2a3/test_aicore_completion_mailbox.cpp @@ -156,10 +156,11 @@ TEST(AICoreCompletionMailbox, ConditionAttachesToExistingEntry) { constexpr uint64_t kAddr1 = 0x1000; constexpr uint64_t kAddr2 = 0x2000; + constexpr uint64_t kPostId = (uint64_t{1} << 40) + 17; ASSERT_TRUE(mb->try_push_condition(token, kAddr1, /*expected=*/3, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_COUNTER)); - ASSERT_TRUE( - mb->try_push_condition(token, kAddr2, /*expected=*/0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD) - ); + ASSERT_TRUE(mb->try_push_condition( + token, kAddr2, kPostId, /*expected=*/0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD + )); int32_t err = PTO2_ERROR_NONE; AsyncWaitList::DrainCompletionSink sink{}; @@ -171,6 +172,7 @@ TEST(AICoreCompletionMailbox, ConditionAttachesToExistingEntry) { EXPECT_EQ(wait_list.entries[0].conditions[0].expected_value, 3u); EXPECT_EQ(wait_list.entries[0].conditions[0].completion_type, COMPLETION_TYPE_COUNTER); EXPECT_EQ(wait_list.entries[0].conditions[1].addr, kAddr2); + EXPECT_EQ(wait_list.entries[0].conditions[1].backend_cookie, kPostId); EXPECT_EQ(wait_list.entries[0].conditions[1].completion_type, COMPLETION_TYPE_SDMA_EVENT_RECORD); // SDMA_EVENT_RECORD conditions don't bind counter_addr. EXPECT_EQ(wait_list.entries[0].conditions[1].counter_addr, nullptr); @@ -179,6 +181,25 @@ TEST(AICoreCompletionMailbox, ConditionAttachesToExistingEntry) { destroy_mailbox(mb); } +TEST(AICoreCompletionMailbox, SdmaPostDoneRecordUsesMonotonicPostId) { + alignas(PTO2_ALIGN_SIZE) uint64_t completed_post_id = (uint64_t{1} << 40) + 16; + CompletionCondition cond{}; + cond.completion_type = COMPLETION_TYPE_SDMA_EVENT_RECORD; + cond.addr = reinterpret_cast(&completed_post_id); + cond.backend_cookie = completed_post_id + 1; + + EXPECT_EQ(cond.test().state, CompletionPollState::PENDING); + completed_post_id += 1; + EXPECT_EQ(cond.test().state, CompletionPollState::READY); + completed_post_id += 1; + EXPECT_EQ(cond.test().state, CompletionPollState::READY); + EXPECT_EQ(poll_sdma_post_done_record(/*record_addr=*/0, cond.backend_cookie).state, CompletionPollState::FAILED); + EXPECT_EQ(poll_sdma_post_done_record(cond.addr, /*expected_post_id=*/0).state, CompletionPollState::FAILED); + + cond.retire(); + EXPECT_EQ(completed_post_id, (uint64_t{1} << 40) + 18); +} + // ============================================================================= // Capacity / overflow // ============================================================================= diff --git a/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp b/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp index e3aae03bfb..3c0a9a62db 100644 --- a/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp +++ b/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp @@ -136,12 +136,13 @@ TEST(A5AICoreCompletionMailbox, ConditionAttachesToExistingEntry) { wait_list.entries[0].normal_done = false; wait_list.count = 1; - // Both conditions here are COUNTER and bind counter_addr / expected_value; - // a5 also registers SDMA event-record and URMA event-handle backends. constexpr uint64_t kAddr1 = 0x1000; constexpr uint64_t kAddr2 = 0x2000; + constexpr uint64_t kPostId = (uint64_t{1} << 40) + 17; ASSERT_TRUE(mb->try_push_condition(token, kAddr1, /*expected=*/3, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_COUNTER)); - ASSERT_TRUE(mb->try_push_condition(token, kAddr2, /*expected=*/9, COMPLETION_ENGINE_ROCE, COMPLETION_TYPE_COUNTER)); + ASSERT_TRUE(mb->try_push_condition( + token, kAddr2, kPostId, /*expected=*/0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD + )); int32_t err = PTO2_ERROR_NONE; AsyncWaitList::DrainCompletionSink sink{}; @@ -151,13 +152,33 @@ TEST(A5AICoreCompletionMailbox, ConditionAttachesToExistingEntry) { ASSERT_EQ(wait_list.entries[0].condition_count, 2); EXPECT_EQ(reinterpret_cast(wait_list.entries[0].conditions[0].counter_addr), kAddr1); EXPECT_EQ(wait_list.entries[0].conditions[0].expected_value, 3u); - EXPECT_EQ(reinterpret_cast(wait_list.entries[0].conditions[1].counter_addr), kAddr2); - EXPECT_EQ(wait_list.entries[0].conditions[1].expected_value, 9u); + EXPECT_EQ(wait_list.entries[0].conditions[1].addr, kAddr2); + EXPECT_EQ(wait_list.entries[0].conditions[1].backend_cookie, kPostId); + EXPECT_EQ(wait_list.entries[0].conditions[1].counter_addr, nullptr); EXPECT_EQ(wait_list.entries[0].waiting_completion_count, 2); destroy_mailbox(mb); } +TEST(A5AICoreCompletionMailbox, SdmaPostDoneRecordUsesMonotonicPostId) { + alignas(PTO2_ALIGN_SIZE) uint64_t completed_post_id = (uint64_t{1} << 40) + 16; + CompletionCondition cond{}; + cond.completion_type = COMPLETION_TYPE_SDMA_EVENT_RECORD; + cond.addr = reinterpret_cast(&completed_post_id); + cond.backend_cookie = completed_post_id + 1; + + EXPECT_EQ(cond.test().state, CompletionPollState::PENDING); + completed_post_id += 1; + EXPECT_EQ(cond.test().state, CompletionPollState::READY); + completed_post_id += 1; + EXPECT_EQ(cond.test().state, CompletionPollState::READY); + EXPECT_EQ(poll_sdma_post_done_record(/*record_addr=*/0, cond.backend_cookie).state, CompletionPollState::FAILED); + EXPECT_EQ(poll_sdma_post_done_record(cond.addr, /*expected_post_id=*/0).state, CompletionPollState::FAILED); + + cond.retire(); + EXPECT_EQ(completed_post_id, (uint64_t{1} << 40) + 18); +} + TEST(A5AICoreCompletionMailbox, PushReturnsFalseWhenFull) { AICoreCompletionMailbox *mb = fresh_mailbox(); From b331a70384a387eea524350d6eeb4992c0b28a4b Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Fri, 7 Aug 2026 03:01:06 -0700 Subject: [PATCH 03/11] fix(pr): resolve issues for #1644 - fetch the pinned PTO-ISA commit by exact SHA - retain advertised-ref fallback for restrictive Git servers - cover both resolver fetch paths with unit tests --- simpler_setup/pto_isa.py | 16 ++++++++++++++-- tests/ut/py/test_pto_isa.py | 23 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/simpler_setup/pto_isa.py b/simpler_setup/pto_isa.py index 88352c7766..bc1c53c0a5 100644 --- a/simpler_setup/pto_isa.py +++ b/simpler_setup/pto_isa.py @@ -334,6 +334,9 @@ def _land_on_commit(clone_path: Path, commit: str, verbose: bool) -> bool: would be overwritten." Forcing discards that pseudo-dirt and lands exactly on the pin. A full clone already carries every branch's history, but keep a fetch fallback in case the pin is not reachable from the default fetch. + Fetch the exact pin first so a stale remote-ref advertisement cannot make a + newly-pushed commit invisible; fall back to the advertised refs for servers + that do not allow fetching a reachable object by SHA. """ try: result = _run_git_resilient( @@ -341,8 +344,17 @@ def _land_on_commit(clone_path: Path, commit: str, verbose: bool) -> bool: ) if result.returncode != 0: if verbose: - logger.info(f"pto-isa commit {commit} missing locally, fetching origin...") - _run_git_resilient(["fetch", "origin"], cwd=clone_path, timeout=120, check=True, verbose=verbose) + logger.info(f"pto-isa commit {commit} missing locally, fetching the exact pin from origin...") + fetch_result = _run_git_resilient( + ["fetch", "--no-tags", "origin", commit], + cwd=clone_path, + timeout=120, + verbose=verbose, + ) + if fetch_result.returncode != 0: + if verbose: + logger.info("Exact PTO-ISA pin fetch failed; falling back to advertised origin refs...") + _run_git_resilient(["fetch", "origin"], cwd=clone_path, timeout=120, check=True, verbose=verbose) _run_git_resilient( ["checkout", "--detach", "--force", commit], cwd=clone_path, timeout=30, check=True, verbose=verbose ) diff --git a/tests/ut/py/test_pto_isa.py b/tests/ut/py/test_pto_isa.py index ea2879becd..5e4cf67ee5 100644 --- a/tests/ut/py/test_pto_isa.py +++ b/tests/ut/py/test_pto_isa.py @@ -194,6 +194,29 @@ def fake_run_git(args, cwd=None, timeout=30, check=False): assert pto_isa._land_on_commit(tmp_path, PIN_A, verbose=False) assert calls == [ (["checkout", "--detach", "--force", PIN_A], tmp_path, False), + (["fetch", "--no-tags", "origin", PIN_A], tmp_path, False), + (["checkout", "--detach", "--force", PIN_A], tmp_path, False), + ] + + +def test_land_on_commit_falls_back_when_exact_fetch_is_unsupported(tmp_path, monkeypatch): + calls = [] + + def fake_run_git(args, cwd=None, timeout=30, check=False): + calls.append((args, cwd, check)) + if args == ["checkout", "--detach", "--force", PIN_A] and len([c for c in calls if c[0] == args]) == 1: + return subprocess.CompletedProcess(["git", *args], returncode=1, stdout="", stderr="missing") + if args == ["fetch", "--no-tags", "origin", PIN_A]: + return subprocess.CompletedProcess(["git", *args], returncode=1, stdout="", stderr="not our ref") + return subprocess.CompletedProcess(["git", *args], returncode=0, stdout="", stderr="") + + monkeypatch.setattr(pto_isa, "_run_git", fake_run_git) + monkeypatch.setattr(pto_isa, "get_pto_isa_head", lambda root: PIN_A) + + assert pto_isa._land_on_commit(tmp_path, PIN_A, verbose=False) + assert calls == [ + (["checkout", "--detach", "--force", PIN_A], tmp_path, False), + (["fetch", "--no-tags", "origin", PIN_A], tmp_path, False), (["fetch", "origin"], tmp_path, False), (["checkout", "--detach", "--force", PIN_A], tmp_path, False), ] From 1b4017fbb6f0a1d12e1b0311bb7731bcf99ebbfc Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Sun, 9 Aug 2026 18:53:38 -0700 Subject: [PATCH 04/11] fix(pr): resolve issues for #1644 - adapt host-build-graph SDMA completion to post IDs - carry SDMA backend cookies through deferred completion --- .../runtime/aicore_completion_mailbox.h | 18 +++++-- .../runtime/aicore_completion_mailbox_types.h | 3 +- .../backend/sdma/sdma_completion_kernel.h | 33 ++++++------- .../backend/sdma/sdma_completion_scheduler.h | 49 +++++-------------- .../runtime/pto_async_kernel_api.h | 1 + .../host_build_graph/runtime/pto_async_wait.h | 16 +++--- .../runtime/pto_completion_token.h | 1 + .../scheduler/scheduler_completion.cpp | 4 +- 8 files changed, 61 insertions(+), 64 deletions(-) diff --git a/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox.h b/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox.h index 0f73a043a2..f0d8a5a506 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox.h +++ b/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox.h @@ -40,7 +40,7 @@ static_assert( // carries the slot_state pointer in `addr` so the consumer can finalize the // AsyncWaitEntry.slot_state binding for tasks whose conditions arrived // before the FIN thread saw task_complete. New kinds may be added in future -// without growing the message — the `_pad[5]` slack is reserved for +// without growing the message — the `_pad[3]` slack is reserved for // kind-specific payload extension. #define MSG_KIND_CONDITION 0u #define MSG_KIND_TASK_NORMAL_DONE 1u @@ -52,15 +52,16 @@ struct AICoreCompletionMailboxMessage { // other fields below as a side effect, so they stay plain. std::atomic seq; PTO2TaskId task_token; - // CONDITION: completion observation addr (counter / SDMA event record). + // CONDITION: completion observation addr (counter / backend record). // TASK_NORMAL_DONE: PTO2TaskSlotState pointer carried over to the consumer // so it can finalize the AsyncWaitEntry.slot_state binding. uint64_t addr; + uint64_t backend_cookie; uint32_t expected_value; uint32_t engine; int32_t completion_type; uint32_t kind; - uint32_t _pad[5]; + uint32_t _pad[3]; }; static_assert(sizeof(AICoreCompletionMailboxMessage) == PTO2_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); @@ -79,6 +80,7 @@ static_assert( struct AICoreCompletionMsgView { PTO2TaskId task_token{PTO2TaskId::invalid()}; uint64_t addr{0}; + uint64_t backend_cookie{0}; uint32_t expected_value{0}; uint32_t engine{0}; int32_t completion_type{0}; @@ -115,6 +117,13 @@ struct AICoreCompletionMailbox { // independent of the AsyncWaitList::busy lock. bool try_push_condition( PTO2TaskId task_token, uint64_t addr, uint32_t expected_value, uint32_t engine, int32_t completion_type + ) { + return try_push_condition(task_token, addr, 0, expected_value, engine, completion_type); + } + + bool try_push_condition( + PTO2TaskId task_token, uint64_t addr, uint64_t backend_cookie, uint32_t expected_value, uint32_t engine, + int32_t completion_type ) { while (true) { uint64_t h = head.load(std::memory_order_relaxed); @@ -125,6 +134,7 @@ struct AICoreCompletionMailbox { AICoreCompletionMailboxMessage *slot = &entries[h & AICORE_COMPLETION_MAILBOX_MASK]; slot->task_token.raw = task_token.raw; slot->addr = addr; + slot->backend_cookie = backend_cookie; slot->expected_value = expected_value; slot->engine = engine; slot->completion_type = completion_type; @@ -149,6 +159,7 @@ struct AICoreCompletionMailbox { AICoreCompletionMailboxMessage *slot = &entries[h & AICORE_COMPLETION_MAILBOX_MASK]; slot->task_token.raw = task_token.raw; slot->addr = slot_state_addr; + slot->backend_cookie = 0; slot->expected_value = 0; slot->engine = 0; slot->completion_type = 0; @@ -173,6 +184,7 @@ struct AICoreCompletionMailbox { if (slot->seq.load(std::memory_order_acquire) != t + 1) return false; out.task_token.raw = slot->task_token.raw; out.addr = slot->addr; + out.backend_cookie = slot->backend_cookie; out.expected_value = slot->expected_value; out.engine = slot->engine; out.completion_type = slot->completion_type; diff --git a/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h b/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h index da0d89ad7e..b687b45309 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h +++ b/src/a2a3/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h @@ -45,13 +45,14 @@ inline constexpr int32_t MAX_COMPLETIONS_PER_TASK = 64; // pin the compiler against caching / reordering on either side. struct DeferredCompletionEntry { uint64_t addr; + uint64_t backend_cookie; uint32_t expected_value; uint32_t engine; int32_t completion_type; uint32_t _pad; }; -static_assert(sizeof(DeferredCompletionEntry) == 24, "DeferredCompletionEntry layout drift"); +static_assert(sizeof(DeferredCompletionEntry) == 32, "DeferredCompletionEntry layout drift"); struct alignas(PTO2_ALIGN_SIZE) DeferredCompletionSlab { volatile uint32_t count; diff --git a/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_kernel.h b/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_kernel.h index 81aaab111b..560d05b50c 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_kernel.h +++ b/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_kernel.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include "pto_async_kernel_api.h" #include "aicore_completion_mailbox_types.h" @@ -28,7 +28,7 @@ #endif // Re-exposed PTO-ISA constant so examples / callers don't need to include -// just to spell their scratch tile. +// just to spell their scratch tile. inline constexpr uint32_t SDMA_SCRATCH_ALIGNMENT = pto::comm::sdma::UB_ALIGN_SIZE; enum class SdmaOp : uint8_t { @@ -42,11 +42,9 @@ enum class SdmaOp : uint8_t { // time; the SdmaTget() / SdmaTput() helpers below let callers skip the // template arguments. // -// sync_id selects which event-record slot inside the workspace the engine -// writes into. Concurrent dispatches must use distinct sync_ids; today every -// caller submits one request per kernel invocation so passing 0 is safe. -// Future work -// will fold sync_id allocation into the adapter. +// sync_id is forwarded to PTO-ISA's session builder for its internal pipe +// synchronization. Today every caller submits one request per kernel +// invocation, so passing 0 is safe. template struct SdmaRequestDescriptor { SdmaOp op; @@ -77,9 +75,10 @@ inline __aicore__ SdmaRequestDescriptor Sdma namespace pto2::detail { -inline __aicore__ void register_sdma_event_record(AsyncCtx &ctx, volatile __gm__ void *record_addr) { +inline __aicore__ void +register_sdma_post_done_record(AsyncCtx &ctx, volatile __gm__ void *record_addr, uint64_t post_id) { CompletionToken token{ - reinterpret_cast(record_addr), 0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD, 0 + reinterpret_cast(record_addr), 0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD, post_id }; (void)register_completion_condition(ctx, token); } @@ -101,25 +100,25 @@ register_pto_async_event(AsyncCtx &ctx, const PtoAsyncEvent &event, const PtoAsy return; } - ::pto::comm::sdma::detail::UbTmpBuf tmp_buf; - uint32_t sync_id = 0; - __gm__ uint8_t *recv_workspace = nullptr; + uint64_t post_id = 0; uint32_t queue_num = 0; - if (!::pto::comm::sdma::detail::PrepareEventCheck( - session.sdmaSession, tmp_buf, sync_id, recv_workspace, queue_num - )) { + __gm__ uint8_t *post_done_base = session.sdmaRuntimeCtx.postDoneBase; + if (!::pto::comm::sdma::detail::DecodeSdmaEventHandle(event.handle, post_id, queue_num) || + post_done_base == nullptr) { defer_error(ctx, PTO2_ERROR_ASYNC_COMPLETION_INVALID); return; } for (uint32_t queue_id = 0; queue_id < queue_num; ++queue_id) { - register_sdma_event_record(ctx, ::pto::comm::sdma::detail::GetEventRecord(recv_workspace, queue_id)); + register_sdma_post_done_record( + ctx, ::pto::comm::sdma::detail::GetPostDoneRecordAddr(post_done_base, queue_id), post_id + ); } } } // namespace pto2::detail // SDMA overload of the runtime's send_request_entry. Submits the descriptor -// to PTO-ISA, then registers the resulting AsyncEvent's GM flag(s) into the +// to PTO-ISA, then registers the resulting AsyncEvent's GM post-done records into the // AsyncCtx deferred-wait slab and flushes. Returns false on submit/session // failure (also records the error in ctx.completion_error_code). template diff --git a/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h b/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h index 689219c355..fdbfbc8f13 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h +++ b/src/a2a3/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h @@ -19,48 +19,25 @@ #include "pto_completion_token.h" #include "pto_runtime_status.h" -// runtime-side mirror of the PTO-ISA SdmaEventRecord. SDMA backend is the only -// allowed holder of this ABI knowledge; the generic scheduler dispatches into -// the helpers below through the completion ops table. -struct SdmaEventRecord { - uint32_t flag; - uint32_t sq_tail; - uint64_t channel_info; -}; - -static_assert(sizeof(SdmaEventRecord) == 16, "SDMA event record ABI drift"); -static_assert(offsetof(SdmaEventRecord, sq_tail) == 4, "SDMA event record ABI drift"); - inline uintptr_t sdma_completion_cache_line(const volatile void *addr) { return reinterpret_cast(addr) & ~(uintptr_t(PTO2_ALIGN_SIZE) - 1u); } -inline CompletionPollResult poll_sdma_event_record(uint64_t record_addr) { - if (record_addr == 0) { +// PTO-ISA stores the latest completed post ID as a monotonic uint64_t in one +// cache-line-sized record per queue. The completion type keeps its historical +// name, but the scheduler must compare the record with the post ID carried in +// backend_cookie and must not clear or otherwise retire the shared record. +inline CompletionPollResult poll_sdma_post_done_record(uint64_t record_addr, uint64_t expected_post_id) { + if (record_addr == 0 || expected_post_id == 0) { return {CompletionPollState::FAILED, PTO2_ERROR_ASYNC_COMPLETION_INVALID}; } - volatile SdmaEventRecord *record = - reinterpret_cast(static_cast(record_addr)); + volatile uint64_t *record = reinterpret_cast(static_cast(record_addr)); cache_invalidate_range(reinterpret_cast(sdma_completion_cache_line(record)), PTO2_ALIGN_SIZE); - uint32_t flag = __atomic_load_n(&record->flag, __ATOMIC_ACQUIRE); - return {flag != 0 ? CompletionPollState::READY : CompletionPollState::PENDING, PTO2_ERROR_NONE}; + uint64_t completed_post_id = __atomic_load_n(record, __ATOMIC_ACQUIRE); + return { + completed_post_id >= expected_post_id ? CompletionPollState::READY : CompletionPollState::PENDING, + PTO2_ERROR_NONE + }; } -inline void retire_sdma_event_record(uint64_t record_addr) { - if (record_addr == 0) return; - volatile SdmaEventRecord *record = - reinterpret_cast(static_cast(record_addr)); - cache_invalidate_range(reinterpret_cast(sdma_completion_cache_line(record)), PTO2_ALIGN_SIZE); - uint32_t completed_tail = __atomic_load_n(&record->sq_tail, __ATOMIC_ACQUIRE); - uint64_t channel_info_addr = __atomic_load_n(&record->channel_info, __ATOMIC_ACQUIRE); - - volatile uint64_t *record_head = reinterpret_cast(record); - __atomic_store_n(record_head, 0ULL, __ATOMIC_RELEASE); - cache_flush_range(const_cast(reinterpret_cast(record_head)), sizeof(uint64_t)); - - if (channel_info_addr == 0) return; - uint64_t packed = (static_cast(completed_tail) << 32) | static_cast(completed_tail); - volatile uint64_t *channel_info = reinterpret_cast(static_cast(channel_info_addr)); - __atomic_store_n(channel_info, packed, __ATOMIC_RELEASE); - cache_flush_range(const_cast(reinterpret_cast(channel_info)), sizeof(uint64_t)); -} +inline void retire_sdma_post_done_record(uint64_t /*record_addr*/, uint64_t /*expected_post_id*/) {} diff --git a/src/a2a3/runtime/host_build_graph/runtime/pto_async_kernel_api.h b/src/a2a3/runtime/host_build_graph/runtime/pto_async_kernel_api.h index cf6eb47901..f350d7c3e3 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/pto_async_kernel_api.h +++ b/src/a2a3/runtime/host_build_graph/runtime/pto_async_kernel_api.h @@ -130,6 +130,7 @@ inline __aicore__ bool register_completion_condition(AsyncCtx &ctx, const Comple volatile __gm__ DeferredCompletionEntry *slot = &ctx.completion_entries[idx]; slot->addr = token.addr; + slot->backend_cookie = token.backend_cookie; slot->expected_value = token.expected_value; slot->engine = token.engine; slot->completion_type = token.completion_type; diff --git a/src/a2a3/runtime/host_build_graph/runtime/pto_async_wait.h b/src/a2a3/runtime/host_build_graph/runtime/pto_async_wait.h index 4a7cf9ffcf..ac2d54619c 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/pto_async_wait.h +++ b/src/a2a3/runtime/host_build_graph/runtime/pto_async_wait.h @@ -54,6 +54,7 @@ struct CompletionCondition { bool retired{false}; volatile uint32_t *counter_addr{nullptr}; uint64_t addr{0}; + uint64_t backend_cookie{0}; uint32_t expected_value{0}; CompletionPollResult test() const; @@ -76,10 +77,12 @@ inline CompletionPollResult counter_poll_op(const CompletionCondition &cond) { inline void counter_retire_op(CompletionCondition & /*cond*/) {} inline CompletionPollResult sdma_event_record_poll_op(const CompletionCondition &cond) { - return poll_sdma_event_record(cond.addr); + return poll_sdma_post_done_record(cond.addr, cond.backend_cookie); } -inline void sdma_event_record_retire_op(CompletionCondition &cond) { retire_sdma_event_record(cond.addr); } +inline void sdma_event_record_retire_op(CompletionCondition &cond) { + retire_sdma_post_done_record(cond.addr, cond.backend_cookie); +} inline const CompletionBackendOps *completion_backend_ops_for(int completion_type) { static const CompletionBackendOps kOps[] = { @@ -216,8 +219,8 @@ struct AsyncWaitList { entry->normal_done = false; } if (!append_condition_locked( - *entry, msg.addr, msg.expected_value, static_cast(msg.engine), msg.completion_type, - error_code + *entry, msg.addr, msg.backend_cookie, msg.expected_value, static_cast(msg.engine), + msg.completion_type, error_code )) { return drained; } @@ -261,8 +264,8 @@ struct AsyncWaitList { } bool append_condition_locked( - AsyncWaitEntry &entry, uint64_t addr, uint32_t expected_value, AsyncEngine engine, int32_t completion_type, - int32_t &error_code + AsyncWaitEntry &entry, uint64_t addr, uint64_t backend_cookie, uint32_t expected_value, AsyncEngine engine, + int32_t completion_type, int32_t &error_code ) { if (entry.condition_count >= MAX_COMPLETIONS_PER_TASK) { error_code = PTO2_ERROR_ASYNC_REGISTRATION_FAILED; @@ -274,6 +277,7 @@ struct AsyncWaitList { cond.satisfied = false; cond.retired = false; cond.addr = addr; + cond.backend_cookie = backend_cookie; cond.counter_addr = completion_type == COMPLETION_TYPE_COUNTER ? reinterpret_cast(static_cast(addr)) : nullptr; diff --git a/src/a2a3/runtime/host_build_graph/runtime/pto_completion_token.h b/src/a2a3/runtime/host_build_graph/runtime/pto_completion_token.h index c5a8c345f3..3c165b3879 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/pto_completion_token.h +++ b/src/a2a3/runtime/host_build_graph/runtime/pto_completion_token.h @@ -28,6 +28,7 @@ struct CompletionToken { uint32_t expected_value; uint32_t engine; int32_t completion_type; + // Backend-specific 64-bit metadata (for example, an SDMA post ID). uint64_t backend_cookie; }; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index fd44a7c529..2c9a7b6173 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -136,7 +136,9 @@ void SchedulerContext::complete_slot_task( const PTO2TaskId token = slot_state.task->task_id; for (uint32_t i = 0; i < cond_count; ++i) { volatile DeferredCompletionEntry *e = &deferred_slab->entries[i]; - while (!mailbox->try_push_condition(token, e->addr, e->expected_value, e->engine, e->completion_type)) { + while (!mailbox->try_push_condition( + token, e->addr, e->backend_cookie, e->expected_value, e->engine, e->completion_type + )) { sched_->async_wait_list.mpsc_skipped_count.fetch_add(1, std::memory_order_relaxed); SPIN_WAIT_HINT(); } From 9d774b44702f5ae48fbc253246eb4548dc37d6c2 Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Sun, 9 Aug 2026 23:26:36 -0700 Subject: [PATCH 05/11] fix(pr): resolve issues for #1644 - retry CI after transient PTO-ISA fetch failure From e26c3d7bd281d3e37991a03ebc2610e92fe3bf62 Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Mon, 10 Aug 2026 01:02:48 -0700 Subject: [PATCH 06/11] ci: retry PTO-ISA fetch From 5b8e08b20513ca66733080d4bd48b657bdd54e5d Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Mon, 10 Aug 2026 02:56:35 -0700 Subject: [PATCH 07/11] fix(pr): resolve issues for #1644 - retry the complete GitHub PTO-ISA acquisition three times - fall back to the GitCode mirror while preserving the exact pin --- simpler_setup/pto_isa.py | 129 +++++++++++++++++++----------------- tests/ut/py/test_pto_isa.py | 53 +++++++++++++++ 2 files changed, 123 insertions(+), 59 deletions(-) diff --git a/simpler_setup/pto_isa.py b/simpler_setup/pto_isa.py index bc1c53c0a5..1e877a074d 100644 --- a/simpler_setup/pto_isa.py +++ b/simpler_setup/pto_isa.py @@ -16,7 +16,8 @@ as-is — it already *is* the pinned ISA, so no checkout and no network. 3. Otherwise (missing, wrong revision, or dirty) obtain the pin fresh: clone over HTTPS (``--no-checkout`` so the default branch is never materialized) - and force-check-out the pin. + and force-check-out the pin. GitHub is attempted three times; if it cannot + provide the complete pinned checkout, fall back to the GitCode mirror. 4. Verify HEAD exactly matches the pin before returning. Two deliberate choices: @@ -51,14 +52,15 @@ from .environment import PROJECT_ROOT # A fresh clone is only needed when no usable local checkout exists; when it is, -# guard the single network hop against transient failures (e.g. GitHub -# SSL_ERROR_SYSCALL) with a few backed-off retries before giving up. +# guard the complete GitHub acquisition (clone plus landing on the pin) against +# transient failures before falling back to the GitCode mirror. _CLONE_ATTEMPTS = 3 _CLONE_RETRY_BACKOFF_S = 2 logger = logging.getLogger(__name__) -_PTO_ISA_HTTPS = "https://github.com/hw-native-sys/pto-isa.git" +_PTO_ISA_GITHUB_HTTPS = "https://github.com/hw-native-sys/pto-isa.git" +_PTO_ISA_GITCODE_HTTPS = "https://gitcode.com/luohuan40/pto-isa.git" _PTO_ISA_PIN_RE = re.compile(r"^[0-9a-fA-F]{40}$") PTO_ISA_PIN_FILE = "pto_isa.pin" PTO_ISA_BUILD_METADATA = "pto_isa_build.json" @@ -293,25 +295,12 @@ def _run_git_resilient( return result -def _discard_incomplete_clone(target: Path, verbose: bool) -> None: - """Remove `target` if it exists but is not a usable clone.""" - if not (target.exists() or target.is_symlink()) or _is_cloned(target): - return - if verbose: - logger.warning(f"Removing incomplete pto-isa clone at {target}") - if target.is_dir() and not target.is_symlink(): - shutil.rmtree(target, ignore_errors=True) - else: - target.unlink(missing_ok=True) - - def _remove_clone(target: Path, verbose: bool) -> None: - """Unconditionally remove `target` so a fresh clone can replace it. + """Unconditionally remove `target` so a fresh pinned clone can replace it. - Unlike ``_discard_incomplete_clone``, this removes even a *valid* clone — - used when the managed checkout is at the wrong (or a dirty) revision and we - re-clone at the pinned commit instead of checking out over local changes. - Handles a directory, a plain file, or a (possibly broken) symlink. + This removes even a *valid* clone when it is at the wrong (or a dirty) + revision, and also clears partial clones between source attempts. Handles a + directory, a plain file, or a (possibly broken) symlink. """ if not (target.exists() or target.is_symlink()): return @@ -394,12 +383,53 @@ def _is_pristine_at_commit(clone_path: Path, commit: str, verbose: bool) -> bool return result.returncode == 0 and not result.stdout.strip() +def _clone_from_remote(target: Path, commit: str, remote: str, attempts: int, verbose: bool) -> bool: + """Try complete clone-and-pin acquisitions from one remote.""" + for attempt in range(1, attempts + 1): + if attempt > 1: + time.sleep(_CLONE_RETRY_BACKOFF_S * (attempt - 1)) + _remove_clone(target, verbose) + logger.info( + f"Cloning pto-isa from {remote} to {target} at {commit} " + f"(attempt {attempt}/{attempts}, may take up to a minute)..." + ) + + failure = "" + try: + # --no-checkout: never materialize the default branch. Its working + # tree carries case-colliding docs/isa/TADDDEQRELU* paths; only lay + # down the pinned commit's tree in _land_on_commit. + result = _run_git(["clone", "--no-checkout", remote, str(target)], timeout=300) + if result.returncode != 0: + failure = result.stderr.strip() or f"git clone exited with status {result.returncode}" + elif _land_on_commit(target, commit, verbose=verbose): + if verbose: + logger.info(f"pto-isa cloned from {remote} at {commit}: {target}") + return True + else: + failure = f"clone succeeded but commit {commit} could not be checked out" + except subprocess.TimeoutExpired: + failure = "clone operation timed out" + except Exception as e: # noqa: BLE001 + failure = str(e) + + if verbose: + logger.warning(f"pto-isa acquisition from {remote} attempt {attempt}/{attempts} failed:\n{failure}") + # A successful clone that could not land on the pin is still unusable; + # remove it just like a partial clone before retrying or changing source. + _remove_clone(target, verbose) + + return False + + def _clone(target: Path, commit: str, verbose: bool) -> bool: """Fresh-clone PTO-ISA to `target` over HTTPS and land it on `commit`. Any existing checkout at `target` is removed first, so this always yields a clean tree at exactly `commit` — the sync can never be blocked by local - modifications in a preexisting (cached/preset) managed checkout. + modifications in a preexisting (cached/preset) managed checkout. GitHub is + tried three times for the complete clone-and-pin operation before GitCode is + used as a fallback source. """ if not _is_git_available(): if verbose: @@ -413,46 +443,25 @@ def _clone(target: Path, commit: str, verbose: bool) -> bool: logger.warning(f"Failed to create clone parent dir: {e}") return False - _remove_clone(target, verbose) - logger.info(f"Cloning pto-isa to {target} over HTTPS at {commit} (may take up to a minute)...") - try: - # --no-checkout: never materialize the default branch. Its working tree - # is what carries the case-colliding docs/isa/TADDDEQRELU* paths; we only - # ever want the pinned commit's tree, laid down by _land_on_commit. - result = _run_git(["clone", "--no-checkout", _PTO_ISA_HTTPS, str(target)], timeout=300) - for attempt in range(2, _CLONE_ATTEMPTS + 1): - if result.returncode == 0: - break - if verbose: - logger.warning(f"pto-isa clone attempt {attempt - 1}/{_CLONE_ATTEMPTS} failed:\n{result.stderr}") - _remove_clone(target, verbose) # clear the partial clone before retrying - time.sleep(_CLONE_RETRY_BACKOFF_S * (attempt - 1)) - result = _run_git(["clone", "--no-checkout", _PTO_ISA_HTTPS, str(target)], timeout=300) - if result.returncode != 0: - if verbose: - logger.warning(f"Failed to clone pto-isa after {_CLONE_ATTEMPTS} attempts:\n{result.stderr}") - _discard_incomplete_clone(target, verbose) - return False - if not _land_on_commit(target, commit, verbose=verbose): - # The clone succeeded but sits at the wrong commit (default HEAD), - # so it looks like a valid clone and `_discard_incomplete_clone` - # would leave it. Remove it outright so no wrong-revision checkout - # is stranded on disk for a later run to mistake for the pin. - _remove_clone(target, verbose) - return False - if verbose: - logger.info(f"pto-isa cloned at {commit}: {target}") - return True - except subprocess.TimeoutExpired: - if verbose: - logger.warning("Clone operation timed out") - _discard_incomplete_clone(target, verbose) - return False + if _clone_from_remote( + target, + commit, + _PTO_ISA_GITHUB_HTTPS, + attempts=_CLONE_ATTEMPTS, + verbose=verbose, + ): + return True + + logger.warning( + f"GitHub could not provide PTO-ISA commit {commit} after {_CLONE_ATTEMPTS} attempts; " + f"falling back to {_PTO_ISA_GITCODE_HTTPS}" + ) + return _clone_from_remote(target, commit, _PTO_ISA_GITCODE_HTTPS, attempts=1, verbose=verbose) except Exception as e: # noqa: BLE001 if verbose: logger.warning(f"Failed to clone pto-isa: {e}") - _discard_incomplete_clone(target, verbose) + _remove_clone(target, verbose) return False @@ -472,7 +481,9 @@ def ensure_pto_isa_root(verbose: bool = False) -> str: f"PTO-ISA not available.\n" f" The managed checkout must live at {clone_path} and match {PROJECT_ROOT / PTO_ISA_PIN_FILE}.\n" f" If auto-clone failed, manually run:\n" - f" git clone {_PTO_ISA_HTTPS} {clone_path}" + f" git clone {_PTO_ISA_GITHUB_HTTPS} {clone_path}\n" + f" Or use the fallback mirror:\n" + f" git clone {_PTO_ISA_GITCODE_HTTPS} {clone_path}" ) return resolved diff --git a/tests/ut/py/test_pto_isa.py b/tests/ut/py/test_pto_isa.py index 5e4cf67ee5..9c68a36c1b 100644 --- a/tests/ut/py/test_pto_isa.py +++ b/tests/ut/py/test_pto_isa.py @@ -69,6 +69,59 @@ def fake_run_git(args, cwd=None, timeout=30, check=False): ] +def test_clone_falls_back_to_gitcode_after_three_github_pin_failures(tmp_path, monkeypatch): + target = tmp_path / "build" / "pto-isa" + clone_remotes = [] + land_results = iter([False, False, False, True]) + sleeps = [] + + monkeypatch.setattr(pto_isa, "_is_git_available", lambda: True) + monkeypatch.setattr(pto_isa.time, "sleep", sleeps.append) + + def fake_run_git(args, cwd=None, timeout=30, check=False): + assert args[:2] == ["clone", "--no-checkout"] + clone_remotes.append(args[2]) + return subprocess.CompletedProcess(["git", *args], returncode=0, stdout="", stderr="") + + monkeypatch.setattr(pto_isa, "_run_git", fake_run_git) + monkeypatch.setattr(pto_isa, "_land_on_commit", lambda *args, **kwargs: next(land_results)) + + assert pto_isa._clone(target, PIN_A, verbose=False) + assert clone_remotes == [ + "https://github.com/hw-native-sys/pto-isa.git", + "https://github.com/hw-native-sys/pto-isa.git", + "https://github.com/hw-native-sys/pto-isa.git", + "https://gitcode.com/luohuan40/pto-isa.git", + ] + assert sleeps == [2, 4] + + +def test_clone_falls_back_to_gitcode_after_three_github_clone_failures(tmp_path, monkeypatch): + target = tmp_path / "build" / "pto-isa" + clone_remotes = [] + + monkeypatch.setattr(pto_isa, "_is_git_available", lambda: True) + monkeypatch.setattr(pto_isa.time, "sleep", lambda _seconds: None) + + def fake_run_git(args, cwd=None, timeout=30, check=False): + assert args[:2] == ["clone", "--no-checkout"] + remote = args[2] + clone_remotes.append(remote) + returncode = 1 if remote == "https://github.com/hw-native-sys/pto-isa.git" else 0 + return subprocess.CompletedProcess(["git", *args], returncode=returncode, stdout="", stderr="unavailable") + + monkeypatch.setattr(pto_isa, "_run_git", fake_run_git) + monkeypatch.setattr(pto_isa, "_land_on_commit", lambda *args, **kwargs: True) + + assert pto_isa._clone(target, PIN_A, verbose=False) + assert clone_remotes == [ + "https://github.com/hw-native-sys/pto-isa.git", + "https://github.com/hw-native-sys/pto-isa.git", + "https://github.com/hw-native-sys/pto-isa.git", + "https://gitcode.com/luohuan40/pto-isa.git", + ] + + def test_ensure_pto_isa_root_reuses_pristine_checkout(tmp_path, monkeypatch): clone_path = tmp_path / "build" / "pto-isa" (clone_path / "include").mkdir(parents=True) From dcc73c7a6a597763fb1e151f0ddd5fe33fe861be Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Mon, 10 Aug 2026 18:59:08 -0700 Subject: [PATCH 08/11] ci: retry artifact upload From 547f74fae435527b5d84ef564f64283ff9ca94b8 Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Mon, 10 Aug 2026 19:52:46 -0700 Subject: [PATCH 09/11] fix(pr): resolve issues for #1644 - increase the a2a3 unit-test job timeout from 30 to 45 minutes --- .github/workflows/_ut-npu-a2a3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_ut-npu-a2a3.yml b/.github/workflows/_ut-npu-a2a3.yml index 1968da6dbd..b9c9210dc2 100644 --- a/.github/workflows/_ut-npu-a2a3.yml +++ b/.github/workflows/_ut-npu-a2a3.yml @@ -24,7 +24,7 @@ jobs: run: name: ut-a2a3 runs-on: ${{ fromJSON(inputs.runs_on) }} - timeout-minutes: 30 + timeout-minutes: 45 env: SIMPLER_SCHEDULER_TIMEOUT_MS: "2000" SIMPLER_OP_EXECUTE_TIMEOUT_US: "3000000" From c1aaa35f2af407a4b2affa399981947e75e0ddfd Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Mon, 10 Aug 2026 23:22:28 -0700 Subject: [PATCH 10/11] fix(pr): resolve issues for #1644 - migrate A5 host_build_graph SDMA completion to post-done records - cover the A5 HBG completion path and align mailbox padding comments --- .../runtime/aicore_completion_mailbox.h | 18 ++++++-- .../runtime/aicore_completion_mailbox_types.h | 3 +- .../backend/sdma/sdma_completion_kernel.h | 31 ++++++------- .../backend/sdma/sdma_completion_scheduler.h | 46 ++++++------------- .../runtime/pto_async_kernel_api.h | 1 + .../host_build_graph/runtime/pto_async_wait.h | 16 ++++--- .../runtime/pto_completion_token.h | 1 + .../scheduler/scheduler_completion.cpp | 4 +- .../runtime/aicore_completion_mailbox.h | 2 +- tests/ut/cpp/CMakeLists.txt | 1 + .../cpp/a5/test_aicore_completion_mailbox.cpp | 4 +- 11 files changed, 64 insertions(+), 63 deletions(-) diff --git a/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox.h b/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox.h index 62bbaeda9c..aff860f9a0 100644 --- a/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox.h +++ b/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox.h @@ -39,7 +39,7 @@ static_assert( // carries the slot_state pointer in `addr` so the consumer can finalize the // AsyncWaitEntry.slot_state binding for tasks whose conditions arrived // before the FIN thread saw task_complete. New kinds may be added in future -// without growing the message — the `_pad[5]` slack is reserved for +// without growing the message — the `_pad[3]` slack is reserved for // kind-specific payload extension. #define MSG_KIND_CONDITION 0u #define MSG_KIND_TASK_NORMAL_DONE 1u @@ -51,15 +51,16 @@ struct AICoreCompletionMailboxMessage { // other fields below as a side effect, so they stay plain. std::atomic seq; PTO2TaskId task_token; - // CONDITION: completion observation addr (counter / SDMA event record). + // CONDITION: completion observation addr (counter / backend record). // TASK_NORMAL_DONE: PTO2TaskSlotState pointer carried over to the consumer // so it can finalize the AsyncWaitEntry.slot_state binding. uint64_t addr; + uint64_t backend_cookie; uint32_t expected_value; uint32_t engine; int32_t completion_type; uint32_t kind; - uint32_t _pad[5]; + uint32_t _pad[3]; }; static_assert(sizeof(AICoreCompletionMailboxMessage) == PTO2_ALIGN_SIZE, "AICoreCompletionMailboxMessage layout drift"); @@ -78,6 +79,7 @@ static_assert( struct AICoreCompletionMsgView { PTO2TaskId task_token{PTO2TaskId::invalid()}; uint64_t addr{0}; + uint64_t backend_cookie{0}; uint32_t expected_value{0}; uint32_t engine{0}; int32_t completion_type{0}; @@ -114,6 +116,13 @@ struct AICoreCompletionMailbox { // independent of the AsyncWaitList::busy lock. bool try_push_condition( PTO2TaskId task_token, uint64_t addr, uint32_t expected_value, uint32_t engine, int32_t completion_type + ) { + return try_push_condition(task_token, addr, 0, expected_value, engine, completion_type); + } + + bool try_push_condition( + PTO2TaskId task_token, uint64_t addr, uint64_t backend_cookie, uint32_t expected_value, uint32_t engine, + int32_t completion_type ) { while (true) { uint64_t h = head.load(std::memory_order_relaxed); @@ -124,6 +133,7 @@ struct AICoreCompletionMailbox { AICoreCompletionMailboxMessage *slot = &entries[h & AICORE_COMPLETION_MAILBOX_MASK]; slot->task_token.raw = task_token.raw; slot->addr = addr; + slot->backend_cookie = backend_cookie; slot->expected_value = expected_value; slot->engine = engine; slot->completion_type = completion_type; @@ -148,6 +158,7 @@ struct AICoreCompletionMailbox { AICoreCompletionMailboxMessage *slot = &entries[h & AICORE_COMPLETION_MAILBOX_MASK]; slot->task_token.raw = task_token.raw; slot->addr = slot_state_addr; + slot->backend_cookie = 0; slot->expected_value = 0; slot->engine = 0; slot->completion_type = 0; @@ -172,6 +183,7 @@ struct AICoreCompletionMailbox { if (slot->seq.load(std::memory_order_acquire) != t + 1) return false; out.task_token.raw = slot->task_token.raw; out.addr = slot->addr; + out.backend_cookie = slot->backend_cookie; out.expected_value = slot->expected_value; out.engine = slot->engine; out.completion_type = slot->completion_type; diff --git a/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h b/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h index d2f1f2c2e3..ac4a59c784 100644 --- a/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h +++ b/src/a5/runtime/host_build_graph/runtime/aicore_completion_mailbox_types.h @@ -44,13 +44,14 @@ inline constexpr int32_t MAX_COMPLETIONS_PER_TASK = 64; // pin the compiler against caching / reordering on either side. struct DeferredCompletionEntry { uint64_t addr; + uint64_t backend_cookie; uint32_t expected_value; uint32_t engine; int32_t completion_type; uint32_t _pad; }; -static_assert(sizeof(DeferredCompletionEntry) == 24, "DeferredCompletionEntry layout drift"); +static_assert(sizeof(DeferredCompletionEntry) == 32, "DeferredCompletionEntry layout drift"); struct alignas(PTO2_ALIGN_SIZE) DeferredCompletionSlab { volatile uint32_t count; diff --git a/src/a5/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_kernel.h b/src/a5/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_kernel.h index 8a24b48f2b..560d05b50c 100644 --- a/src/a5/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_kernel.h +++ b/src/a5/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_kernel.h @@ -28,7 +28,7 @@ #endif // Re-exposed PTO-ISA constant so examples / callers don't need to include -// just to spell their scratch tile. +// just to spell their scratch tile. inline constexpr uint32_t SDMA_SCRATCH_ALIGNMENT = pto::comm::sdma::UB_ALIGN_SIZE; enum class SdmaOp : uint8_t { @@ -42,11 +42,9 @@ enum class SdmaOp : uint8_t { // time; the SdmaTget() / SdmaTput() helpers below let callers skip the // template arguments. // -// sync_id selects which event-record slot inside the workspace the engine -// writes into. Concurrent dispatches must use distinct sync_ids; today every -// caller submits one request per kernel invocation so passing 0 is safe. -// Future work -// will fold sync_id allocation into the adapter. +// sync_id is forwarded to PTO-ISA's session builder for its internal pipe +// synchronization. Today every caller submits one request per kernel +// invocation, so passing 0 is safe. template struct SdmaRequestDescriptor { SdmaOp op; @@ -77,9 +75,10 @@ inline __aicore__ SdmaRequestDescriptor Sdma namespace pto2::detail { -inline __aicore__ void register_sdma_event_record(AsyncCtx &ctx, volatile __gm__ void *record_addr) { +inline __aicore__ void +register_sdma_post_done_record(AsyncCtx &ctx, volatile __gm__ void *record_addr, uint64_t post_id) { CompletionToken token{ - reinterpret_cast(record_addr), 0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD, 0 + reinterpret_cast(record_addr), 0, COMPLETION_ENGINE_SDMA, COMPLETION_TYPE_SDMA_EVENT_RECORD, post_id }; (void)register_completion_condition(ctx, token); } @@ -101,25 +100,25 @@ register_pto_async_event(AsyncCtx &ctx, const PtoAsyncEvent &event, const PtoAsy return; } - ::pto::comm::sdma::detail::UbTmpBuf tmp_buf; - uint32_t sync_id = 0; - __gm__ uint8_t *recv_workspace = nullptr; + uint64_t post_id = 0; uint32_t queue_num = 0; - if (!::pto::comm::sdma::detail::PrepareEventCheck( - session.sdmaSession, tmp_buf, sync_id, recv_workspace, queue_num - )) { + __gm__ uint8_t *post_done_base = session.sdmaRuntimeCtx.postDoneBase; + if (!::pto::comm::sdma::detail::DecodeSdmaEventHandle(event.handle, post_id, queue_num) || + post_done_base == nullptr) { defer_error(ctx, PTO2_ERROR_ASYNC_COMPLETION_INVALID); return; } for (uint32_t queue_id = 0; queue_id < queue_num; ++queue_id) { - register_sdma_event_record(ctx, ::pto::comm::sdma::detail::GetEventRecord(recv_workspace, queue_id)); + register_sdma_post_done_record( + ctx, ::pto::comm::sdma::detail::GetPostDoneRecordAddr(post_done_base, queue_id), post_id + ); } } } // namespace pto2::detail // SDMA overload of the runtime's send_request_entry. Submits the descriptor -// to PTO-ISA, then registers the resulting AsyncEvent's GM flag(s) into the +// to PTO-ISA, then registers the resulting AsyncEvent's GM post-done records into the // AsyncCtx deferred-wait slab and flushes. Returns false on submit/session // failure (also records the error in ctx.completion_error_code). template diff --git a/src/a5/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h b/src/a5/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h index 5a31754a3d..8bfbb9f9c6 100644 --- a/src/a5/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h +++ b/src/a5/runtime/host_build_graph/runtime/backend/sdma/sdma_completion_scheduler.h @@ -19,40 +19,20 @@ #include "pto_completion_token.h" #include "pto_runtime_status.h" -// runtime-side mirror of the PTO-ISA SdmaEventRecord. SDMA backend is the only -// allowed holder of this ABI knowledge; the generic scheduler dispatches into -// the helpers below through the completion ops table. -struct SdmaEventRecord { - uint32_t flag; - uint32_t sq_tail; - uint64_t channel_info; -}; - -static_assert(sizeof(SdmaEventRecord) == 16, "SDMA event record ABI drift"); -static_assert(offsetof(SdmaEventRecord, sq_tail) == 4, "SDMA event record ABI drift"); - -inline CompletionPollResult poll_sdma_event_record(uint64_t record_addr) { - if (record_addr == 0) { +// PTO-ISA stores the latest completed post ID as a monotonic uint64_t in one +// cache-line-sized record per queue. The completion type keeps its historical +// name, but the scheduler must compare the record with the post ID carried in +// backend_cookie and must not clear or otherwise retire the shared record. +inline CompletionPollResult poll_sdma_post_done_record(uint64_t record_addr, uint64_t expected_post_id) { + if (record_addr == 0 || expected_post_id == 0) { return {CompletionPollState::FAILED, PTO2_ERROR_ASYNC_COMPLETION_INVALID}; } - volatile SdmaEventRecord *record = - reinterpret_cast(static_cast(record_addr)); - uint32_t flag = __atomic_load_n(&record->flag, __ATOMIC_ACQUIRE); - return {flag != 0 ? CompletionPollState::READY : CompletionPollState::PENDING, PTO2_ERROR_NONE}; + volatile uint64_t *record = reinterpret_cast(static_cast(record_addr)); + uint64_t completed_post_id = __atomic_load_n(record, __ATOMIC_ACQUIRE); + return { + completed_post_id >= expected_post_id ? CompletionPollState::READY : CompletionPollState::PENDING, + PTO2_ERROR_NONE + }; } -inline void retire_sdma_event_record(uint64_t record_addr) { - if (record_addr == 0) return; - volatile SdmaEventRecord *record = - reinterpret_cast(static_cast(record_addr)); - uint32_t completed_tail = __atomic_load_n(&record->sq_tail, __ATOMIC_ACQUIRE); - uint64_t channel_info_addr = __atomic_load_n(&record->channel_info, __ATOMIC_ACQUIRE); - - volatile uint64_t *record_head = reinterpret_cast(record); - __atomic_store_n(record_head, 0ULL, __ATOMIC_RELEASE); - - if (channel_info_addr == 0) return; - uint64_t packed = (static_cast(completed_tail) << 32) | static_cast(completed_tail); - volatile uint64_t *channel_info = reinterpret_cast(static_cast(channel_info_addr)); - __atomic_store_n(channel_info, packed, __ATOMIC_RELEASE); -} +inline void retire_sdma_post_done_record(uint64_t /*record_addr*/, uint64_t /*expected_post_id*/) {} diff --git a/src/a5/runtime/host_build_graph/runtime/pto_async_kernel_api.h b/src/a5/runtime/host_build_graph/runtime/pto_async_kernel_api.h index cf6eb47901..f350d7c3e3 100644 --- a/src/a5/runtime/host_build_graph/runtime/pto_async_kernel_api.h +++ b/src/a5/runtime/host_build_graph/runtime/pto_async_kernel_api.h @@ -130,6 +130,7 @@ inline __aicore__ bool register_completion_condition(AsyncCtx &ctx, const Comple volatile __gm__ DeferredCompletionEntry *slot = &ctx.completion_entries[idx]; slot->addr = token.addr; + slot->backend_cookie = token.backend_cookie; slot->expected_value = token.expected_value; slot->engine = token.engine; slot->completion_type = token.completion_type; diff --git a/src/a5/runtime/host_build_graph/runtime/pto_async_wait.h b/src/a5/runtime/host_build_graph/runtime/pto_async_wait.h index 4a7cf9ffcf..ac2d54619c 100644 --- a/src/a5/runtime/host_build_graph/runtime/pto_async_wait.h +++ b/src/a5/runtime/host_build_graph/runtime/pto_async_wait.h @@ -54,6 +54,7 @@ struct CompletionCondition { bool retired{false}; volatile uint32_t *counter_addr{nullptr}; uint64_t addr{0}; + uint64_t backend_cookie{0}; uint32_t expected_value{0}; CompletionPollResult test() const; @@ -76,10 +77,12 @@ inline CompletionPollResult counter_poll_op(const CompletionCondition &cond) { inline void counter_retire_op(CompletionCondition & /*cond*/) {} inline CompletionPollResult sdma_event_record_poll_op(const CompletionCondition &cond) { - return poll_sdma_event_record(cond.addr); + return poll_sdma_post_done_record(cond.addr, cond.backend_cookie); } -inline void sdma_event_record_retire_op(CompletionCondition &cond) { retire_sdma_event_record(cond.addr); } +inline void sdma_event_record_retire_op(CompletionCondition &cond) { + retire_sdma_post_done_record(cond.addr, cond.backend_cookie); +} inline const CompletionBackendOps *completion_backend_ops_for(int completion_type) { static const CompletionBackendOps kOps[] = { @@ -216,8 +219,8 @@ struct AsyncWaitList { entry->normal_done = false; } if (!append_condition_locked( - *entry, msg.addr, msg.expected_value, static_cast(msg.engine), msg.completion_type, - error_code + *entry, msg.addr, msg.backend_cookie, msg.expected_value, static_cast(msg.engine), + msg.completion_type, error_code )) { return drained; } @@ -261,8 +264,8 @@ struct AsyncWaitList { } bool append_condition_locked( - AsyncWaitEntry &entry, uint64_t addr, uint32_t expected_value, AsyncEngine engine, int32_t completion_type, - int32_t &error_code + AsyncWaitEntry &entry, uint64_t addr, uint64_t backend_cookie, uint32_t expected_value, AsyncEngine engine, + int32_t completion_type, int32_t &error_code ) { if (entry.condition_count >= MAX_COMPLETIONS_PER_TASK) { error_code = PTO2_ERROR_ASYNC_REGISTRATION_FAILED; @@ -274,6 +277,7 @@ struct AsyncWaitList { cond.satisfied = false; cond.retired = false; cond.addr = addr; + cond.backend_cookie = backend_cookie; cond.counter_addr = completion_type == COMPLETION_TYPE_COUNTER ? reinterpret_cast(static_cast(addr)) : nullptr; diff --git a/src/a5/runtime/host_build_graph/runtime/pto_completion_token.h b/src/a5/runtime/host_build_graph/runtime/pto_completion_token.h index e460abf433..a60627864e 100644 --- a/src/a5/runtime/host_build_graph/runtime/pto_completion_token.h +++ b/src/a5/runtime/host_build_graph/runtime/pto_completion_token.h @@ -27,6 +27,7 @@ struct CompletionToken { uint32_t expected_value; uint32_t engine; int32_t completion_type; + // Backend-specific 64-bit metadata (for example, an SDMA post ID). uint64_t backend_cookie; }; diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index 34e2cba796..3381e3fd63 100644 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -136,7 +136,9 @@ void SchedulerContext::complete_slot_task( const PTO2TaskId token = slot_state.task->task_id; for (uint32_t i = 0; i < cond_count; ++i) { volatile DeferredCompletionEntry *e = &deferred_slab->entries[i]; - while (!mailbox->try_push_condition(token, e->addr, e->expected_value, e->engine, e->completion_type)) { + while (!mailbox->try_push_condition( + token, e->addr, e->backend_cookie, e->expected_value, e->engine, e->completion_type + )) { sched_->async_wait_list.mpsc_skipped_count.fetch_add(1, std::memory_order_relaxed); SPIN_WAIT_HINT(); } diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h index 2f12be49ab..8f88b88d39 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/aicore_completion_mailbox.h @@ -40,7 +40,7 @@ static_assert( // carries the slot_state pointer in `addr` so the consumer can finalize the // AsyncWaitEntry.slot_state binding for tasks whose conditions arrived // before the FIN thread saw task_complete. New kinds may be added in future -// without growing the message — the `_pad[5]` slack is reserved for +// without growing the message — the `_pad[3]` slack is reserved for // kind-specific payload extension. #define MSG_KIND_CONDITION 0u #define MSG_KIND_TASK_NORMAL_DONE 1u diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index f0e7d5772f..135841c67d 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -784,6 +784,7 @@ target_sources(test_hbg_submit_poison PRIVATE ${CMAKE_SOURCE_DIR}/../../../src/common/platform/shared/aicpu/args_dump_aicpu.cpp ) add_a5_hbg_runtime_test(test_a5_hbg_core_tracker common/test_hbg_core_tracker.cpp) +add_a5_hbg_runtime_test(test_a5_hbg_aicore_completion_mailbox a5/test_aicore_completion_mailbox.cpp) add_a2a3_hbg_runtime_test(test_graph_cache a2a3/test_graph_cache.cpp) target_sources(test_graph_cache PRIVATE ${CMAKE_SOURCE_DIR}/../../../src/a2a3/runtime/host_build_graph/runtime/scheduler/graph_execution.cpp diff --git a/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp b/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp index 3c0a9a62db..284afbaff8 100644 --- a/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp +++ b/tests/ut/cpp/a5/test_aicore_completion_mailbox.cpp @@ -11,8 +11,8 @@ /** * Unit tests for the a5 AICoreCompletionMailbox MPSC push protocol. * - * Mirrors tests/ut/cpp/a2a3/test_aicore_completion_mailbox.cpp but adapted - * to a5's simpler CompletionCondition (no completion_type field; counter-only). + * This source is compiled against both a5 runtime variants so their mirrored + * deferred-completion transports cannot drift apart. */ #include From 65695db8d61763f30c97c27ceedf1afbb22d49b9 Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Mon, 10 Aug 2026 23:43:42 -0700 Subject: [PATCH 11/11] ci: retry pod log upload