From 756204c51f8ce5404121c9c3c896098e9304e9a9 Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Mon, 8 Jun 2026 14:42:14 +0800 Subject: [PATCH] Fix: contain a5 AICore op-timeout cascade in shared L2 worker An AICore op-timeout (STARS-reaped op, 507000/507046 at AICPU stream sync) poisons the a5 device context for the life of the process. The shared L2 st_worker pool hands one ChipWorker to every test class on a device, so that one error then fails every later test on the xdist worker (halResMap rc=62 at init_aicore_register_addresses, or rtMalloc 507899). The poison is unrecoverable in-process: aclrtSynchronizeDevice returns the same error and close()+reset+re-init fails at rtStreamCreate 507899; only a fresh process recovers, and aclrtResetDeviceForce is unsafe on shared silicon. Two independent containment guards: - Driver fail-fast (src/a5/platform/onboard/host/device_runner.{h,cpp}): on a launch_aicore_kernel or sync_run_streams error, best-effort drain the device via the bounded aclrtSynchronizeDeviceWithTimeout; if it also errors, mark the DeviceRunner unusable so run() fails fast with an actionable message instead of cascading into halResMap / 507899. - Fixture rebuild-then-skip (conftest.py): a pytest_runtest_makereport hook stashes the call-phase exception; the st_worker L2 path drops the pooled Worker on a device-runtime error, rebuilds where the arch allows it, and otherwise marks the runtime poisoned and skips the rest with a clear reason. One failure stays one failure (1 failed, N skipped) rather than an N-test 507899 storm. Verified on a5 (Ascend950PR): the cascade reproduces 10/10 in-process; after the fix a pooled hang->noop sequence is contained 10/10 (noop skipped, 0 cascades), with no regression on a normal L2 test. a2a3 is untouched (shared device_runner_base.cpp not modified). Investigation write-up in docs/investigations/. --- conftest.py | 152 ++++++++++++++- .../2026-06-a5-aicore-op-timeout-cascade.md | 175 ++++++++++++++++++ docs/investigations/README.md | 1 + .../platform/onboard/host/device_runner.cpp | 62 ++++++- src/a5/platform/onboard/host/device_runner.h | 19 ++ 5 files changed, 405 insertions(+), 4 deletions(-) create mode 100644 docs/investigations/2026-06-a5-aicore-op-timeout-cascade.md diff --git a/conftest.py b/conftest.py index 54811c2235..08057b2abd 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,7 @@ import faulthandler import logging import os +import re import signal import subprocess import sys @@ -959,6 +960,105 @@ def pytest_sessionfinish(session, exitstatus): # noqa: ARG001 clear_compile_cache() +# --------------------------------------------------------------------------- +# L2 worker-pool self-healing +# +# The L2 ``st_worker`` pool hands ONE ``ChipWorker`` to every SceneTestCase on +# a (runtime, device) for the life of an xdist worker process. That amortizes +# init cost, but it also means a single device-runtime error (an AICore +# op-timeout reaped by STARS, or a kernel-launch failure) poisons the shared +# ACL/device context: every later test on that worker then fails at +# ``halResMap`` (rc=62) / ``rtMalloc`` (507899) — one failure cascades into the +# whole worker's remaining tests. The driver-side guard +# (``DeviceRunner::run`` fail-fast on ``device_unusable_``) keeps that cascade +# from wedging, but the Worker stays poisoned. +# +# On a5 the poison survives close()+device-reset for the life of the process +# (an in-process Worker.init rebuild fails with rtStreamCreate 507899; a +# force-reset is unsafe on this shared box), so there is no in-process +# recovery — only a fresh worker process gets a clean device. The hook below +# stashes the call-phase exception so the ``st_worker`` finalizer can, after +# — and ONLY after — a device-runtime error, drop the pooled Worker; the L2 +# branch then rebuilds where the arch allows it and otherwise skips the +# remaining tests for that runtime (``_l2_poisoned``). Golden mismatches / +# ordinary assertions leave the Worker reusable. +# --------------------------------------------------------------------------- + +# Plain attribute name on the test item — NOT pytest.StashKey()/item.stash, +# which are pytest>=7.0 only; the test extra pins `pytest>=6.0`, and StashKey +# at import time would AttributeError on pytest 6.x. +_ST_CALL_EXCINFO = "_st_call_excinfo" + + +# hookwrapper (not the 8.0+ `wrapper=True`) so this works across the whole +# declared pytest range (test extra pins only `pytest>=6.0`). We stash +# call.excinfo for the side effect and don't touch the result. +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + yield + if call.when == "call": + setattr(item, _ST_CALL_EXCINFO, call.excinfo) + + +# Only these error codes mean the device/ACL context is sticky-errored — i.e. +# the worker is poisoned and must be rebuilt/skipped. Matching on the bare +# " failed with code" prefix would also catch ordinary kernel/run failures +# (every non-zero rc is wrapped that way), turning a normal failing test into a +# spurious worker rebuild and, downstream, a misleading runtime-wide skip. So +# we extract the trailing and match only the known poison codes: +# 207001 AICore launch failure (the CI cascade trigger) +# 507000 ACL_ERROR_RT_INTERNAL_ERROR (a5 op-timeout at AICPU stream sync) +# 507018 ACL_ERROR_RT_AICPU_EXCEPTION +# 507046 ACL_ERROR_RT_STREAM_SYNC_TIMEOUT +# 507899 sticky-error rtMalloc / rtStreamCreate on the poisoned context +# -1 a5 DeviceRunner fail-fast sentinel ("marked unusable; refusing to run") +# Worker surfaces these as "run_prepared/prepare_callable/simpler_init failed +# with code " — never as an AssertionError, so golden mismatches are already +# excluded. +_DEVICE_POISON_CODES = frozenset({207001, 507000, 507018, 507046, 507899, -1}) +_DEVICE_ERROR_CODE_RE = re.compile(r"(?:run_prepared|prepare_callable|simpler_init) failed with code (-?\d+)") + + +def _is_device_runtime_error_msg(msg: str) -> bool: + match = _DEVICE_ERROR_CODE_RE.search(msg) + return match is not None and int(match.group(1)) in _DEVICE_POISON_CODES + + +def _is_device_runtime_error(excinfo) -> bool: + if excinfo is None or not issubclass(excinfo.type, RuntimeError): + return False + return _is_device_runtime_error_msg(str(excinfo.value)) + + +def _register_l2_pool_heal(request, pool, key): + """Drop + close the pooled L2 Worker iff this test raised a device-runtime + error, so the next test on this (runtime, device) gets a chance to rebuild a + fresh Worker / ACL context instead of inheriting a poisoned one. No-op on + pass or on a non-device failure (golden mismatch, assertion). + + NOTE: on a5, an AICore op-timeout poisons the device context for the life of + the *process*: an in-process rebuild's Worker.init fails (rtStreamCreate + 507899), and aclrtResetDeviceForce is unsafe on this shared box (it resets + the physical device for other users). So this heal can only *recover* on + arches where in-process re-init works; where it cannot, the st_worker L2 + branch falls back to skipping the remaining tests for that runtime (see + _l2_poisoned). Either way one device error stops cascading into a worker-wide + storm of confusing 507899 failures.""" + + def _heal(): + if not _is_device_runtime_error(getattr(request.node, _ST_CALL_EXCINFO, None)): + return + w = pool.pop(key, None) + if w is None: + return + try: + w.close() + except Exception: # noqa: BLE001 + pass + + request.addfinalizer(_heal) + + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -1004,8 +1104,19 @@ def _l2_worker_pool(request, st_platform): pool.clear() +@pytest.fixture(scope="session") +def _l2_poisoned(): + """Runtimes whose L2 device context was poisoned by a device-runtime error + and could not be re-initialized in this process. ``st_worker`` skips + remaining tests for a poisoned runtime instead of letting them fail with + confusing 507899 cascades. Lives for the xdist worker process; only a fresh + process (process exit fully releases the device) recovers — which is exactly + how the standalone phase + a new xdist worker get a clean device.""" + return set() + + @pytest.fixture() -def st_worker(request, st_platform, device_pool, _l2_worker_pool): +def st_worker(request, st_platform, device_pool, _l2_worker_pool, _l2_poisoned): """Per-test Worker. L2: session-scoped, reused across classes with the same (runtime, device). @@ -1019,6 +1130,18 @@ def st_worker(request, st_platform, device_pool, _l2_worker_pool): runtime = cls._st_runtime if level == 2: + # A prior test on this runtime poisoned the device and the rebuild below + # could not re-init it in-process (a5 op-timeout: rtStreamCreate 507899). + # Skip the rest for this runtime so one device error stops cascading — + # the triggering failure is already reported; a fresh worker process is + # the only real recovery. + if runtime in _l2_poisoned: + pytest.skip( + f"L2 device context for runtime '{runtime}' was poisoned by an earlier device-runtime error on " + f"this worker process and cannot be re-initialized in-process; skipping to avoid a misleading " + f"507899 cascade (a fresh worker process recovers)." + ) + # L2 share: reuse any Worker already created for this runtime in the # current process. Under xdist, each worker process is sliced to a # single device so there's at most one matching entry. On first call @@ -1029,6 +1152,7 @@ def st_worker(request, st_platform, device_pool, _l2_worker_pool): # same xdist worker. for (rt, dev_id), existing in _l2_worker_pool.items(): if rt == runtime: + _register_l2_pool_heal(request, _l2_worker_pool, (rt, dev_id)) yield existing return @@ -1042,10 +1166,32 @@ def st_worker(request, st_platform, device_pool, _l2_worker_pool): w = Worker(level=2, device_id=dev_id, platform=st_platform, runtime=runtime) w._st_device_id = dev_id - w.init() + # First rebuild after a poison-and-heal lands here. On arches where the + # device re-inits cleanly this just works; on a5 the op-timeout poison + # survives close()+reset, so Worker.init raises a device-runtime error — + # mark the runtime poisoned and skip (this test and every later one for + # the runtime) instead of surfacing a raw 507899 setup ERROR. + try: + w.init() + except RuntimeError as e: + if _is_device_runtime_error_msg(str(e)): + _l2_poisoned.add(runtime) + try: + w.close() + except Exception: # noqa: BLE001 + pass + pytest.skip( + f"L2 Worker.init for runtime '{runtime}' failed with a device-runtime error ({e}); the device " + f"context is not recoverable in-process after an earlier AICore error — skipping remaining " + f"'{runtime}' L2 tests (a fresh worker process recovers)." + ) + raise _l2_worker_pool[key] = w + _register_l2_pool_heal(request, _l2_worker_pool, key) yield w - # No close here — pool handles teardown at session end. + # No close here on success — the pool handles teardown at session end. + # On a device-runtime failure, the finalizer registered above closes + # this Worker and drops it from the pool so the next test rebuilds. elif level == 3: max_devices = max((c.get("config", {}).get("device_count", 1) for c in cls.CASES), default=1) diff --git a/docs/investigations/2026-06-a5-aicore-op-timeout-cascade.md b/docs/investigations/2026-06-a5-aicore-op-timeout-cascade.md new file mode 100644 index 0000000000..cf072e449e --- /dev/null +++ b/docs/investigations/2026-06-a5-aicore-op-timeout-cascade.md @@ -0,0 +1,175 @@ +# a5 AICore op-timeout poisons the shared L2 worker for the rest of an xdist session + +**Date**: 2026-06-08 +**Verdict**: contained (two independent guards landed — driver fail-fast + +fixture rebuild-then-skip). The poison itself is unrecoverable in-process on +a5; containment stops it from cascading. + +## Question + +CI run 27090242388, job `st-onboard-a5` (branch `swimlane-queue-depth-viz`, +PR #1000 — PR only touches a2a3 + tooling, so the failure is **not** PR +code). On the gw0 xdist worker: + +```text +10:47:04 standalone test_aicore_op_timeout ... PASS (separate process, dev=[5]) +10:48:17 TestPagedAttentionUnrollManualScope rtKernelLaunchWithHandleV2 failed: 207001 <- trigger +10:48:22..58 7 subsequent gw0 tests all fail rtMalloc failed: 507899 (ChipCallable buffer) +``` + +gw1 (other device) ran clean throughout. One device-side error took down +every remaining test on the worker. The hypothesis to check: an AICore +op-timeout (or a launch failure) leaves the ACL/device context in a +sticky-error state, and because the L2 `st_worker` pool reuses **one** +`ChipWorker` per (runtime, device) for the life of an xdist worker process +(`conftest.py::_l2_worker_pool`), the poison is never cleared — every later +test inherits it. + +## What was tried + +All runs on a dedicated, exclusively-locked a5 device +(`Ascend950PR_9599`) via `task-submit --device auto` (see +`.claude/rules/running-onboard.md`). Three experiments: + +1. **Cascade variant** — `aicore_op_timeout` (standalone subprocess) then + `paged_attention_unroll_manual_scope` (L2 subprocess), 10×, exactly the + CI ordering. +2. **Control** — `paged_attention_unroll_manual_scope` alone, 10×. +3. **Within-process probe** (`/tmp/cascade_probe/probe.py`) — one L2 + `Worker`: `run(HANG)` (forever-spinning AIC task → op-timeout) then + `run(NOOP)` (trivial AIC task) **on the same worker**, 10×. This mirrors + gw0's persistent process reusing one pooled `ChipWorker`. + +## Result + +| Experiment | Cascades / failures | +| ---------- | ------------------- | +| Cascade variant (separate processes) | **0 / 10** | +| Control (PA-unroll alone) | **0 / 10** | +| Within-process probe (hang → noop, same worker) | **10 / 10** | + +The cascade is **not** about the standalone→xdist ordering: with the device +held exclusively, `aicore_op_timeout` resets the device on its own +`worker.close()` / process exit, and the next process starts clean (0/10). + +The cascade **is** a within-process, same-worker effect, and it is +deterministic (10/10). The exact a5 trace: + +- STEP1 `run(HANG)`: `aclrtSynchronizeStreamWithTimeout (AICPU) failed: + 507000/507046` — STARS reaps the op, surfaced at AICPU stream sync. +- STEP2 `run(NOOP)` on the same worker: fails **early**, at + `init_aicore_register_addresses → get_aicore_reg_info → halResMap failed + for core 0 (rc=62)` (`device_runner.cpp:160`). a5 hits the poisoned + context at register mapping; a2a3/CI hit it slightly later at `rtMalloc` + (507899). Same root cause, different surfacing call. + +**Which side leaks:** the **driver/device context**. The op-timeout leaves +the device sticky-errored for the rest of the process. It does *not* survive +a process exit + device reset (cascade variant 0/10). The amplifier is the +`_l2_worker_pool` reuse: nothing rebuilds the `Worker` after a failed run, +so the poison spreads to every later test in the worker's process — exactly +the CI cascade. + +**Recovery is not possible in-process.** Two reset paths were measured: + +- `aclrtSynchronizeDeviceWithTimeout` (the bounded device drain in fix #1) + returns the *same* 507046 — it does not clear the sticky error. +- `Worker.close()` → `DeviceRunner::finalize` → `rtDeviceReset` followed by a + fresh `Worker.init()` on the same device **in the same process** fails at + `aclrtSetOpExecuteTimeOutV2 (507000)` / `rtStreamCreate (AICPU) 507899` + (`simpler_init failed with code 507899`). The op-timeout poison survives a + device reset within the process. + +The only reset that recovers is **process exit** — which is exactly why the +separate-process cascade variant is 0/10. `aclrtResetDeviceForce` *would* +reset the physical device, but this box is shared (see +`.claude/rules/running-onboard.md`) and force-resetting kills other users' +work on the same die, so it is not an option. Net: after an AICore +op-timeout an a5 worker process cannot get a usable device back; only a new +process can. + +## Fix + +Two independent guards, both landed: + +1. **Driver fail-fast (root-cause containment)** — + `src/a5/platform/onboard/host/device_runner.{h,cpp}`. On any + `launch_aicore_kernel` error **or** `sync_run_streams` error, + `recover_device_or_mark_unusable()` best-effort drains via the bounded + `aclrtSynchronizeDeviceWithTimeout`; if that also errors, it sets + `device_unusable_`. `run()` checks the flag on entry and fails fast with + an actionable message instead of cascading into the confusing + `halResMap rc=62` / `rtMalloc 507899`. Verified: after the fix STEP2 + fails immediately at the run() guard (`code -1`, "Rebuild the Worker"), + not at `halResMap`. + +2. **Fixture rebuild-then-skip (CI containment)** — `conftest.py`. A + `pytest_runtest_makereport` hook stashes the call-phase exception; the + `st_worker` L2 path heals **only** on a device-runtime `RuntimeError` + (`run_prepared failed …` / `prepare_callable failed …` / `DeviceRunner + marked unusable` / `simpler_init failed …`), never on golden mismatches. + The heal `close()`s the pooled `Worker` and drops it so the next test + **rebuilds**. Because the a5 rebuild's `Worker.init()` then fails (device + still poisoned — see above), the create path catches that device error, + marks the runtime poisoned (`_l2_poisoned`), and `pytest.skip`s — that + test and every later one for the runtime — with a clear reason. On an + arch where in-process re-init *does* work, the rebuild succeeds and tests + continue; on a5 they skip cleanly. Either way the worker-wide 507899 + failure storm is gone. + +Verification (pooled L2 cascade through the real `st_worker` fixture: a hang +class poisons, a noop class follows on the same pool key; plus a normal L2 +class for no-regression): + +| Phase | TestBNoopAfter outcome | +| ----- | ---------------------- | +| before (fix #2 off, HEAD conftest) | FAILED 3/3 (cascade) | +| after fix #2 = rebuild-then-skip | SKIPPED 10/10 — 0/10 cascades | +| normal L2 (PA-unroll) regression check | PASSED 2/2 | + +Each "after" run is "1 failed, 1 skipped": the one real failure +(TestAHangPoison) stands, every later test is a clean skip. + +The skip carries its reason, e.g.: *"L2 Worker.init for runtime +'tensormap_and_ringbuffer' failed with a device-runtime error (simpler_init +failed with code 507899); the device context is not recoverable in-process +after an earlier AICore error — skipping remaining 'tensormap_and_ringbuffer' +L2 tests (a fresh worker process recovers)."* + +## Why this shape + +The cascade cannot be *recovered* on a5 — the device is dead in-process and +force-reset is unsafe on shared hardware — so the goal is **containment**, +not recovery: + +- fix #1 (driver) turns the confusing `halResMap rc=62` / `rtMalloc 507899` + cascade into an immediate, self-explanatory "DeviceRunner marked unusable; + rebuild the Worker" — for *any* caller, not just pytest. +- fix #2 (fixture) turns the rest of an xdist worker's L2 tests from a storm + of red 507899 errors into **one real failure + clean skips** with a + reason. A fresh worker process (the next CI shard, or the standalone phase) + starts clean. + +Both are cheap, independent, and additive. + +## When to reconsider + +- If a future CANN release makes `aclrtSynchronizeDeviceWithTimeout` (or a + scoped reset API) actually clear an op-timeout sticky error in-place, + fix #1 could be upgraded from fail-fast to true in-place recovery, and + the worker would not need rebuilding at all. +- a2a3 shows the same `finalize_common` cascade note (507018/507899/507901) + but the trigger there has historically been blamed on runner contention; + the a2a3 side was intentionally **not** touched (could not reproduce the + cascade deterministically on a2a3 hardware here). If an a2a3 repro lands, + port the same two guards to `src/a2a3/`. + +## References + +- CI run 27090242388, job `st-onboard-a5` (`gh run view --job 79952403308`). +- Fix: `src/a5/platform/onboard/host/device_runner.{h,cpp}` + (`device_unusable_`, `recover_device_or_mark_unusable`), `conftest.py` + (`pytest_runtest_makereport`, `_register_l2_pool_heal`). +- `DeviceRunnerBase::finalize_common` — prior note on why finalize skips a + pre-destroy stream sync on the error-state stream. +- `.claude/rules/running-onboard.md` — `task-submit` device isolation. diff --git a/docs/investigations/README.md b/docs/investigations/README.md index 33de5db4b7..05ec92ff27 100644 --- a/docs/investigations/README.md +++ b/docs/investigations/README.md @@ -86,6 +86,7 @@ Newest first. - [2026-06 — Cross-task batched publish: hoist wmb across distinct tasks in one pop](2026-06-cross-task-batched-publish.md) - [2026-06 — AICore first-task cold-start: pre-warm dispatch path](2026-06-aicore-cold-start-warmup.md) +- [2026-06 — a5 AICore op-timeout poisons the shared L2 worker (cascade)](2026-06-a5-aicore-op-timeout-cascade.md) - [2026-06 — a5 AICPU filter gate: Scenario B fail-fast guard not added](2026-06-a5-aicpu-filter-gate-scenario-b-validation.md) - [2026-06 — Sanitizer rollout scope: macOS, TSAN gating, LSan](2026-06-sanitizer-scope.md) - [2026-06 — L2 swimlane: defer per-task wmb to rotation](2026-06-l2-swimlane-defer-wmb.md) diff --git a/src/a5/platform/onboard/host/device_runner.cpp b/src/a5/platform/onboard/host/device_runner.cpp index a275c4be20..f65bac1ba2 100644 --- a/src/a5/platform/onboard/host/device_runner.cpp +++ b/src/a5/platform/onboard/host/device_runner.cpp @@ -143,6 +143,22 @@ int DeviceRunner::destroy_comm_stream(void *stream) { } int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { + // A prior AICore launch/sync error poisoned the device context and the + // in-place drain could not clear it. Refuse to run rather than cascade + // into halResMap rc=62 (init_aicore_register_addresses) or rtMalloc + // 507899. On a5 the poison is unrecoverable in-process (even close()+reset + // + a fresh Worker.init fails with rtStreamCreate 507899); only a fresh + // process clears it. Failing fast here turns the rest of an xdist worker + // session's tests from a slow, confusing failure cascade into a single + // fast, self-explanatory error (the test fixture then skips the remainder). + if (device_unusable_) { + LOG_ERROR( + "DeviceRunner marked unusable by a prior AICore failure; refusing to run. " + "The device context is unrecoverable in-process on a5 (close + re-create does " + "not reset it); a fresh worker process is required to recover." + ); + return -1; + } if (validate_launch_aicpu_num(launch_aicpu_num) != 0) return -1; int rc = ensure_device_initialized(); @@ -352,11 +368,20 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { rc = launch_aicore_kernel(stream_aicore_, kernel_args_.device_k_args_); if (rc != 0) { LOG_ERROR("launch_aicore_kernel failed: %d", rc); + recover_device_or_mark_unusable(rc); return rc; } rc = sync_run_streams(); - if (rc != 0) return rc; + if (rc != 0) { + // sync_run_streams surfaces the AICore op-timeout (STARS-reaped op -> + // 507000/507018/507046 at AICPU/AICore stream sync). The op-timeout + // leaves the device context poisoned for the SAME DeviceRunner's next + // run, so attempt recovery / mark-unusable here too, not only on the + // launch-error path above. + recover_device_or_mark_unusable(rc); + return rc; + } read_device_wall_ns(); @@ -381,6 +406,38 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { return 0; } +void DeviceRunner::recover_device_or_mark_unusable(int aicore_rc) { + // An AICore launch failure (207001) or an op-timeout reaped by STARS + // (surfaced as 507000/507018/507046 at stream sync) leaves the device + // context in a sticky-error state: the streams stay poisoned and the + // SAME DeviceRunner's next run() fails early — observed on a5 as + // `halResMap failed (rc=62)` in init_aicore_register_addresses, and on + // a2a3 as `rtMalloc failed: 507899`. Reused across a session (the L2 + // st_worker pool hands one ChipWorker to every test class on a device), + // that one error poisons every later test in the xdist worker process. + // + // Try to drain the device so a subsequent run can recover in place. Use + // the BOUNDED aclrtSynchronizeDeviceWithTimeout, NOT an unbounded + // aclrtSynchronizeStream* on the error-state stream — the latter wedges + // subsequent tests (see DeviceRunnerBase::finalize_common's comment on + // why finalize deliberately skips a pre-destroy sync). If the bounded + // device drain itself errors, the context is unrecoverable without a full + // device reset, so mark the runner unusable: run() then fails fast with a + // clear message instead of cascading, and the fixture rebuilds the Worker. + int sync_rc = aclrtSynchronizeDeviceWithTimeout(PLATFORM_STREAM_SYNC_TIMEOUT_MS); + if (sync_rc != ACL_SUCCESS) { + LOG_ERROR( + "Device unrecoverable after AICore error %d: aclrtSynchronizeDeviceWithTimeout failed: %d. " + "Marking DeviceRunner unusable; run() will fail fast until a fresh worker process runs " + "(in-process reset does not clear the poison on a5).", + aicore_rc, sync_rc + ); + device_unusable_ = true; + } else { + LOG_WARN("AICore error %d: device drained via aclrtSynchronizeDeviceWithTimeout", aicore_rc); + } +} + // `print_handshake_results`, `prepare_orch_so`, `register_callable`, // `register_callable_host_orch`, `unregister_callable`, `has_callable`, // `bind_callable_to_runtime`, and `upload_chip_callable_buffer` live on @@ -450,6 +507,9 @@ int DeviceRunner::finalize() { } device_id_ = -1; + // A finalized runner has reset the device; clear the poison flag so a + // reused DeviceRunner object (re-init after close) starts clean. + device_unusable_ = false; LOG_INFO_V0("DeviceRunner finalized"); return rc; } diff --git a/src/a5/platform/onboard/host/device_runner.h b/src/a5/platform/onboard/host/device_runner.h index a3c1c8b54b..7f0ca87943 100644 --- a/src/a5/platform/onboard/host/device_runner.h +++ b/src/a5/platform/onboard/host/device_runner.h @@ -200,6 +200,25 @@ class DeviceRunner : public DeviceRunnerBase { // acl_ready_, so runtimes that never ask for ACL (e.g. pure rt-layer) stay unaffected. bool acl_ready_{false}; + // Set true when an AICore launch/sync error (e.g. an op-timeout reaped by + // STARS, surfaced as 507000/507018 at stream sync, or a 207001 launch + // failure) left the device context in a sticky-error state that an + // in-place drain could not clear. Once set, run() fails fast instead of + // cascading into the confusing downstream failures (halResMap rc=62 at + // init_aicore_register_addresses, or rtMalloc 507899) that a poisoned + // context produces. On a5 the poison survives close()+device-reset for the + // life of the process (an in-process re-init fails with rtStreamCreate + // 507899) and a force-reset is unsafe on shared silicon, so the only real + // recovery is a fresh process — this flag just contains the blast radius. + // See run() and recover_device_or_mark_unusable(). + bool device_unusable_{false}; + + // On an AICore launch/sync error, best-effort drain the device so a later + // run() on the same DeviceRunner can recover in place; if the drain itself + // errors the context is unrecoverable without a full reset, so flip + // device_unusable_ and let run() fail fast. + void recover_device_or_mark_unusable(int aicore_rc); + /** * Initialize performance profiling device buffers *