From 1cdaed899dfa2574bf6b26e432a9419789d309fd Mon Sep 17 00:00:00 2001 From: wcwxy <26245345+ChaoWao@users.noreply.github.com> Date: Fri, 29 May 2026 14:50:34 +0800 Subject: [PATCH] Fix: per-device staged-SO naming to stop paired-die AICPU device fault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit st-onboard-a2a3 fails intermittently with a whole-suite collapse (507899/507018/prepare_callable -1). It is NOT OOM (npu-smi shows HBM ~3G/64G free): the earliest error is an AICPU exception in simpler_aicpu_exec (507018, errorCode 0x2a) that faults the whole chip, after which every rtStreamCreate/rtMalloc on it returns 507899 [driver error:internal error] / 507901 [hdc disconnect]. Root cause: on a2a3 the two dies of one chip (npu-smi Phy-IDs 8/9, 4/5, ...) share the preinstall filesystem, but the runtime staged SOs there under names that are identical across dies. Concurrent bootstrap/staging on the same file corrupts the mmap'd image and traps the AICPU kernel on both dies at once. A single-die 50x solo loop never reproduces; only the parallel multi-die suite does — a cross-die shared-file race. Fix — make every SO staged under /usr/lib64/aicpu_kernels/0/ per-device: - Dispatcher inner SO: simpler_inner__.so. The real device_id (was hardcoded 0) is threaded from DeviceRunner through BootstrapDispatcher into the host JSON reader name (MakeInnerSoBasename) and the device-side writer (DeviceArgs.device_id -> MakeInnerSoPath). Bootstrap cache keyed by (fp, device_id). - AICPU executor orch SO: libdevice_orch___.so. device_id reaches the AICPU via a new trailing KernelArgs.device_id field, pushed in kernel.cpp through set_orch_device_id() and read by the executor via get_orch_device_id() (platform_regs). - Update tests/ut/cpp test_orch_so_file for the new signature + add a distinct-device-id case. Applied symmetrically to a2a3 and a5 (onboard + sim). Validated with 5/5 consecutive st-onboard-a2a3 passes; investigation written up in docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/ci.md | 1 + .../a2a3-507899-aicpu-shared-so-fault.md | 111 ++++++++++++++++++ .../platform/include/aicpu/orch_so_file.h | 6 +- .../platform/include/aicpu/platform_regs.h | 11 ++ .../platform/include/common/kernel_args.h | 7 ++ .../platform/onboard/aicpu/CMakeLists.txt | 4 +- src/a2a3/platform/onboard/aicpu/kernel.cpp | 2 + .../platform/onboard/aicpu/orch_so_file.cpp | 25 ++-- .../platform/onboard/host/device_runner.cpp | 6 +- src/a2a3/platform/sim/aicpu/orch_so_file.cpp | 13 +- src/a2a3/platform/src/aicpu/platform_regs.cpp | 5 + .../aicpu/aicpu_executor.cpp | 4 +- src/a5/platform/include/aicpu/orch_so_file.h | 6 +- src/a5/platform/include/aicpu/platform_regs.h | 11 ++ src/a5/platform/include/common/kernel_args.h | 5 + src/a5/platform/onboard/aicpu/kernel.cpp | 2 + .../platform/onboard/aicpu/orch_so_file.cpp | 25 ++-- .../platform/onboard/host/device_runner.cpp | 6 +- src/a5/platform/sim/aicpu/orch_so_file.cpp | 13 +- src/a5/platform/src/aicpu/platform_regs.cpp | 5 + .../aicpu/aicpu_executor.cpp | 4 +- src/common/aicpu_dispatcher/README.md | 9 +- .../aicpu_dispatcher/aicpu_dispatcher.cpp | 8 +- .../aicpu_dispatcher/aicpu_dispatcher.h | 15 ++- src/common/host/load_aicpu_op.cpp | 43 ++++--- src/common/host/load_aicpu_op.h | 9 +- tests/ut/cpp/common/test_orch_so_file.cpp | 48 ++++++-- 27 files changed, 324 insertions(+), 80 deletions(-) create mode 100644 docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md diff --git a/docs/ci.md b/docs/ci.md index c39d2b95f9..6e6b6ae4f1 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -224,3 +224,4 @@ No `--platform` means "run all sims" — tests with no sim in their `platforms` - **macOS libomp collision**: on macOS, the root `conftest.py` sets `KMP_DUPLICATE_LIB_OK=TRUE` before `import pytest` to work around a duplicate-libomp abort triggered by homebrew numpy and pip torch coexisting in one Python process (see [troubleshooting/macos-libomp-collision.md](troubleshooting/macos-libomp-collision.md)). Standalone `python test_*.py` bypasses conftest — rely on the env var being exported by the shell or `tools/verify_packaging.sh`. - **sim hangs / `rc=-1` under CPU oversubscription**: on a few-vCPU runner, high `--max-parallel` (or many concurrent sim cases) oversubscribes the host CPUs, where sim's busy-spin handshake can livelock (hang → `rc=124`) or the deinit timeout can false-trip (`run_prepared failed with code -1`). Mitigate with `--max-parallel 2`; onboard is unaffected (see [troubleshooting/sim-oversubscription-hang.md](troubleshooting/sim-oversubscription-hang.md)). +- **`st-onboard-a2a3` mass 507899 is not OOM**: a whole-suite collapse of `507899`/`507018`/`prepare_callable -1` is an AICPU device-fault cascade (`simpler_aicpu_exec` exception), not memory exhaustion. Diagnosis recipe and the per-device preinstall-name fix are in [troubleshooting/a2a3-507899-aicpu-shared-so-fault.md](troubleshooting/a2a3-507899-aicpu-shared-so-fault.md). diff --git a/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md b/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md new file mode 100644 index 0000000000..ca44d32fb0 --- /dev/null +++ b/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md @@ -0,0 +1,111 @@ +# st-onboard-a2a3 mass 507899/507018 — AICPU shared-SO device fault + +## Symptom + +`st-onboard-a2a3` fails intermittently (observed ~19% of runs after #870, vs +~10% before) with a whole-suite collapse: the `L2 tensormap_and_ringbuffer` +phase reports ~10 failed + ~23 errors at once. The surfaced Python errors are + +```text +RuntimeError: simpler_init failed with code 507899 +RuntimeError: prepare_callable failed with code -1 +RuntimeError: run_prepared failed with code 507018 / 507046 / 507901 +[ERROR] rtMalloc failed: 507899 (size=...) +[ERROR] Failed to allocate device GM for ChipCallable buffer +``` + +It looks like an out-of-memory. **It is not.** `npu-smi info` taken right after +the failure shows every chip `Health=OK` with HBM ~3 GB / 64 GB used. + +## Root cause + +`507899` decodes (in the CANN runtime slog) as `[driver error:internal error]` +and `507901` as `[hdc disconnect]`. They are a **cascade after a device fault**, +not the cause. The earliest real error is an **AICPU exception**: + +```text +ProcLogicCqReport: Task run failed, sqe_type=1(aicpu), errType=0x1(task exception) +ProcessStarsAicpuErrorInfo: error from device(chipId:N, dieId:0/1), + an exception occurred during AICPU execution +PrintAicpuErrorInfo: Aicpu kernel execute failed, + soName=simpler_inner_.so, funcName=simpler_aicpu_exec, + errorCode=0x2a # 507018 ACL_ERROR_RT_AICPU_EXCEPTION +``` + +`simpler_aicpu_exec` faults the **whole chip**; afterwards every +`rtStreamCreate` / `rtMalloc` on that chip returns 507899/507901, so the next +test's `simpler_init` (which eagerly creates streams since #870) fails and the +rest of the suite collapses. + +Two facts pin the mechanism: + +1. On a2a3 the `npu-smi` Phy-IDs pair as **die0/die1 of one Ascend910** + (devices 8/9, 4/5, …). The exception fires on **both dies of one chip at the + same instant** — a chip-shared resource was corrupted. +2. The runtime stages SOs under the **shared preinstall directory** + `/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/`. The dispatcher (#870) + wrote the AICPU runtime SO there under a **content-fingerprint-only** name + `simpler_inner_.so` — identical across both dies. Paired dies share that + filesystem, so both wrote/renamed/executed the same file; concurrent + bootstrap corrupted the mmap'd image and trapped `simpler_aicpu_exec`. + +A single-die 50× solo loop of the unregister/re-prepare/dedup tests **never** +reproduced; only the parallel multi-die suite did — consistent with a cross-die +shared-file race, not an intra-process use-after-free. + +## How this was diagnosed (repeatable recipe) + +The host-side simpler log shows only the cascade. To see the originating AICPU +exception you must surface the CANN device slog, which is otherwise hidden: + +1. Run the job with the Ascend slog redirected to stdout (the test harness's + `parallel_scheduler` captures each case's stdout into a per-case `::group::`): + + ```bash + ASCEND_SLOG_PRINT_TO_STDOUT=1 ASCEND_GLOBAL_LOG_LEVEL=1 \ + python -m pytest examples tests/st --platform a2a3 --device -v \ + --pto-session-timeout 600 + ``` + + (`pto_runtime_c_api.cpp` skips its own `dlog_setlevel` when + `ASCEND_GLOBAL_LOG_LEVEL` is set, so the env value wins.) + +2. In the captured log, find the **first** `PrintAicpuErrorInfo` / + `ProcessStarsAicpuErrorInfo` — note `soName`, `funcName`, `errorCode`, and the + `chipId/dieId`. Everything after it that says `507899 [driver error...]` is + noise. + +3. `npu-smi info` confirms whether it is really OOM (it is not here). + +4. Device-side AICPU dumps under `/var/log/npu/...` and `msnpureport` are + **not readable by the CI runner user** (`Not have permission to export + files`), so the stdout slog above is the only channel from CI. The exception + *type* (0x2a) plus the faulting `soName/funcName` is enough to localize. + +## Fix + +Make the staged SO names **per-device** so paired dies never touch the same +file: + +- Dispatcher inner SO: `simpler_inner__.so` + (`src/common/aicpu_dispatcher/aicpu_dispatcher.cpp` `MakeInnerSoPath`, + `src/common/host/load_aicpu_op.cpp` `MakeInnerSoBasename`). The real + `device_id` (was hardcoded `0`) is threaded from `DeviceRunner` through + `BootstrapDispatcher` into both the host JSON reader name and the device-side + writer (`DeviceArgs.device_id`). Bootstrap cache keyed by `(fp, device_id)`. +- AICPU executor orchestration SO: `libdevice_orch___.so` + (`create_orch_so_file`). `device_id` reaches the executor via a new + `KernelArgs.device_id` field → `set_orch_device_id()` in `kernel.cpp` → + `get_orch_device_id()` in `aicpu_executor.cpp`. This one was already pid-named + (mostly safe); the suffix is defense-in-depth for the case where two dies' + device-side pids collide. + +Applied symmetrically to a2a3 and a5. + +## Confirming the fix + +Rerun `st-onboard-a2a3` several times (it is flaky, so one green run is weak +evidence). The fix landed with 5/5 consecutive passes plus the root-cause +match. If the AICPU 0x2a fault ever returns, re-run the slog recipe above and +check whether any *other* SO staged under the shared `aicpu_kernels/0/` +directory still uses a non-per-device name. diff --git a/src/a2a3/platform/include/aicpu/orch_so_file.h b/src/a2a3/platform/include/aicpu/orch_so_file.h index 29318f5ea6..ef13c029d0 100644 --- a/src/a2a3/platform/include/aicpu/orch_so_file.h +++ b/src/a2a3/platform/include/aicpu/orch_so_file.h @@ -44,10 +44,14 @@ * resident orch SOs (one per cid) do not collide on * the same on-disk file. Pass -1 for the legacy * single-slot dispatch path. + * @param device_id ACL device ordinal, suffixed onto the onboard name so + * the paired dies of one chip (which share this preinstall + * filesystem) never stage/execute the same on-disk file. * @param out_path Buffer that receives the full file path on success * @param out_path_size Size of `out_path` in bytes * @return Open writable fd on success, -1 on failure */ -int32_t create_orch_so_file(const char *dir, int32_t callable_id, char *out_path, size_t out_path_size); +int32_t +create_orch_so_file(const char *dir, int32_t callable_id, int32_t device_id, char *out_path, size_t out_path_size); #endif // PLATFORM_AICPU_ORCH_SO_FILE_H_ diff --git a/src/a2a3/platform/include/aicpu/platform_regs.h b/src/a2a3/platform/include/aicpu/platform_regs.h index f3347b3897..83e0f770c6 100644 --- a/src/a2a3/platform/include/aicpu/platform_regs.h +++ b/src/a2a3/platform/include/aicpu/platform_regs.h @@ -66,6 +66,17 @@ void set_platform_pmu_reg_addrs(uint64_t pmu_regs); */ uint64_t get_platform_pmu_reg_addrs(); +/** + * Set the ACL device ordinal for the current run. Pushed by the platform layer + * (kernel.cpp) before aicpu_execute() from KernelArgs.device_id; the executor + * reads it to make the staged orchestration SO filename unique per device so + * paired dies sharing the preinstall filesystem never collide. + */ +void set_orch_device_id(int device_id); + +/** Get the ACL device ordinal set for the current run (0 if unset). */ +int get_orch_device_id(); + #ifdef __cplusplus } #endif diff --git a/src/a2a3/platform/include/common/kernel_args.h b/src/a2a3/platform/include/common/kernel_args.h index 99e72fdea2..60c9499579 100644 --- a/src/a2a3/platform/include/common/kernel_args.h +++ b/src/a2a3/platform/include/common/kernel_args.h @@ -108,6 +108,13 @@ struct KernelArgs { // (no special-casing — write-through works because sim AICPU and host // share memory). Zero when the buffer was not allocated. uint64_t device_wall_data_base{0}; + // ACL device ordinal. Pushed to the AICPU so the executor can suffix the + // staged orchestration SO name (libdevice_orch___.so): + // paired a2a3 dies share the preinstall filesystem, and a content/pid-only + // name risks a cross-die write/execute collision (see simpler_inner fix). + // Trailing field — keeps the CANN-fixed front offsets and AICore-read + // fields in place. + uint32_t device_id{0}; }; #ifdef __cplusplus diff --git a/src/a2a3/platform/onboard/aicpu/CMakeLists.txt b/src/a2a3/platform/onboard/aicpu/CMakeLists.txt index 0de287fa67..fbdbf92482 100644 --- a/src/a2a3/platform/onboard/aicpu/CMakeLists.txt +++ b/src/a2a3/platform/onboard/aicpu/CMakeLists.txt @@ -91,8 +91,8 @@ set_target_properties(aicpu_kernel PROPERTIES OUTPUT_NAME aicpu_kernel) # rtAicpuKernelLaunchExWithArgs(KERNEL_TYPE_AICPU_KFC), invokes # DynTileFwkBackendKernelServerInit, which writes the bundled inner SO bytes # (passed via the extended DeviceArgs at offsets 120/128) to -# /usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner_.so under -# a content-fingerprint basename. After bootstrap the host registers the +# /usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner__.so +# under a content-fingerprint + device-id basename. After bootstrap the host registers the # preinstall file via rtsBinaryLoadFromFile + rtsFuncGetByName and launches # per-task through rtsLaunchCpuKernel; the dispatcher SO itself is never # referenced again. diff --git a/src/a2a3/platform/onboard/aicpu/kernel.cpp b/src/a2a3/platform/onboard/aicpu/kernel.cpp index 2e28997e89..3998d3a0d6 100644 --- a/src/a2a3/platform/onboard/aicpu/kernel.cpp +++ b/src/a2a3/platform/onboard/aicpu/kernel.cpp @@ -104,6 +104,8 @@ extern "C" __attribute__((visibility("default"))) int simpler_aicpu_exec(void *a // Dump enable is an execution control flag propagated via handshake. // The dump base address is only the backing storage location. set_platform_regs(k_args->regs); + // Device ordinal for per-device orchestration-SO naming in the executor. + set_orch_device_id(static_cast(k_args->device_id)); set_platform_dump_base(k_args->dump_data_base); set_dump_tensor_enabled(GET_PROFILING_FLAG(k_args->enable_profiling_flag, PROFILING_FLAG_DUMP_TENSOR)); set_platform_l2_perf_base(k_args->l2_perf_data_base); diff --git a/src/a2a3/platform/onboard/aicpu/orch_so_file.cpp b/src/a2a3/platform/onboard/aicpu/orch_so_file.cpp index 4e7f552324..97d9df97c2 100644 --- a/src/a2a3/platform/onboard/aicpu/orch_so_file.cpp +++ b/src/a2a3/platform/onboard/aicpu/orch_so_file.cpp @@ -15,19 +15,24 @@ #include -int32_t create_orch_so_file(const char *dir, int32_t callable_id, char *out_path, size_t out_path_size) { - // Pid + callable_id naming: AICPU device libc may lack mkstemps. With - // per-callable_id dispatch, multiple orch SOs can be resident in the - // same device process at once (one per cid in `orch_so_table_`), so - // the on-disk file name must be unique per cid — otherwise the - // second cid's `O_TRUNC` would silently shred the first cid's already - // dlopen'd file image and the next launch on cid=0 would SIGBUS. - // callable_id < 0 is the legacy single-slot path: pid alone is fine. +int32_t +create_orch_so_file(const char *dir, int32_t callable_id, int32_t device_id, char *out_path, size_t out_path_size) { + // Pid + callable_id + device_id naming: AICPU device libc may lack mkstemps. + // With per-callable_id dispatch, multiple orch SOs can be resident in the + // same device process at once (one per cid in `orch_so_table_`), so the + // on-disk file name must be unique per cid — otherwise the second cid's + // `O_TRUNC` would silently shred the first cid's already dlopen'd file + // image and the next launch on cid=0 would SIGBUS. The device_id suffix + // additionally isolates the paired dies of one chip, which share this + // preinstall filesystem and would otherwise race on one shared file if + // their device-side pids collided (mirrors the simpler_inner fix). + // callable_id < 0 is the legacy single-slot path. int32_t written; if (callable_id >= 0) { - written = snprintf(out_path, out_path_size, "%s/libdevice_orch_%d_%d.so", dir, getpid(), callable_id); + written = + snprintf(out_path, out_path_size, "%s/libdevice_orch_%d_%d_%d.so", dir, getpid(), callable_id, device_id); } else { - written = snprintf(out_path, out_path_size, "%s/libdevice_orch_%d.so", dir, getpid()); + written = snprintf(out_path, out_path_size, "%s/libdevice_orch_%d_dev%d.so", dir, getpid(), device_id); } if (written < 0 || static_cast(written) >= out_path_size) { return -1; diff --git a/src/a2a3/platform/onboard/host/device_runner.cpp b/src/a2a3/platform/onboard/host/device_runner.cpp index 9fc408a6a8..f01746f90d 100644 --- a/src/a2a3/platform/onboard/host/device_runner.cpp +++ b/src/a2a3/platform/onboard/host/device_runner.cpp @@ -352,7 +352,7 @@ int DeviceRunner::ensure_binaries_loaded() { // One-shot bootstrap: libaicpu_extend_kernels invokes our dispatcher, // which writes the runtime AICPU SO bytes to - // /usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner_.so + // /usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner__.so // using sched-thread (HwHiAiUser) write permission. The dispatcher SO // itself is never persisted to disk — only the transient // libaicpu_extend_kernels dlopen. Subsequent per-task AICPU launches @@ -360,7 +360,7 @@ int DeviceRunner::ensure_binaries_loaded() { // rtsLaunchCpuKernel directly against the preinstall file. int rc = load_aicpu_op_.BootstrapDispatcher( dispatcher_so_binary_.data(), dispatcher_so_binary_.size(), aicpu_so_binary_.data(), aicpu_so_binary_.size(), - stream_aicpu_ + stream_aicpu_, device_id_ ); if (rc != 0) { LOG_ERROR("LoadAicpuOp::BootstrapDispatcher failed: %d", rc); @@ -667,6 +667,8 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { // dlopen'd). Read it directly when populating KernelArgs. kernel_args_.args.log_level = static_cast(HostLogger::get_instance().level()); kernel_args_.args.log_info_v = static_cast(HostLogger::get_instance().info_v()); + // Device ordinal for the AICPU executor's per-device orchestration-SO name. + kernel_args_.args.device_id = static_cast(device_id_); rc = kernel_args_init_ffts_base_addr(kernel_args_); if (rc != 0) { diff --git a/src/a2a3/platform/sim/aicpu/orch_so_file.cpp b/src/a2a3/platform/sim/aicpu/orch_so_file.cpp index 114fe4826d..2467f363bc 100644 --- a/src/a2a3/platform/sim/aicpu/orch_so_file.cpp +++ b/src/a2a3/platform/sim/aicpu/orch_so_file.cpp @@ -24,16 +24,19 @@ #include -int32_t create_orch_so_file(const char *dir, int32_t callable_id, char *out_path, size_t out_path_size) { +int32_t +create_orch_so_file(const char *dir, int32_t callable_id, int32_t device_id, char *out_path, size_t out_path_size) { // mkstemps: multiple sim workers can share a process, so names must be // unique per call. The "XXXXXX" template is replaced in-place. - // callable_id is embedded purely for log readability (mkstemps already - // guarantees uniqueness regardless). + // callable_id / device_id are embedded purely for log readability + // (mkstemps already guarantees uniqueness; sim has no shared preinstall + // filesystem, so the onboard cross-die concern does not apply here). int32_t written; if (callable_id >= 0) { - written = snprintf(out_path, out_path_size, "%s/libdevice_orch_cid%d_XXXXXX.so", dir, callable_id); + written = + snprintf(out_path, out_path_size, "%s/libdevice_orch_cid%d_dev%d_XXXXXX.so", dir, callable_id, device_id); } else { - written = snprintf(out_path, out_path_size, "%s/libdevice_orch_XXXXXX.so", dir); + written = snprintf(out_path, out_path_size, "%s/libdevice_orch_dev%d_XXXXXX.so", dir, device_id); } if (written < 0 || static_cast(written) >= out_path_size) { return -1; diff --git a/src/a2a3/platform/src/aicpu/platform_regs.cpp b/src/a2a3/platform/src/aicpu/platform_regs.cpp index ee2b84af00..87335b423e 100644 --- a/src/a2a3/platform/src/aicpu/platform_regs.cpp +++ b/src/a2a3/platform/src/aicpu/platform_regs.cpp @@ -35,6 +35,7 @@ static uint64_t g_platform_regs = 0; static uint64_t g_platform_pmu_reg_addrs = 0; +static int g_orch_device_id = 0; void set_platform_regs(uint64_t regs) { g_platform_regs = regs; } @@ -44,6 +45,10 @@ void set_platform_pmu_reg_addrs(uint64_t pmu_regs) { g_platform_pmu_reg_addrs = uint64_t get_platform_pmu_reg_addrs() { return g_platform_pmu_reg_addrs; } +void set_orch_device_id(int device_id) { g_orch_device_id = device_id; } + +int get_orch_device_id() { return g_orch_device_id; } + volatile uint32_t *get_reg_ptr(uint64_t reg_base_addr, RegId reg) { return reinterpret_cast(reg_base_addr + reg_offset(reg)); } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index 5c31c5b9ad..5477c29a14 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -275,7 +275,9 @@ int32_t AicpuExecutor::run(Runtime *runtime) { const int32_t num_candidates = sizeof(candidate_dirs) / sizeof(candidate_dirs[0]); for (int32_t i = 0; i < num_candidates && !file_created; i++) { - int32_t fd = create_orch_so_file(candidate_dirs[i], callable_id, so_path, sizeof(so_path)); + int32_t fd = create_orch_so_file( + candidate_dirs[i], callable_id, get_orch_device_id(), so_path, sizeof(so_path) + ); if (fd < 0) { LOG_INFO_V0( "Thread %d: Cannot create SO at %s (errno=%d), trying next path", thread_idx, so_path, errno diff --git a/src/a5/platform/include/aicpu/orch_so_file.h b/src/a5/platform/include/aicpu/orch_so_file.h index 33862527e0..162e12dcb0 100644 --- a/src/a5/platform/include/aicpu/orch_so_file.h +++ b/src/a5/platform/include/aicpu/orch_so_file.h @@ -44,10 +44,14 @@ * resident orch SOs (one per cid) do not collide on * the same on-disk file. Pass -1 for the legacy * single-slot dispatch path. + * @param device_id ACL device ordinal, suffixed onto the onboard name so + * the paired dies of one chip (which share this preinstall + * filesystem) never stage/execute the same on-disk file. * @param out_path Buffer that receives the full file path on success * @param out_path_size Size of `out_path` in bytes * @return Open writable fd on success, -1 on failure */ -int32_t create_orch_so_file(const char *dir, int32_t callable_id, char *out_path, size_t out_path_size); +int32_t +create_orch_so_file(const char *dir, int32_t callable_id, int32_t device_id, char *out_path, size_t out_path_size); #endif // PLATFORM_AICPU_ORCH_SO_FILE_H_ diff --git a/src/a5/platform/include/aicpu/platform_regs.h b/src/a5/platform/include/aicpu/platform_regs.h index 9e8e3a635c..765cee220a 100644 --- a/src/a5/platform/include/aicpu/platform_regs.h +++ b/src/a5/platform/include/aicpu/platform_regs.h @@ -57,6 +57,17 @@ void set_platform_regs(uint64_t regs); */ uint64_t get_platform_regs(); +/** + * Set the ACL device ordinal for the current run. Pushed by the platform layer + * (kernel.cpp) before aicpu_execute() from KernelArgs.device_id; the executor + * reads it to make the staged orchestration SO filename unique per device so + * paired dies sharing the preinstall filesystem never collide. + */ +void set_orch_device_id(int device_id); + +/** Get the ACL device ordinal set for the current run (0 if unset). */ +int get_orch_device_id(); + #ifdef __cplusplus } #endif diff --git a/src/a5/platform/include/common/kernel_args.h b/src/a5/platform/include/common/kernel_args.h index 9a54168061..f138e7af82 100644 --- a/src/a5/platform/include/common/kernel_args.h +++ b/src/a5/platform/include/common/kernel_args.h @@ -87,6 +87,11 @@ struct KernelArgs { // See the a2a3 kernel_args.h for the full design rationale (CANN's // AICPU args copy makes inline fields write-only). uint64_t device_wall_data_base{0}; + // ACL device ordinal. Pushed to the AICPU so the executor can suffix the + // staged orchestration SO name (libdevice_orch___.so), + // mirroring the per-device simpler_inner preinstall fix. Trailing field — + // keeps the CANN-fixed front offsets and AICore-read fields in place. + uint32_t device_id{0}; }; #ifdef __cplusplus diff --git a/src/a5/platform/onboard/aicpu/kernel.cpp b/src/a5/platform/onboard/aicpu/kernel.cpp index 64bf019b3e..9bce80d2a8 100644 --- a/src/a5/platform/onboard/aicpu/kernel.cpp +++ b/src/a5/platform/onboard/aicpu/kernel.cpp @@ -100,6 +100,8 @@ extern "C" __attribute__((visibility("default"))) int simpler_aicpu_exec(void *a // mirrored into Handshake), so decode the umbrella bitmask once and // hand it to the existing platform-state setters. set_platform_regs(k_args->regs); + // Device ordinal for per-device orchestration-SO naming in the executor. + set_orch_device_id(static_cast(k_args->device_id)); set_platform_dump_base(k_args->dump_data_base); set_dump_tensor_enabled(GET_PROFILING_FLAG(k_args->enable_profiling_flag, PROFILING_FLAG_DUMP_TENSOR)); set_platform_l2_perf_base(k_args->l2_perf_data_base); diff --git a/src/a5/platform/onboard/aicpu/orch_so_file.cpp b/src/a5/platform/onboard/aicpu/orch_so_file.cpp index 4e7f552324..97d9df97c2 100644 --- a/src/a5/platform/onboard/aicpu/orch_so_file.cpp +++ b/src/a5/platform/onboard/aicpu/orch_so_file.cpp @@ -15,19 +15,24 @@ #include -int32_t create_orch_so_file(const char *dir, int32_t callable_id, char *out_path, size_t out_path_size) { - // Pid + callable_id naming: AICPU device libc may lack mkstemps. With - // per-callable_id dispatch, multiple orch SOs can be resident in the - // same device process at once (one per cid in `orch_so_table_`), so - // the on-disk file name must be unique per cid — otherwise the - // second cid's `O_TRUNC` would silently shred the first cid's already - // dlopen'd file image and the next launch on cid=0 would SIGBUS. - // callable_id < 0 is the legacy single-slot path: pid alone is fine. +int32_t +create_orch_so_file(const char *dir, int32_t callable_id, int32_t device_id, char *out_path, size_t out_path_size) { + // Pid + callable_id + device_id naming: AICPU device libc may lack mkstemps. + // With per-callable_id dispatch, multiple orch SOs can be resident in the + // same device process at once (one per cid in `orch_so_table_`), so the + // on-disk file name must be unique per cid — otherwise the second cid's + // `O_TRUNC` would silently shred the first cid's already dlopen'd file + // image and the next launch on cid=0 would SIGBUS. The device_id suffix + // additionally isolates the paired dies of one chip, which share this + // preinstall filesystem and would otherwise race on one shared file if + // their device-side pids collided (mirrors the simpler_inner fix). + // callable_id < 0 is the legacy single-slot path. int32_t written; if (callable_id >= 0) { - written = snprintf(out_path, out_path_size, "%s/libdevice_orch_%d_%d.so", dir, getpid(), callable_id); + written = + snprintf(out_path, out_path_size, "%s/libdevice_orch_%d_%d_%d.so", dir, getpid(), callable_id, device_id); } else { - written = snprintf(out_path, out_path_size, "%s/libdevice_orch_%d.so", dir, getpid()); + written = snprintf(out_path, out_path_size, "%s/libdevice_orch_%d_dev%d.so", dir, getpid(), device_id); } if (written < 0 || static_cast(written) >= out_path_size) { return -1; diff --git a/src/a5/platform/onboard/host/device_runner.cpp b/src/a5/platform/onboard/host/device_runner.cpp index bdfb77f276..75cb35e2ce 100644 --- a/src/a5/platform/onboard/host/device_runner.cpp +++ b/src/a5/platform/onboard/host/device_runner.cpp @@ -240,14 +240,14 @@ int DeviceRunner::ensure_binaries_loaded() { } // One-shot bootstrap: libaicpu_extend_kernels invokes our dispatcher, - // which writes the runtime AICPU SO bytes to simpler_inner_.so in + // which writes the runtime AICPU SO bytes to simpler_inner__.so in // the device-side preinstall path. The dispatcher SO itself is never // persisted. Subsequent per-task AICPU launches resolve symbols via // rtsBinaryLoadFromFile + rtsFuncGetByName + rtsLaunchCpuKernel // directly against that preinstall file. int rc = load_aicpu_op_.BootstrapDispatcher( dispatcher_so_binary_.data(), dispatcher_so_binary_.size(), aicpu_so_binary_.data(), aicpu_so_binary_.size(), - stream_aicpu_ + stream_aicpu_, device_id_ ); if (rc != 0) { LOG_ERROR("LoadAicpuOp::BootstrapDispatcher failed: %d", rc); @@ -512,6 +512,8 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { // dlopen'd). Read it directly when populating KernelArgs. kernel_args_.args.log_level = static_cast(HostLogger::get_instance().level()); kernel_args_.args.log_info_v = static_cast(HostLogger::get_instance().info_v()); + // Device ordinal for the AICPU executor's per-device orchestration-SO name. + kernel_args_.args.device_id = static_cast(device_id_); // Start collector mgmt + poll threads now, just before kernels launch. // Starting earlier wastes CPU on empty queues and risks tripping diff --git a/src/a5/platform/sim/aicpu/orch_so_file.cpp b/src/a5/platform/sim/aicpu/orch_so_file.cpp index 114fe4826d..2467f363bc 100644 --- a/src/a5/platform/sim/aicpu/orch_so_file.cpp +++ b/src/a5/platform/sim/aicpu/orch_so_file.cpp @@ -24,16 +24,19 @@ #include -int32_t create_orch_so_file(const char *dir, int32_t callable_id, char *out_path, size_t out_path_size) { +int32_t +create_orch_so_file(const char *dir, int32_t callable_id, int32_t device_id, char *out_path, size_t out_path_size) { // mkstemps: multiple sim workers can share a process, so names must be // unique per call. The "XXXXXX" template is replaced in-place. - // callable_id is embedded purely for log readability (mkstemps already - // guarantees uniqueness regardless). + // callable_id / device_id are embedded purely for log readability + // (mkstemps already guarantees uniqueness; sim has no shared preinstall + // filesystem, so the onboard cross-die concern does not apply here). int32_t written; if (callable_id >= 0) { - written = snprintf(out_path, out_path_size, "%s/libdevice_orch_cid%d_XXXXXX.so", dir, callable_id); + written = + snprintf(out_path, out_path_size, "%s/libdevice_orch_cid%d_dev%d_XXXXXX.so", dir, callable_id, device_id); } else { - written = snprintf(out_path, out_path_size, "%s/libdevice_orch_XXXXXX.so", dir); + written = snprintf(out_path, out_path_size, "%s/libdevice_orch_dev%d_XXXXXX.so", dir, device_id); } if (written < 0 || static_cast(written) >= out_path_size) { return -1; diff --git a/src/a5/platform/src/aicpu/platform_regs.cpp b/src/a5/platform/src/aicpu/platform_regs.cpp index 6a74d2a21c..501942fd4b 100644 --- a/src/a5/platform/src/aicpu/platform_regs.cpp +++ b/src/a5/platform/src/aicpu/platform_regs.cpp @@ -26,11 +26,16 @@ #include "spin_hint.h" static uint64_t g_platform_regs = 0; +static int g_orch_device_id = 0; void set_platform_regs(uint64_t regs) { g_platform_regs = regs; } uint64_t get_platform_regs() { return g_platform_regs; } +void set_orch_device_id(int device_id) { g_orch_device_id = device_id; } + +int get_orch_device_id() { return g_orch_device_id; } + void platform_init_aicore_regs(uint64_t reg_addr) { // Initialize task dispatch register to idle state write_reg(reg_addr, RegId::DATA_MAIN_BASE, AICPU_IDLE_TASK_ID); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index 49d55380f1..d7fd61c396 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -276,7 +276,9 @@ int32_t AicpuExecutor::run(Runtime *runtime) { const int32_t num_candidates = sizeof(candidate_dirs) / sizeof(candidate_dirs[0]); for (int32_t i = 0; i < num_candidates && !file_created; i++) { - int32_t fd = create_orch_so_file(candidate_dirs[i], callable_id, so_path, sizeof(so_path)); + int32_t fd = create_orch_so_file( + candidate_dirs[i], callable_id, get_orch_device_id(), so_path, sizeof(so_path) + ); if (fd < 0) { LOG_INFO_V0( "Thread %d: Cannot create SO at %s (errno=%d), trying next path", thread_idx, so_path, errno diff --git a/src/common/aicpu_dispatcher/README.md b/src/common/aicpu_dispatcher/README.md index b742ce1a0a..95fbc13b34 100644 --- a/src/common/aicpu_dispatcher/README.md +++ b/src/common/aicpu_dispatcher/README.md @@ -3,12 +3,17 @@ Source for `libsimpler_aicpu_dispatcher.so` — a transient bootstrap-only helper loaded by CANN's preinstalled `libaicpu_extend_kernels.so`. Its only job is to write the bundled runtime SO bytes to the main `aicpu_scheduler`'s preinstall -path under a content-fingerprint filename: +path under a content-fingerprint + device-id filename: ```text -/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner_.so +/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner__.so ``` +The `` suffix isolates the paired dies of one a2a3 chip (which share +the preinstall filesystem) so they never write/rename/execute one shared file — +concurrent bootstrap on a shared file corrupted the mmap'd image and faulted +`simpler_aicpu_exec` (507018 → chip fault → 507899 cascade). + The dispatcher SO itself is **never** persisted to disk and **never** dispatches at per-task launch time. After bootstrap, the host registers the preinstall file via `rtsBinaryLoadFromFile` (JSON load, cpuKernelMode=0) and diff --git a/src/common/aicpu_dispatcher/aicpu_dispatcher.cpp b/src/common/aicpu_dispatcher/aicpu_dispatcher.cpp index 9eadae7663..43a0b803aa 100644 --- a/src/common/aicpu_dispatcher/aicpu_dispatcher.cpp +++ b/src/common/aicpu_dispatcher/aicpu_dispatcher.cpp @@ -97,9 +97,11 @@ uint64_t Fingerprint(const char *data, uint64_t len) { // Preinstall path — HwHiAiUser owns this dir, the sched thread can write here. // device-side /tmp is mounted read-only / restricted in CANN 9.0. -std::string MakeInnerSoPath(uint64_t fp) { +std::string MakeInnerSoPath(uint64_t fp, uint64_t device_id) { char buf[256]; - snprintf(buf, sizeof(buf), "/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner_%016lx.so", fp); + snprintf( + buf, sizeof(buf), "/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner_%016lx_%lu.so", fp, device_id + ); return buf; } @@ -190,7 +192,7 @@ __attribute__((visibility("default"))) uint32_t DynTileFwkBackendKernelServerIni } const char *inner_bytes = reinterpret_cast(d->inner_so_bin); uint64_t fp = simpler_dispatcher::Fingerprint(inner_bytes, d->inner_so_len); - std::string path = simpler_dispatcher::MakeInnerSoPath(fp); + std::string path = simpler_dispatcher::MakeInnerSoPath(fp, d->device_id); if (!simpler_dispatcher::WriteBytes(path, inner_bytes, d->inner_so_len)) { return 1; } diff --git a/src/common/aicpu_dispatcher/aicpu_dispatcher.h b/src/common/aicpu_dispatcher/aicpu_dispatcher.h index f626558fd5..cd2f4cec7b 100644 --- a/src/common/aicpu_dispatcher/aicpu_dispatcher.h +++ b/src/common/aicpu_dispatcher/aicpu_dispatcher.h @@ -29,12 +29,15 @@ * path (some /tmp on device, often unlinked after open), dlopens us, * dlsym's the three CANN-contract symbols (Static + DynInit + Dyn), * invokes our `DynTileFwkBackendKernelServerInit`. - * 3. Our Init reads inner_so_bin/inner_so_len from DeviceArgs, fingerprints - * the bytes (FNV-1a over first 64 bytes XOR len), and writes them to - * `/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner_.so`. - * The sched thread (HwHiAiUser) owns this dir, so the write succeeds. - * 4. host computes the same fingerprint locally to derive the same - * preinstall filename. + * 3. Our Init reads inner_so_bin/inner_so_len/device_id from DeviceArgs, + * fingerprints the bytes (ELF Build-ID), and writes them to + * `/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner__.so`. + * The sched thread (HwHiAiUser) owns this dir, so the write succeeds. The + * device_id suffix keeps paired dies sharing this filesystem from writing + * and executing one shared file (which corrupted the image under + * concurrent bootstrap and faulted simpler_aicpu_exec). + * 4. host computes the same fingerprint + uses the same device_id to derive + * the same preinstall filename. * 5. Per-task launches: host calls `rtsBinaryLoadFromFile` to * JSON-register the preinstall file (cpuKernelMode=0), resolves * `simpler_aicpu_init` / `simpler_aicpu_exec` via `rtsFuncGetByName`, diff --git a/src/common/host/load_aicpu_op.cpp b/src/common/host/load_aicpu_op.cpp index 71bcfb922d..7765da7c2c 100644 --- a/src/common/host/load_aicpu_op.cpp +++ b/src/common/host/load_aicpu_op.cpp @@ -22,7 +22,8 @@ #include #include #include -#include +#include +#include #include #include "acl/acl.h" @@ -34,9 +35,9 @@ namespace host { namespace { -std::string MakeInnerSoBasename(uint64_t fp) { +std::string MakeInnerSoBasename(uint64_t fp, int device_id) { char buf[64]; - snprintf(buf, sizeof(buf), "simpler_inner_%016lx.so", fp); + snprintf(buf, sizeof(buf), "simpler_inner_%016lx_%d.so", fp, device_id); return buf; } @@ -65,16 +66,19 @@ struct DeviceBuf { aclError alloc(size_t bytes) { return aclrtMalloc(&ptr, bytes, ACL_MEM_MALLOC_HUGE_FIRST); } }; -// Process-level cache of inner-SO fingerprints we've already bootstrapped. -// Multiple DeviceRunner instances in the same process share one entry per -// runtime here; same-content uploads short-circuit. Guarded by a mutex so -// that callers releasing the Python GIL (e.g. nanobind methods marked +// Process-level cache of (inner-SO fingerprint, device_id) pairs we've already +// bootstrapped. Keyed by both because the preinstall file is now per-device +// (simpler_inner__.so) — a single process driving multiple +// devices must bootstrap each device's file even when the fp matches. +// Multiple DeviceRunner instances in the same process share this; same-content +// same-device uploads short-circuit. Guarded by a mutex so that callers +// releasing the Python GIL (e.g. nanobind methods marked // `nb::call_guard`) cannot race on the set's // internals. The lock is uncontended on the steady-state path and only // touched at DeviceRunner init time, so the overhead is negligible // compared to keeping the invariant alive in a comment. -std::unordered_set &BootstrappedFps() { - static std::unordered_set kSet; +std::set> &BootstrappedFps() { + static std::set> kSet; return kSet; } std::mutex &BootstrappedFpsMutex() { @@ -86,7 +90,7 @@ std::mutex &BootstrappedFpsMutex() { int LoadAicpuOp::BootstrapDispatcher( const void *dispatcher_so_data, size_t dispatcher_so_len, const void *inner_so_data, size_t inner_so_len, - rtStream_t stream + rtStream_t stream, int device_id ) { if (dispatcher_so_data == nullptr || dispatcher_so_len == 0) { LOG_ERROR("BootstrapDispatcher: empty dispatcher SO bytes"); @@ -96,13 +100,22 @@ int LoadAicpuOp::BootstrapDispatcher( LOG_ERROR("BootstrapDispatcher: empty inner SO bytes"); return -1; } + device_id_ = device_id; inner_fp_ = FingerprintBytes(inner_so_data, inner_so_len); - inner_so_basename_ = MakeInnerSoBasename(inner_fp_); + // Per-device basename: the paired dies of one a2a3 chip (e.g. devices 8/9 = + // chipId N die0/die1) share the preinstall filesystem. A content-only name + // would have both dies write/rename/execute one shared + // simpler_inner_.so; concurrent bootstrap there corrupts the mmap'd + // image and faults simpler_aicpu_exec (507018 aicpu exception → chip fault + // → 507899 cascade). Suffixing device_id isolates each die's file. + inner_so_basename_ = MakeInnerSoBasename(inner_fp_, device_id_); { std::lock_guard lk(BootstrappedFpsMutex()); - if (BootstrappedFps().count(inner_fp_) > 0) { - LOG_INFO_V2("BootstrapDispatcher: inner SO fp=%016lx already bootstrapped, skipping", inner_fp_); + if (BootstrappedFps().count({inner_fp_, device_id_}) > 0) { + LOG_INFO_V2( + "BootstrapDispatcher: inner SO fp=%016lx dev=%d already bootstrapped, skipping", inner_fp_, device_id_ + ); return 0; } } @@ -148,7 +161,7 @@ int LoadAicpuOp::BootstrapDispatcher( }; write_qword(96, reinterpret_cast(dev_dispatcher.ptr)); write_qword(104, static_cast(dispatcher_len)); - write_qword(112, 0); + write_qword(112, static_cast(device_id_)); write_qword(120, reinterpret_cast(dev_inner.ptr)); write_qword(128, static_cast(inner_len)); @@ -203,7 +216,7 @@ int LoadAicpuOp::BootstrapDispatcher( ); { std::lock_guard lk(BootstrappedFpsMutex()); - BootstrappedFps().insert(inner_fp_); + BootstrappedFps().insert({inner_fp_, device_id_}); } return 0; } diff --git a/src/common/host/load_aicpu_op.h b/src/common/host/load_aicpu_op.h index 2239ceb3d1..95bb701caf 100644 --- a/src/common/host/load_aicpu_op.h +++ b/src/common/host/load_aicpu_op.h @@ -20,7 +20,7 @@ * `rtAicpuKernelLaunchExWithArgs` (kernel_type = * `KERNEL_TYPE_AICPU_KFC`) targeting libaicpu_extend_kernels. Our * dispatcher then writes the runtime SO to - * `/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner_.so` + * `/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/simpler_inner__.so` * using sched-thread (HwHiAiUser) write permission. The dispatcher SO * itself is never persisted to disk. * @@ -93,11 +93,15 @@ class LoadAicpuOp { * @param inner_so_data Runtime SO bytes (caller-owned, must outlive call) * @param inner_so_len Runtime SO size * @param stream Stream on which to enqueue the bootstrap + * @param device_id ACL device ordinal; suffixed onto the preinstall + * basename (simpler_inner__.so) so + * paired dies sharing the filesystem do not write + * and execute the same on-disk SO. * @return 0 on success, error code on failure */ int BootstrapDispatcher( const void *dispatcher_so_data, size_t dispatcher_so_len, const void *inner_so_data, size_t inner_so_len, - rtStream_t stream + rtStream_t stream, int device_id ); /** @@ -124,6 +128,7 @@ class LoadAicpuOp { std::unordered_map func_handles_; std::string json_file_path_; uint64_t inner_fp_ = 0; + int device_id_ = 0; std::string inner_so_basename_; bool GenerateAicpuOpJson(const std::string &json_path, const std::string &kernel_so); diff --git a/tests/ut/cpp/common/test_orch_so_file.cpp b/tests/ut/cpp/common/test_orch_so_file.cpp index 6e1b32bd69..59dab66553 100644 --- a/tests/ut/cpp/common/test_orch_so_file.cpp +++ b/tests/ut/cpp/common/test_orch_so_file.cpp @@ -18,10 +18,12 @@ // on cid=0 SIGBUS'd inside the AICPU executor (manifesting as // `rtStreamSynchronize (AICPU) failed: 507018` on the host). // -// The fix is to embed `callable_id` in the file name when cid >= 0. This -// test exercises the contract directly: distinct cids must produce distinct -// paths, and the legacy cid=-1 path must remain pid-only (no behavioural -// change for variants that never adopt per-cid dispatch). +// The fix embeds `callable_id` in the file name when cid >= 0, plus a +// `device_id` suffix so the paired dies of one chip (which share the +// preinstall filesystem) never stage/execute the same on-disk SO. This test +// exercises the contract directly: distinct cids must produce distinct paths, +// distinct device_ids must produce distinct paths, and the legacy cid=-1 path +// stays pid + device_id named. #include @@ -61,11 +63,11 @@ TEST(OrchSoFile, DistinctCallableIdsProduceDistinctPaths) { char path0[256] = {}; char path1[256] = {}; - int32_t fd0 = create_orch_so_file(dir.c_str(), /*callable_id=*/0, path0, sizeof(path0)); + int32_t fd0 = create_orch_so_file(dir.c_str(), /*callable_id=*/0, /*device_id=*/0, path0, sizeof(path0)); ASSERT_GE(fd0, 0) << "create_orch_so_file(cid=0) failed"; close(fd0); - int32_t fd1 = create_orch_so_file(dir.c_str(), /*callable_id=*/1, path1, sizeof(path1)); + int32_t fd1 = create_orch_so_file(dir.c_str(), /*callable_id=*/1, /*device_id=*/0, path1, sizeof(path1)); ASSERT_GE(fd1, 0) << "create_orch_so_file(cid=1) failed"; close(fd1); @@ -75,19 +77,41 @@ TEST(OrchSoFile, DistinctCallableIdsProduceDistinctPaths) { rmtree(dir); } -TEST(OrchSoFile, LegacySentinelKeepsPidOnlyNaming) { - // Variants that never adopt per-cid dispatch pass cid=-1; the file - // name must remain pid-only so existing callers see no change. +TEST(OrchSoFile, DistinctDeviceIdsProduceDistinctPaths) { + // Repro for the cross-die 507018 fault: paired dies share the preinstall + // filesystem, so the same cid on two device ids must not collide on one + // on-disk SO (concurrent bootstrap there corrupts the mmap'd image). + const std::string dir = mkscratch_dir(); + char path_dev0[256] = {}; + char path_dev1[256] = {}; + + int32_t fd0 = create_orch_so_file(dir.c_str(), /*callable_id=*/0, /*device_id=*/0, path_dev0, sizeof(path_dev0)); + ASSERT_GE(fd0, 0) << "create_orch_so_file(dev=0) failed"; + close(fd0); + + int32_t fd1 = create_orch_so_file(dir.c_str(), /*callable_id=*/0, /*device_id=*/1, path_dev1, sizeof(path_dev1)); + ASSERT_GE(fd1, 0) << "create_orch_so_file(dev=1) failed"; + close(fd1); + + EXPECT_STRNE(path_dev0, path_dev1) << "Distinct device ids must yield distinct file paths " + "(paired dies share the preinstall filesystem)."; + + rmtree(dir); +} + +TEST(OrchSoFile, LegacySentinelKeepsPidDeviceNaming) { + // Variants that never adopt per-cid dispatch pass cid=-1; the file name is + // pid + device_id (no callable_id segment). const std::string dir = mkscratch_dir(); char path[256] = {}; - int32_t fd = create_orch_so_file(dir.c_str(), /*callable_id=*/-1, path, sizeof(path)); + int32_t fd = create_orch_so_file(dir.c_str(), /*callable_id=*/-1, /*device_id=*/3, path, sizeof(path)); ASSERT_GE(fd, 0); close(fd); char expected[256]; - std::snprintf(expected, sizeof(expected), "%s/libdevice_orch_%d.so", dir.c_str(), getpid()); - EXPECT_STREQ(path, expected) << "Legacy (cid=-1) path must remain pid-only"; + std::snprintf(expected, sizeof(expected), "%s/libdevice_orch_%d_dev3.so", dir.c_str(), getpid()); + EXPECT_STREQ(path, expected) << "Legacy (cid=-1) path must be pid + device_id named"; rmtree(dir); }