From de2e78472d577ab3ccc5652c2787c2d754a257b3 Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Mon, 14 Sep 2026 00:08:47 -0700 Subject: [PATCH 1/3] Fix: preserve device orchestration during clock capture Capturing level-4 clock anchors incorrectly marked device orchestration as host orchestration. The swimlane exporter then rejected the mixed clock domains and omitted the profiling JSON despite successful execution. Preserve the orchestration source in onboard and simulator runners. Use the clock provider name to detect an attempted capture when publishing the correlation session, including unavailable-provider sessions. Add a regression that checks numerical results, AICPU orchestration records, and clock anchors in the exported capture. --- .../onboard/host/device_runner_base.cpp | 4 +-- .../platform/sim/host/device_runner_base.cpp | 4 +-- .../dfx/chip_swimlane/test_chip_swimlane.py | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/common/platform/onboard/host/device_runner_base.cpp b/src/common/platform/onboard/host/device_runner_base.cpp index 9fdde47b62..02bd8d9f7a 100644 --- a/src/common/platform/onboard/host/device_runner_base.cpp +++ b/src/common/platform/onboard/host/device_runner_base.cpp @@ -1460,7 +1460,8 @@ void DeviceRunnerBase::publish_host_phase_run_to_collector(uint32_t pipeline_slo // Read by the collector's initialize() when it sizes the orch phase pool, so // this has to precede it — both now run from the launch arming. chip_swimlane_collector_.set_host_orchestrated(run.host_orchestrated); - if (!run.host_orchestrated || chip_swimlane_collector_.clock_correlation_active()) return; + // A clock-capture attempt supplies a provider name even when unavailable. + if (run.clock_provider_name.empty() || chip_swimlane_collector_.clock_correlation_active()) return; try { chip_swimlane_collector_.begin_clock_correlation_session( run.clock_provider_name.c_str(), run.clock_provider_unit.c_str() @@ -1482,7 +1483,6 @@ void DeviceRunnerBase::begin_clock_correlation_session_if_needed(uint32_t pipeli if (run.chip_swimlane_level != ChipSwimlaneLevel::ORCH_PHASES) return; // The device-orchestrating path has no bind-time capture, so both halves run // here, under the execution claim. - run.host_orchestrated = true; capture_clock_correlation_begin(run); publish_host_phase_run_to_collector(pipeline_slot); } diff --git a/src/common/platform/sim/host/device_runner_base.cpp b/src/common/platform/sim/host/device_runner_base.cpp index 1d6bb8c84b..da140c5a34 100644 --- a/src/common/platform/sim/host/device_runner_base.cpp +++ b/src/common/platform/sim/host/device_runner_base.cpp @@ -827,7 +827,8 @@ void SimDeviceRunnerBase::publish_host_phase_run_to_collector(uint32_t pipeline_ if (pipeline_slot >= host_phase_runs_.size()) return; HostPhaseRunState &run = host_phase_runs_[pipeline_slot]; chip_swimlane_collector_.set_host_orchestrated(run.host_orchestrated); - if (!run.host_orchestrated || chip_swimlane_collector_.clock_correlation_active()) return; + // A clock-capture attempt supplies a provider name even when unavailable. + if (run.clock_provider_name.empty() || chip_swimlane_collector_.clock_correlation_active()) return; try { chip_swimlane_collector_.begin_clock_correlation_session( run.clock_provider_name.c_str(), run.clock_provider_unit.c_str() @@ -846,7 +847,6 @@ void SimDeviceRunnerBase::begin_clock_correlation_session_if_needed(uint32_t pip HostPhaseRunState &run = host_phase_runs_[pipeline_slot]; if (run.chip_swimlane_level != ChipSwimlaneLevel::ORCH_PHASES) return; // No bind-time capture on this path, so both halves run here. - run.host_orchestrated = true; capture_clock_correlation_begin(run); publish_host_phase_run_to_collector(pipeline_slot); } diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py index 20631585c8..8dbf0fca16 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py @@ -22,12 +22,15 @@ ``compute_dag_stats_from_deps`` which this AIV-only workload doesn't. """ +import json import time +from pathlib import Path import torch from simpler.task_interface import ArgDirection as D from simpler_setup import SceneTestCase, TaskArgsBuilder, TensorArg, scene_test +from simpler_setup.scene_test import build_output_prefix from ._swimlane_validate import validate_perf_artifact @@ -112,5 +115,31 @@ def test_run(self, st_platform, st_worker, request): ) +@scene_test(level=2, runtime="tensormap_and_ringbuffer") +class TestDeviceOrchestrationClockCapture(TestChipSwimlane): + """Clock anchors and AICPU orchestration records coexist in one capture.""" + + def test_run(self, st_platform, st_worker, request): + SceneTestCase.test_run(self, st_platform, st_worker, request) + + def _build_config(self, config_dict, *args, **kwargs): + config = super()._build_config(config_dict, *args, **kwargs) + config.enable_chip_swimlane = 4 + config.capture_clock_anchors = True + if not config.output_prefix: + config.output_prefix = str(build_output_prefix(f"{type(self).__name__}_{time.monotonic_ns()}")) + self._clock_capture_path = Path(config.output_prefix) / "chip_swimlane_records.json" + return config + + def compare_outputs(self, test_args, golden_args, output_names, params): + super().compare_outputs(test_args, golden_args, output_names, params) + path = self._clock_capture_path + assert path.is_file(), f"missing device orchestration capture: {path}" + records = json.loads(path.read_text()) + assert any(records.get("aicpu_orchestrator_phases", [])), "AICPU orchestration records missing" + assert records["metadata"].get("orchestrator_source") != "host" + assert "clock_anchors" in records["metadata"], "clock capture was dropped" + + if __name__ == "__main__": SceneTestCase.run_module(__name__) From ad10dae740c1b6b88361cbb60391658fd248de0c Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Mon, 14 Sep 2026 01:24:18 -0700 Subject: [PATCH 2/3] Fix: reset device profiling state before each launch A disabled run skips profiling initialization, so a reused AICPU module can retain the preceding run's level and phase writers. Orchestration then waits on old profiling queues and triggers scheduler starvation. Invalidate the cached level, phase header, thread index, and phase pool and buffer pointers when each launch publishes its profiling enable bit. This happens before the onboard affinity barrier or sim thread startup, and covers enabled and disabled runs without depending on prior teardown. Extend the clock-capture regression to alternate profiling on and off twice on the same worker, preserving numerical and valid-anchor checks. --- .../aicpu/chip_swimlane_collector_aicpu.h | 4 ++- .../aicpu/chip_swimlane_collector_aicpu.cpp | 30 +++++++++++-------- .../dfx/chip_swimlane/test_chip_swimlane.py | 19 ++++++++---- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h b/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h index 60de2e6415..ff27b32504 100644 --- a/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h +++ b/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h @@ -34,7 +34,9 @@ * * Two-channel level transport (mirrors the PMU pattern): * - binary on/off — `enable_profiling_flag` bit1 → `set_chip_swimlane_enabled(bool)` - * at kernel entry; queried via `is_chip_swimlane_enabled()`. + * at kernel entry; queried via `is_chip_swimlane_enabled()`. Each call + * invalidates the preceding launch's cached level and phase writers, + * including when profiling is disabled and init will not run. * - granular ChipSwimlaneLevel — `ChipSwimlaneDataHeader::chip_swimlane_level` * (shared memory); read in `chip_swimlane_aicpu_init` and cached, then queried * via `get_chip_swimlane_level()` for diff --git a/src/common/platform/shared/aicpu/chip_swimlane_collector_aicpu.cpp b/src/common/platform/shared/aicpu/chip_swimlane_collector_aicpu.cpp index 262e7d0873..74de5cdac3 100644 --- a/src/common/platform/shared/aicpu/chip_swimlane_collector_aicpu.cpp +++ b/src/common/platform/shared/aicpu/chip_swimlane_collector_aicpu.cpp @@ -116,7 +116,22 @@ extern "C" void set_platform_chip_swimlane_base(uint64_t chip_swimlane_data_base g_platform_chip_swimlane_base = chip_swimlane_data_base; } extern "C" uint64_t get_platform_chip_swimlane_base() { return g_platform_chip_swimlane_base; } -extern "C" void set_chip_swimlane_enabled(bool enable) { g_enable_chip_swimlane = enable; } +extern "C" void set_chip_swimlane_enabled(bool enable) { + // Every launch publishes its enable bit before the onboard affinity barrier + // or before sim threads start. Disabled launches skip profiling init, so + // all phase-writer state must be invalidated at this common entry point. + g_enable_chip_swimlane = enable; + g_chip_swimlane_level = ChipSwimlaneLevel::DISABLED; + s_phase_initialized = false; + s_chip_swimlane_header = nullptr; + s_orch_thread_idx = -1; + for (int t = 0; t < PLATFORM_MAX_AICPU_THREADS; t++) { + s_sched_phase_pools[t] = nullptr; + s_current_sched_phase_buffers[t] = nullptr; + s_orch_phase_pools[t] = nullptr; + s_current_orch_phase_buffers[t] = nullptr; + } +} extern "C" bool is_chip_swimlane_enabled() { return g_enable_chip_swimlane; } extern "C" void set_platform_chip_swimlane_aicore_rotation_table(uint64_t table_addr) { g_platform_chip_swimlane_aicore_rotation_table = table_addr; @@ -275,18 +290,7 @@ try_pop_records_buffer(int core_id, ChipSwimlaneAicpuTaskPool *state, uint32_t n } void chip_swimlane_aicpu_init(int worker_count) { - // Reset cross-launch state up front. AICPU statics persist across launches - // on the same loaded .so; without this reset, an enabled→disabled launch - // sequence would leave s_phase_initialized=true from the prior run, and - // any subsequent record_sched_phase / record_orch_phase call would - // dereference the prior launch's (now-freed) s_sched_phase_pools / - // s_orch_phase_pools pointers. Same shape as the [[block_local]] reset - // in onboard/aicore/kernel.cpp for the AICore-side rotation slot - // (fixed in #936). - s_phase_initialized = false; - - // Reset AICore dispatch-count bookkeeping for the same reason: the next - // launch must start counting from 0 so the rotation boundary check + // Each profiled launch starts counting AICore dispatches from 0 so the rotation boundary check // (count % BUFFER_SIZE == 0) lands on the right dispatches. Stale values // from a prior launch would skip the first rotation (count already past a // boundary) or trigger one prematurely. The deferred-enqueue stash must be diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py index 8dbf0fca16..77e7a16fd5 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py @@ -117,15 +117,19 @@ def test_run(self, st_platform, st_worker, request): @scene_test(level=2, runtime="tensormap_and_ringbuffer") class TestDeviceOrchestrationClockCapture(TestChipSwimlane): - """Clock anchors and AICPU orchestration records coexist in one capture.""" + """Clock captures and disabled runs alternate on the same worker.""" + + _swimlane_level = 4 def test_run(self, st_platform, st_worker, request): - SceneTestCase.test_run(self, st_platform, st_worker, request) + for level in (4, 0, 4, 0): + self._swimlane_level = level + SceneTestCase.test_run(self, st_platform, st_worker, request) def _build_config(self, config_dict, *args, **kwargs): config = super()._build_config(config_dict, *args, **kwargs) - config.enable_chip_swimlane = 4 - config.capture_clock_anchors = True + config.enable_chip_swimlane = self._swimlane_level + config.capture_clock_anchors = bool(self._swimlane_level) if not config.output_prefix: config.output_prefix = str(build_output_prefix(f"{type(self).__name__}_{time.monotonic_ns()}")) self._clock_capture_path = Path(config.output_prefix) / "chip_swimlane_records.json" @@ -133,12 +137,17 @@ def _build_config(self, config_dict, *args, **kwargs): def compare_outputs(self, test_args, golden_args, output_names, params): super().compare_outputs(test_args, golden_args, output_names, params) + if not self._swimlane_level: + return path = self._clock_capture_path assert path.is_file(), f"missing device orchestration capture: {path}" records = json.loads(path.read_text()) assert any(records.get("aicpu_orchestrator_phases", [])), "AICPU orchestration records missing" assert records["metadata"].get("orchestrator_source") != "host" - assert "clock_anchors" in records["metadata"], "clock capture was dropped" + anchors = records["metadata"].get("clock_anchors", {}).get("samples", []) + assert any(sample["error"] is None and sample["device_cycles"] > 0 for sample in anchors), ( + "no valid clock anchor was captured" + ) if __name__ == "__main__": From c326f3708d1f2ad97204505133f9f45976f84d8a Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Mon, 14 Sep 2026 18:03:09 -0700 Subject: [PATCH 3/3] Fix: validate each device profiling capture independently Remove a previous profiling JSON before each enabled run so repeated case directories cannot hide a missing export. Check all case artifacts after each profiling iteration, independently of golden comparison. Require device captures to omit orchestrator_source, matching the raw JSON schema and rejecting unexpected values such as null. --- .../dfx/chip_swimlane/test_chip_swimlane.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py index 77e7a16fd5..725129f179 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py @@ -120,11 +120,23 @@ class TestDeviceOrchestrationClockCapture(TestChipSwimlane): """Clock captures and disabled runs alternate on the same worker.""" _swimlane_level = 4 + _clock_capture_paths: tuple[Path, ...] = () def test_run(self, st_platform, st_worker, request): for level in (4, 0, 4, 0): self._swimlane_level = level + self._clock_capture_paths = () SceneTestCase.test_run(self, st_platform, st_worker, request) + # Artifact checks cover every case even when golden comparison is disabled. + for path in self._clock_capture_paths: + assert path.is_file(), f"missing device orchestration capture: {path}" + records = json.loads(path.read_text()) + assert any(records.get("aicpu_orchestrator_phases", [])), "AICPU orchestration records missing" + assert "orchestrator_source" not in records["metadata"] + anchors = records["metadata"].get("clock_anchors", {}).get("samples", []) + assert any(sample["error"] is None and sample["device_cycles"] > 0 for sample in anchors), ( + "no valid clock anchor was captured" + ) def _build_config(self, config_dict, *args, **kwargs): config = super()._build_config(config_dict, *args, **kwargs) @@ -132,23 +144,13 @@ def _build_config(self, config_dict, *args, **kwargs): config.capture_clock_anchors = bool(self._swimlane_level) if not config.output_prefix: config.output_prefix = str(build_output_prefix(f"{type(self).__name__}_{time.monotonic_ns()}")) - self._clock_capture_path = Path(config.output_prefix) / "chip_swimlane_records.json" + if self._swimlane_level: + path = Path(config.output_prefix) / "chip_swimlane_records.json" + # CLI-generated prefixes can repeat for the same case within one second. + path.unlink(missing_ok=True) + self._clock_capture_paths += (path,) return config - def compare_outputs(self, test_args, golden_args, output_names, params): - super().compare_outputs(test_args, golden_args, output_names, params) - if not self._swimlane_level: - return - path = self._clock_capture_path - assert path.is_file(), f"missing device orchestration capture: {path}" - records = json.loads(path.read_text()) - assert any(records.get("aicpu_orchestrator_phases", [])), "AICPU orchestration records missing" - assert records["metadata"].get("orchestrator_source") != "host" - anchors = records["metadata"].get("clock_anchors", {}).get("samples", []) - assert any(sample["error"] is None and sample["device_cycles"] > 0 for sample in anchors), ( - "no valid clock anchor was captured" - ) - if __name__ == "__main__": SceneTestCase.run_module(__name__)