diff --git a/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h b/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h index 60de2e6415..ff27b32504 100644 --- a/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h +++ b/src/common/platform/include/aicpu/chip_swimlane_collector_aicpu.h @@ -34,7 +34,9 @@ * * Two-channel level transport (mirrors the PMU pattern): * - binary on/off — `enable_profiling_flag` bit1 → `set_chip_swimlane_enabled(bool)` - * at kernel entry; queried via `is_chip_swimlane_enabled()`. + * at kernel entry; queried via `is_chip_swimlane_enabled()`. Each call + * invalidates the preceding launch's cached level and phase writers, + * including when profiling is disabled and init will not run. * - granular ChipSwimlaneLevel — `ChipSwimlaneDataHeader::chip_swimlane_level` * (shared memory); read in `chip_swimlane_aicpu_init` and cached, then queried * via `get_chip_swimlane_level()` for diff --git a/src/common/platform/onboard/host/device_runner_base.cpp b/src/common/platform/onboard/host/device_runner_base.cpp index 9fdde47b62..02bd8d9f7a 100644 --- a/src/common/platform/onboard/host/device_runner_base.cpp +++ b/src/common/platform/onboard/host/device_runner_base.cpp @@ -1460,7 +1460,8 @@ void DeviceRunnerBase::publish_host_phase_run_to_collector(uint32_t pipeline_slo // Read by the collector's initialize() when it sizes the orch phase pool, so // this has to precede it — both now run from the launch arming. chip_swimlane_collector_.set_host_orchestrated(run.host_orchestrated); - if (!run.host_orchestrated || chip_swimlane_collector_.clock_correlation_active()) return; + // A clock-capture attempt supplies a provider name even when unavailable. + if (run.clock_provider_name.empty() || chip_swimlane_collector_.clock_correlation_active()) return; try { chip_swimlane_collector_.begin_clock_correlation_session( run.clock_provider_name.c_str(), run.clock_provider_unit.c_str() @@ -1482,7 +1483,6 @@ void DeviceRunnerBase::begin_clock_correlation_session_if_needed(uint32_t pipeli if (run.chip_swimlane_level != ChipSwimlaneLevel::ORCH_PHASES) return; // The device-orchestrating path has no bind-time capture, so both halves run // here, under the execution claim. - run.host_orchestrated = true; capture_clock_correlation_begin(run); publish_host_phase_run_to_collector(pipeline_slot); } diff --git a/src/common/platform/shared/aicpu/chip_swimlane_collector_aicpu.cpp b/src/common/platform/shared/aicpu/chip_swimlane_collector_aicpu.cpp index 262e7d0873..74de5cdac3 100644 --- a/src/common/platform/shared/aicpu/chip_swimlane_collector_aicpu.cpp +++ b/src/common/platform/shared/aicpu/chip_swimlane_collector_aicpu.cpp @@ -116,7 +116,22 @@ extern "C" void set_platform_chip_swimlane_base(uint64_t chip_swimlane_data_base g_platform_chip_swimlane_base = chip_swimlane_data_base; } extern "C" uint64_t get_platform_chip_swimlane_base() { return g_platform_chip_swimlane_base; } -extern "C" void set_chip_swimlane_enabled(bool enable) { g_enable_chip_swimlane = enable; } +extern "C" void set_chip_swimlane_enabled(bool enable) { + // Every launch publishes its enable bit before the onboard affinity barrier + // or before sim threads start. Disabled launches skip profiling init, so + // all phase-writer state must be invalidated at this common entry point. + g_enable_chip_swimlane = enable; + g_chip_swimlane_level = ChipSwimlaneLevel::DISABLED; + s_phase_initialized = false; + s_chip_swimlane_header = nullptr; + s_orch_thread_idx = -1; + for (int t = 0; t < PLATFORM_MAX_AICPU_THREADS; t++) { + s_sched_phase_pools[t] = nullptr; + s_current_sched_phase_buffers[t] = nullptr; + s_orch_phase_pools[t] = nullptr; + s_current_orch_phase_buffers[t] = nullptr; + } +} extern "C" bool is_chip_swimlane_enabled() { return g_enable_chip_swimlane; } extern "C" void set_platform_chip_swimlane_aicore_rotation_table(uint64_t table_addr) { g_platform_chip_swimlane_aicore_rotation_table = table_addr; @@ -275,18 +290,7 @@ try_pop_records_buffer(int core_id, ChipSwimlaneAicpuTaskPool *state, uint32_t n } void chip_swimlane_aicpu_init(int worker_count) { - // Reset cross-launch state up front. AICPU statics persist across launches - // on the same loaded .so; without this reset, an enabled→disabled launch - // sequence would leave s_phase_initialized=true from the prior run, and - // any subsequent record_sched_phase / record_orch_phase call would - // dereference the prior launch's (now-freed) s_sched_phase_pools / - // s_orch_phase_pools pointers. Same shape as the [[block_local]] reset - // in onboard/aicore/kernel.cpp for the AICore-side rotation slot - // (fixed in #936). - s_phase_initialized = false; - - // Reset AICore dispatch-count bookkeeping for the same reason: the next - // launch must start counting from 0 so the rotation boundary check + // Each profiled launch starts counting AICore dispatches from 0 so the rotation boundary check // (count % BUFFER_SIZE == 0) lands on the right dispatches. Stale values // from a prior launch would skip the first rotation (count already past a // boundary) or trigger one prematurely. The deferred-enqueue stash must be diff --git a/src/common/platform/sim/host/device_runner_base.cpp b/src/common/platform/sim/host/device_runner_base.cpp index 1d6bb8c84b..da140c5a34 100644 --- a/src/common/platform/sim/host/device_runner_base.cpp +++ b/src/common/platform/sim/host/device_runner_base.cpp @@ -827,7 +827,8 @@ void SimDeviceRunnerBase::publish_host_phase_run_to_collector(uint32_t pipeline_ if (pipeline_slot >= host_phase_runs_.size()) return; HostPhaseRunState &run = host_phase_runs_[pipeline_slot]; chip_swimlane_collector_.set_host_orchestrated(run.host_orchestrated); - if (!run.host_orchestrated || chip_swimlane_collector_.clock_correlation_active()) return; + // A clock-capture attempt supplies a provider name even when unavailable. + if (run.clock_provider_name.empty() || chip_swimlane_collector_.clock_correlation_active()) return; try { chip_swimlane_collector_.begin_clock_correlation_session( run.clock_provider_name.c_str(), run.clock_provider_unit.c_str() @@ -846,7 +847,6 @@ void SimDeviceRunnerBase::begin_clock_correlation_session_if_needed(uint32_t pip HostPhaseRunState &run = host_phase_runs_[pipeline_slot]; if (run.chip_swimlane_level != ChipSwimlaneLevel::ORCH_PHASES) return; // No bind-time capture on this path, so both halves run here. - run.host_orchestrated = true; capture_clock_correlation_begin(run); publish_host_phase_run_to_collector(pipeline_slot); } diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py index 20631585c8..725129f179 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py @@ -22,12 +22,15 @@ ``compute_dag_stats_from_deps`` which this AIV-only workload doesn't. """ +import json import time +from pathlib import Path import torch from simpler.task_interface import ArgDirection as D from simpler_setup import SceneTestCase, TaskArgsBuilder, TensorArg, scene_test +from simpler_setup.scene_test import build_output_prefix from ._swimlane_validate import validate_perf_artifact @@ -112,5 +115,42 @@ def test_run(self, st_platform, st_worker, request): ) +@scene_test(level=2, runtime="tensormap_and_ringbuffer") +class TestDeviceOrchestrationClockCapture(TestChipSwimlane): + """Clock captures and disabled runs alternate on the same worker.""" + + _swimlane_level = 4 + _clock_capture_paths: tuple[Path, ...] = () + + def test_run(self, st_platform, st_worker, request): + for level in (4, 0, 4, 0): + self._swimlane_level = level + self._clock_capture_paths = () + SceneTestCase.test_run(self, st_platform, st_worker, request) + # Artifact checks cover every case even when golden comparison is disabled. + for path in self._clock_capture_paths: + assert path.is_file(), f"missing device orchestration capture: {path}" + records = json.loads(path.read_text()) + assert any(records.get("aicpu_orchestrator_phases", [])), "AICPU orchestration records missing" + assert "orchestrator_source" not in records["metadata"] + anchors = records["metadata"].get("clock_anchors", {}).get("samples", []) + assert any(sample["error"] is None and sample["device_cycles"] > 0 for sample in anchors), ( + "no valid clock anchor was captured" + ) + + def _build_config(self, config_dict, *args, **kwargs): + config = super()._build_config(config_dict, *args, **kwargs) + config.enable_chip_swimlane = self._swimlane_level + config.capture_clock_anchors = bool(self._swimlane_level) + if not config.output_prefix: + config.output_prefix = str(build_output_prefix(f"{type(self).__name__}_{time.monotonic_ns()}")) + if self._swimlane_level: + path = Path(config.output_prefix) / "chip_swimlane_records.json" + # CLI-generated prefixes can repeat for the same case within one second. + path.unlink(missing_ok=True) + self._clock_capture_paths += (path,) + return config + + if __name__ == "__main__": SceneTestCase.run_module(__name__)