Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@
*
* Two-channel level transport (mirrors the PMU pattern):
* - binary on/off — `enable_profiling_flag` bit1 → `set_chip_swimlane_enabled(bool)`
* at kernel entry; queried via `is_chip_swimlane_enabled()`.
* at kernel entry; queried via `is_chip_swimlane_enabled()`. Each call
* invalidates the preceding launch's cached level and phase writers,
* including when profiling is disabled and init will not run.
* - granular ChipSwimlaneLevel — `ChipSwimlaneDataHeader::chip_swimlane_level`
* (shared memory); read in `chip_swimlane_aicpu_init` and cached, then queried
* via `get_chip_swimlane_level()` for
Expand Down
4 changes: 2 additions & 2 deletions src/common/platform/onboard/host/device_runner_base.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1460,7 +1460,8 @@ void DeviceRunnerBase::publish_host_phase_run_to_collector(uint32_t pipeline_slo
// Read by the collector's initialize() when it sizes the orch phase pool, so
// this has to precede it — both now run from the launch arming.
chip_swimlane_collector_.set_host_orchestrated(run.host_orchestrated);
if (!run.host_orchestrated || chip_swimlane_collector_.clock_correlation_active()) return;
// A clock-capture attempt supplies a provider name even when unavailable.
if (run.clock_provider_name.empty() || chip_swimlane_collector_.clock_correlation_active()) return;
try {
chip_swimlane_collector_.begin_clock_correlation_session(
run.clock_provider_name.c_str(), run.clock_provider_unit.c_str()
Expand All @@ -1482,7 +1483,6 @@ void DeviceRunnerBase::begin_clock_correlation_session_if_needed(uint32_t pipeli
if (run.chip_swimlane_level != ChipSwimlaneLevel::ORCH_PHASES) return;
// The device-orchestrating path has no bind-time capture, so both halves run
// here, under the execution claim.
run.host_orchestrated = true;
capture_clock_correlation_begin(run);
publish_host_phase_run_to_collector(pipeline_slot);
}
Expand Down
30 changes: 17 additions & 13 deletions src/common/platform/shared/aicpu/chip_swimlane_collector_aicpu.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,22 @@ extern "C" void set_platform_chip_swimlane_base(uint64_t chip_swimlane_data_base
g_platform_chip_swimlane_base = chip_swimlane_data_base;
}
extern "C" uint64_t get_platform_chip_swimlane_base() { return g_platform_chip_swimlane_base; }
extern "C" void set_chip_swimlane_enabled(bool enable) { g_enable_chip_swimlane = enable; }
extern "C" void set_chip_swimlane_enabled(bool enable) {
// Every launch publishes its enable bit before the onboard affinity barrier
// or before sim threads start. Disabled launches skip profiling init, so
// all phase-writer state must be invalidated at this common entry point.
g_enable_chip_swimlane = enable;
g_chip_swimlane_level = ChipSwimlaneLevel::DISABLED;
s_phase_initialized = false;
s_chip_swimlane_header = nullptr;
s_orch_thread_idx = -1;
for (int t = 0; t < PLATFORM_MAX_AICPU_THREADS; t++) {
s_sched_phase_pools[t] = nullptr;
s_current_sched_phase_buffers[t] = nullptr;
s_orch_phase_pools[t] = nullptr;
s_current_orch_phase_buffers[t] = nullptr;
}
}
extern "C" bool is_chip_swimlane_enabled() { return g_enable_chip_swimlane; }
extern "C" void set_platform_chip_swimlane_aicore_rotation_table(uint64_t table_addr) {
g_platform_chip_swimlane_aicore_rotation_table = table_addr;
Expand Down Expand Up @@ -275,18 +290,7 @@ try_pop_records_buffer(int core_id, ChipSwimlaneAicpuTaskPool *state, uint32_t n
}

void chip_swimlane_aicpu_init(int worker_count) {
// Reset cross-launch state up front. AICPU statics persist across launches
// on the same loaded .so; without this reset, an enabled→disabled launch
// sequence would leave s_phase_initialized=true from the prior run, and
// any subsequent record_sched_phase / record_orch_phase call would
// dereference the prior launch's (now-freed) s_sched_phase_pools /
// s_orch_phase_pools pointers. Same shape as the [[block_local]] reset
// in onboard/aicore/kernel.cpp for the AICore-side rotation slot
// (fixed in #936).
s_phase_initialized = false;

// Reset AICore dispatch-count bookkeeping for the same reason: the next
// launch must start counting from 0 so the rotation boundary check
// Each profiled launch starts counting AICore dispatches from 0 so the rotation boundary check
// (count % BUFFER_SIZE == 0) lands on the right dispatches. Stale values
// from a prior launch would skip the first rotation (count already past a
// boundary) or trigger one prematurely. The deferred-enqueue stash must be
Expand Down
4 changes: 2 additions & 2 deletions src/common/platform/sim/host/device_runner_base.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -827,7 +827,8 @@ void SimDeviceRunnerBase::publish_host_phase_run_to_collector(uint32_t pipeline_
if (pipeline_slot >= host_phase_runs_.size()) return;
HostPhaseRunState &run = host_phase_runs_[pipeline_slot];
chip_swimlane_collector_.set_host_orchestrated(run.host_orchestrated);
if (!run.host_orchestrated || chip_swimlane_collector_.clock_correlation_active()) return;
// A clock-capture attempt supplies a provider name even when unavailable.
if (run.clock_provider_name.empty() || chip_swimlane_collector_.clock_correlation_active()) return;
try {
chip_swimlane_collector_.begin_clock_correlation_session(
run.clock_provider_name.c_str(), run.clock_provider_unit.c_str()
Expand All @@ -846,7 +847,6 @@ void SimDeviceRunnerBase::begin_clock_correlation_session_if_needed(uint32_t pip
HostPhaseRunState &run = host_phase_runs_[pipeline_slot];
if (run.chip_swimlane_level != ChipSwimlaneLevel::ORCH_PHASES) return;
// No bind-time capture on this path, so both halves run here.
run.host_orchestrated = true;
capture_clock_correlation_begin(run);
publish_host_phase_run_to_collector(pipeline_slot);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,15 @@
``compute_dag_stats_from_deps`` which this AIV-only workload doesn't.
"""

import json
import time
from pathlib import Path

import torch
from simpler.task_interface import ArgDirection as D

from simpler_setup import SceneTestCase, TaskArgsBuilder, TensorArg, scene_test
from simpler_setup.scene_test import build_output_prefix

from ._swimlane_validate import validate_perf_artifact

Expand Down Expand Up @@ -112,5 +115,42 @@ def test_run(self, st_platform, st_worker, request):
)


@scene_test(level=2, runtime="tensormap_and_ringbuffer")
class TestDeviceOrchestrationClockCapture(TestChipSwimlane):
"""Clock captures and disabled runs alternate on the same worker."""

_swimlane_level = 4
_clock_capture_paths: tuple[Path, ...] = ()

def test_run(self, st_platform, st_worker, request):
for level in (4, 0, 4, 0):
self._swimlane_level = level
self._clock_capture_paths = ()
SceneTestCase.test_run(self, st_platform, st_worker, request)
# Artifact checks cover every case even when golden comparison is disabled.
for path in self._clock_capture_paths:
assert path.is_file(), f"missing device orchestration capture: {path}"
records = json.loads(path.read_text())
assert any(records.get("aicpu_orchestrator_phases", [])), "AICPU orchestration records missing"
assert "orchestrator_source" not in records["metadata"]
anchors = records["metadata"].get("clock_anchors", {}).get("samples", [])
assert any(sample["error"] is None and sample["device_cycles"] > 0 for sample in anchors), (
"no valid clock anchor was captured"
)

def _build_config(self, config_dict, *args, **kwargs):
config = super()._build_config(config_dict, *args, **kwargs)
config.enable_chip_swimlane = self._swimlane_level
config.capture_clock_anchors = bool(self._swimlane_level)
if not config.output_prefix:
config.output_prefix = str(build_output_prefix(f"{type(self).__name__}_{time.monotonic_ns()}"))
if self._swimlane_level:
path = Path(config.output_prefix) / "chip_swimlane_records.json"
# CLI-generated prefixes can repeat for the same case within one second.
path.unlink(missing_ok=True)
self._clock_capture_paths += (path,)
return config


if __name__ == "__main__":
SceneTestCase.run_module(__name__)
Loading