feat(dfx): profile A5 HBG AICore scheduler

.github/workflows/_st-sim-a5.yml

@@ -116,7 +116,19 @@ jobs:
       - name: chip_swimlane smoke (a5 host_build_graph)
         if: inputs.include_dfx_smokes
         run: |
-          .venv/bin/python -m pytest tests/st/a5/host_build_graph/dfx/chip_swimlane/ \
+          for level in 1 2; do
+            .venv/bin/python -m pytest \
+              tests/st/a5/host_build_graph/dfx/chip_swimlane/test_scheduler_phases.py \
+              --case TestSchedulerPhases::resolve_dummy \
+              --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
+              --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" \
+              --enable-chip-swimlane "$level"
+          done
+          .venv/bin/python -m pytest \
+            tests/st/a5/host_build_graph/dfx/chip_swimlane/test_scheduler_phases.py \
+            tests/st/a5/host_build_graph/single_core_dag/test_single_core_dag.py \
+            --case TestSchedulerPhases::resolve_dummy \
+            --case TestHbgSingleCoreDag::mixed_multi_root_64 \
             --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
             --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" \
             --enable-chip-swimlane 3

src/a5/runtime/host_build_graph/aicpu/aicore_lifecycle.cpp

@@ -41,6 +41,8 @@ uint64_t resident_scheduler_timeout_cycles() {
                             SCHEDULER_TIMEOUT_CYCLES;
 }
 
+bool lifecycle_timing_enabled() { return get_chip_swimlane_level() >= ChipSwimlaneLevel::SCHEDULE_TIMING; }
+
 void record_lifecycle_timeout(Runtime *runtime, SchedulerErrorSite error_site) {
     SchedulerWorkerContext *context = aicore_scheduler_bootstrap_context(runtime);
     if (context == nullptr) return;
@@ -69,10 +71,9 @@ int32_t AicoreLifecycle::pre_handshake_init(Runtime *runtime, int32_t aicpu_thre
     handshake_failed_.store(false, std::memory_order_release);
 
     const bool chip_swimlane_enabled = is_chip_swimlane_enabled();
-    if (chip_swimlane_enabled || is_pmu_enabled() || is_dump_args_enabled()) {
+    if (is_pmu_enabled() || is_dump_args_enabled()) {
         LOG_WARN(
-            "A5 HBG AICore Scheduler diagnostics are best-effort: artifacts may be absent or incomplete and do not "
-            "yet describe Scheduler scheduling"
+            "A5 HBG AICore Scheduler PMU/argument diagnostics are best-effort: artifacts may be absent or incomplete"
         );
     }
     if (chip_swimlane_enabled) chip_swimlane_aicpu_init(core_count_);
@@ -121,7 +122,7 @@ void AicoreLifecycle::handshake_partition(Runtime *runtime, int32_t tidx, int32_
             }
             ready[ready_count++] = {
                 i, physical_core_id, regs[physical_core_id], handshake->core_type,
-                is_chip_swimlane_enabled() ? get_sys_cnt_aicpu() : 0
+                lifecycle_timing_enabled() ? get_sys_cnt_aicpu() : 0
             };
         }
         if (scheduler_watchdog_expired(wait_start, get_sys_cnt_aicpu(), timeout_cycles)) {
@@ -138,7 +139,7 @@ void AicoreLifecycle::handshake_partition(Runtime *runtime, int32_t tidx, int32_
                                   nullptr,       core.handshake_observed_cycles, 0};
         physical_core_ids_[core.worker_id] = core.physical_core_id;
     }
-    const uint64_t partition_complete_cycles = is_chip_swimlane_enabled() ? get_sys_cnt_aicpu() : 0;
+    const uint64_t partition_complete_cycles = lifecycle_timing_enabled() ? get_sys_cnt_aicpu() : 0;
     for (int32_t i = lo; i < hi; ++i)
         cores_[i].handshake_partition_complete_cycles = partition_complete_cycles;
 }
@@ -150,8 +151,8 @@ int32_t AicoreLifecycle::post_handshake_init(Runtime *runtime) {
         return -1;
     cache_invalidate_range(bootstrap_context, 256);
     void *scheduler_state_base = aicore_scheduler_state_base(bootstrap_context);
-    const bool trace_enabled = is_chip_swimlane_enabled();
-    const uint64_t config_start_cycles = trace_enabled ? get_sys_cnt_aicpu() : 0;
+    const bool record_lifecycle_timing = lifecycle_timing_enabled();
+    const uint64_t config_start_cycles = record_lifecycle_timing ? get_sys_cnt_aicpu() : 0;
 
     auto *run_control = aicore_scheduler_run_control(bootstrap_context);
     if (scheduler_state_base == nullptr || run_control == nullptr) return -1;
@@ -296,7 +297,7 @@ int32_t AicoreLifecycle::post_handshake_init(Runtime *runtime) {
         run_control->bootstrap_complete = 1;
     }
 
-    const uint64_t topology_complete_cycles = trace_enabled ? get_sys_cnt_aicpu() : 0;
+    const uint64_t topology_complete_cycles = record_lifecycle_timing ? get_sys_cnt_aicpu() : 0;
     for (int32_t i = 0; i < core_count_; ++i)
         lifecycle_traces[i].topology_complete_cycles = topology_complete_cycles;
 
@@ -325,7 +326,7 @@ void AicoreLifecycle::publish_context_partition(Runtime *runtime, int32_t thread
     }
     if (hi > lo) cache_flush_range(&handshakes[lo], static_cast<size_t>(hi - lo) * sizeof(Handshake));
     wmb();
-    const uint64_t publish_complete_cycles = is_chip_swimlane_enabled() ? get_sys_cnt_aicpu() : 0;
+    const uint64_t publish_complete_cycles = lifecycle_timing_enabled() ? get_sys_cnt_aicpu() : 0;
     for (int32_t i = lo; i < hi; ++i) {
         if (cores_[i].trace != nullptr) cores_[i].trace->context_publish_complete_cycles = publish_complete_cycles;
     }
@@ -337,8 +338,8 @@ int32_t AicoreLifecycle::wait_bootstrap_complete(Runtime *runtime) {
     cache_invalidate_range(context, 128);
     auto *run_control = aicore_scheduler_run_control(context);
     if (run_control == nullptr) return -1;
-    const bool trace_enabled = is_chip_swimlane_enabled();
-    const uint64_t wait_start_cycles = trace_enabled ? get_sys_cnt_aicpu() : 0;
+    const bool record_lifecycle_timing = lifecycle_timing_enabled();
+    const uint64_t wait_start_cycles = record_lifecycle_timing ? get_sys_cnt_aicpu() : 0;
     const uint64_t watchdog_start = get_sys_cnt_aicpu();
     const uint64_t timeout_cycles = resident_scheduler_timeout_cycles();
     uint32_t error_poll_count = 0;
@@ -361,7 +362,7 @@ int32_t AicoreLifecycle::wait_bootstrap_complete(Runtime *runtime) {
         }
         SPIN_WAIT_HINT();
     }
-    const uint64_t complete_cycles = trace_enabled ? get_sys_cnt_aicpu() : 0;
+    const uint64_t complete_cycles = record_lifecycle_timing ? get_sys_cnt_aicpu() : 0;
     for (int32_t i = 0; i < core_count_; ++i) {
         if (cores_[i].trace == nullptr) continue;
         cores_[i].trace->bootstrap_wait_start_cycles = wait_start_cycles;

src/a5/runtime/host_build_graph/docs/profiling_levels.md

@@ -8,11 +8,9 @@ The runtime uses a hierarchical profiling system with compile-time macros to con
 
 > **A5 HBG scheduler selection.** Diagnostic flags never select the scheduler.
 > Ordinary DAGs remain on the A5 HBG AICore Scheduler, while Graph replay
-> remains on its explicit legacy compatibility path. Until AICore Scheduler
-> profiling lands, chip-swimlane, PMU, and argument-dump collection for
-> ordinary DAGs is best-effort; artifacts may be absent or incomplete and must
-> not be used as evidence of AICore Scheduler behavior or as a profiling-on
-> performance baseline.
+> remains on its explicit AICPU compatibility path. Both producers export the
+> same `scheduler_records` schema; stream metadata identifies `producer` so
+> tools never apply AICPU scheduling assumptions to AICore intervals.
 > **host_build_graph (host-orch) note.** The profiling **macros** below
 > (`SIMPLER_DFX`, `SIMPLER_ORCH_PROFILING`, …) are shared with
 > `tensormap_and_ringbuffer`. But the orchestrator-timing **device-log lines**
@@ -173,7 +171,7 @@ Thread X: Scheduler summary: total_time=XXXus, loops=XXX, tasks_scheduled=XXX
 ```
 
 Per-thread fanout / fanin edge counts and ready-queue pop hit / miss
-stats live in `aicpu_scheduler_phases[]` (in `chip_swimlane_records.json`
+stats live in `scheduler_records.streams[]` (in `chip_swimlane_records.json`
 captured at chip_swimlane_level >= 3) and `deps.json`; consume them via
 `simpler_setup/tools/sched_overhead_analysis.py`.
 
@@ -415,13 +413,16 @@ mirrors the PMU pattern — two independent channels (one binary, one int):
 
 - **Binary on/off** — `KernelArgs::enable_profiling_flag` bit1
   (`SIMPLER_DFX_FLAG_CHIP_SWIMLANE`). Set by the host whenever level > 0; read
-  by AICore (which only needs on/off to decide whether to write timing) and
-  by AICPU kernel entry via `set_chip_swimlane_enabled(bool)`.
+  by AICore for the initial profiler activation and by AICPU kernel entry via
+  `set_chip_swimlane_enabled(bool)`.
 - **Granular level (0–4)** — `ChipSwimlaneDataHeader::chip_swimlane_level`
   (shared memory). Host writes it in `ChipSwimlaneCollector::initialize`; AICPU
   promotes it from the header in `chip_swimlane_aicpu_init` and exposes it via
   `get_chip_swimlane_level()` (typed `ChipSwimlaneLevel`) for
-  `>= SCHEDULE_TIMING / SCHED_PHASES / ORCH_PHASES` gates.
+  `>= SCHEDULE_TIMING / SCHED_PHASES / ORCH_PHASES` gates. The A5 HBG host also
+  copies the level into `SchedulerRunControl`, where the AICore Scheduler reads
+  it once and gates task timing, per-task scheduling timing, and phase timing
+  independently.
 
 On sim, the binary on/off travels via the dlsym'd `set_chip_swimlane_enabled`
 entry point; the granular level still goes through the shared-memory
@@ -435,6 +436,23 @@ header just like on onboard.
 | 3 | + Scheduler phases (`SCHED_*`) |
 | 4 | + Orchestrator phases (full) |
 
+The A5 HBG AICore Scheduler records the same per-task timing contract as the
+AICPU Scheduler: `dispatch_time` is the end of dispatch publication and
+`finish_time` is when the Scheduler starts processing the completion. The raw
+`scheduler_tasks.producer` field identifies which Scheduler produced these
+timestamps. At level 2 and above, A5 HBG also exports AICPU lifecycle timestamps
+for handshake, topology/configuration, context publication, bootstrap wait,
+register release, and exit; these are supplemental control-plane records.
+
+At level 3 and above, A5 HBG AICore Scheduler task intervals reuse the per-task
+trace. Consecutive taskless scheduler-loop iterations are coalesced into one
+`idle` interval and stored in one fixed-capacity, no-wrap buffer per Scheduler;
+overflow increments `capture.dropped` and sets `capture.truncated`. Coalescing
+keeps capture size dependent on idle-to-active transitions instead of Host CPU
+speed in simulation. The buffer is not allocated below level 3. Interval
+endpoints are captured at operation entry and exit; no interval is synthesized
+from aggregate durations.
+
 At level 1 the AICore record carries the full `task_token_raw`
 (a `TaskId::raw`; see `src/common/host_build_graph/task_id.h`), read straight from
 `LocalContext.async_ctx.task_token.raw` inside the AICore helper —