From a72a32612204252bb0c04603915d225cbb05623f Mon Sep 17 00:00:00 2001 From: Chao Wang <26245345+ChaoWao@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:01:45 -0700 Subject: [PATCH] Add: scene-test hooks for per-case skip_golden and custom compare The framework compared every output against compute_golden with a fixed torch.allclose, and the only escape was the global --skip-golden flag. Outputs whose valid extent is decided by the device (a task that writes one slot per SPMD block only defines the prefix it actually ran) had no way to be checked, and merge_pipeline_barrier reached for a monkey-patch of the private _compare_outputs to drop its diagnostic tensor. - CASES[*]["skip_golden"], defaulting to the class attribute SKIP_GOLDEN, marks a case as having no host-computable golden; compute_golden is then never called for it. The CLI flag still forces it on everywhere. - compare_outputs() is overridable, so a case can compare a subset, use a per-tensor tolerance, or read the valid extent out of the output. _compare_outputs keeps its signature for the tests that import it. - merge_pipeline_barrier uses the override instead of patching module internals. Its 'timing' golden is all zeros while the run writes real counter values, so the case passing is itself proof the override runs. --- docs/testing.md | 22 +++ .../test_merge_pipeline_barrier.py | 51 +++---- simpler_setup/scene_test.py | 24 ++- tests/ut/py/test_scene_test_golden_hooks.py | 137 ++++++++++++++++++ 4 files changed, 201 insertions(+), 33 deletions(-) create mode 100644 tests/ut/py/test_scene_test_golden_hooks.py diff --git a/docs/testing.md b/docs/testing.md index 18fa623d7e..b1fec489c7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -634,6 +634,28 @@ Key fields: - `runtime`: which runtime to use - `CALLABLE.orchestration.source` / `CALLABLE.incores[].source`: paths relative to the test file +### Output Validation + +By default every output tensor is compared against `compute_golden` with +`torch.allclose` at the class `RTOL` / `ATOL`. Two hooks cover the cases that +don't fit: + +- **`CASES[].skip_golden`** (defaults to the class attribute `SKIP_GOLDEN`, + itself `False`) — the case has no host-computable expected output and passes + by running clean; `compute_golden` is never called for it. Use it when the + output depends on a shape only the device knows, such as the SPMD grid width + the runtime resolved. `--skip-golden` on the command line forces this on for + every case (benchmark mode). +- **`compare_outputs(self, test_args, golden_args, output_names, params)`** — + override to compare a subset of the outputs, apply a per-tensor tolerance, or + derive the valid extent from the output itself (a task that writes one slot + per SPMD block reports the grid width it actually ran with, and only that + prefix is defined). Call `super().compare_outputs(...)` with the names you + still want checked the default way. + +Prefer a `compare_outputs` override to `skip_golden`: a self-describing output +still gets checked, whereas a skipped case only proves the run didn't crash. + ### Case Selection and Manual Cases `--case` is repeatable and accepts compound forms (useful when a test file declares multiple `SceneTestCase` classes): diff --git a/examples/a2a3/tensormap_and_ringbuffer/merge_pipeline_barrier/test_merge_pipeline_barrier.py b/examples/a2a3/tensormap_and_ringbuffer/merge_pipeline_barrier/test_merge_pipeline_barrier.py index 0a5c3e6554..4f5b8696ca 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/merge_pipeline_barrier/test_merge_pipeline_barrier.py +++ b/examples/a2a3/tensormap_and_ringbuffer/merge_pipeline_barrier/test_merge_pipeline_barrier.py @@ -18,16 +18,14 @@ The barrier has two compile-time variants (MERGE_BARRIER_COUNTER in merge_pipeline.cpp): a per-slot st_dev poll (O(n) non-cacheable reads) and a single-counter st_atomic+dcci arrival + one-ld_dev poll (O(1) read). The kernel -records each block's per-barrier gap with get_sys_cnt; the hook below prints -them (ticks at 50 MHz -> us) and skips golden-checking the diagnostic output. +records each block's per-barrier gap with get_sys_cnt; the `compare_outputs` +override prints them (ticks at 50 MHz -> us) and leaves the diagnostic 'timing' +output out of the golden comparison. """ -import sys - import torch from simpler.task_interface import ArgDirection as D -import simpler_setup.scene_test # noqa: F401 from simpler_setup import SceneTestCase, TaskArgsBuilder, Tensor, scene_test TILES = 8 @@ -35,32 +33,6 @@ SIZE = TILES * M # 1024 NBLK = 8 -# Print the kernel's per-block timing (get_sys_cnt deltas, ticks->us at 50 MHz) -# and skip golden-checking the diagnostic 'timing' output. Each block's slots are -# strided by 32 int32 (one cacheline) because scalar st_dev only commits the -# first 8 int32 of a 128B line. -_st = sys.modules["simpler_setup.scene_test"] -_orig_cmp = _st._compare_outputs - - -def _cmp_with_gaps(test_args, golden_args, output_names, rtol, atol): - if "timing" in output_names: - tm = test_args.timing.reshape(NBLK, 32)[:, :6].to(torch.float64) / 50.0 - labels = ["segA", "bar1", "segB", "bar2", "segC", "total"] - print("[GAP] per-block us (50 MHz counter):") - print(" blk " + " ".join(f"{lbl:>7}" for lbl in labels)) - for b in range(NBLK): - print(f" {b:>3} " + " ".join(f"{tm[b, k]:7.2f}" for k in range(6))) - print( - f"[GAP] barrier1: max={tm[:, 1].max():.2f} min={tm[:, 1].min():.2f} us | " - f"barrier2: max={tm[:, 3].max():.2f} min={tm[:, 3].min():.2f} us" - ) - output_names = [n for n in output_names if n != "timing"] - return _orig_cmp(test_args, golden_args, output_names, rtol, atol) - - -_st._compare_outputs = _cmp_with_gaps - @scene_test(level=2, runtime="tensormap_and_ringbuffer") class TestMergePipelineBarrier(SceneTestCase): @@ -105,6 +77,23 @@ def generate_args(self, params): def compute_golden(self, args, params): args.out[:] = 2.0 * args.x + 3.0 + def compare_outputs(self, test_args, golden_args, output_names, params): + # 'timing' is a diagnostic output with no golden: print the per-block + # gaps, then compare everything else. Each block's slots are strided by + # 32 int32 (one cacheline) because scalar st_dev only commits the first + # 8 int32 of a 128B line. + tm = test_args.timing.reshape(NBLK, 32)[:, :6].to(torch.float64) / 50.0 + labels = ["segA", "bar1", "segB", "bar2", "segC", "total"] + print("[GAP] per-block us (50 MHz counter):") + print(" blk " + " ".join(f"{lbl:>7}" for lbl in labels)) + for b in range(NBLK): + print(f" {b:>3} " + " ".join(f"{tm[b, k]:7.2f}" for k in range(6))) + print( + f"[GAP] barrier1: max={tm[:, 1].max():.2f} min={tm[:, 1].min():.2f} us | " + f"barrier2: max={tm[:, 3].max():.2f} min={tm[:, 3].min():.2f} us" + ) + super().compare_outputs(test_args, golden_args, [n for n in output_names if n != "timing"], params) + if __name__ == "__main__": SceneTestCase.run_module(__name__) diff --git a/simpler_setup/scene_test.py b/simpler_setup/scene_test.py index 216a1671ef..81c9218f8d 100644 --- a/simpler_setup/scene_test.py +++ b/simpler_setup/scene_test.py @@ -981,6 +981,13 @@ class SceneTestCase: CASES: list[dict] = [] RTOL: float = 1e-5 ATOL: float = 1e-5 + # Class-wide default for `CASES[*]["skip_golden"]`. True means the case has + # no host-computable expected output — it passes by running clean, and + # `compute_golden` is never called. Use it for cases whose output depends on + # a shape only the device knows (e.g. the SPMD grid width the runtime + # resolved). Prefer a `compare_outputs` override when the output is + # self-describing enough to still be checked. + SKIP_GOLDEN: bool = False RUNTIME_ENV: dict = {} def generate_args(self, params) -> TaskArgsBuilder: @@ -991,6 +998,17 @@ def compute_golden(self, args: TaskArgsBuilder, params) -> None: """Compute expected outputs in-place on a cloned TaskArgsBuilder.""" raise NotImplementedError + def compare_outputs(self, test_args, golden_args, output_names, params) -> None: + """Assert the run's outputs match golden. Raises AssertionError on mismatch. + + The default is `torch.allclose` on every output at the class RTOL/ATOL. + Override to compare a subset, to apply a per-tensor tolerance, or when + the valid extent of an output is carried in the output itself (a task + that writes one slot per SPMD block reports the grid width it actually + ran with, and only that prefix is defined). + """ + _compare_outputs(test_args, golden_args, output_names, self.RTOL, self.ATOL) + # ------------------------------------------------------------------ # Callable compilation # ------------------------------------------------------------------ @@ -1168,6 +1186,7 @@ def _run_and_validate_l2( # noqa: PLR0913 -- threads CLI diagnostic flags + cas ): params = case.get("params", {}) config_dict = case.get("config", {}) + skip_golden = skip_golden or bool(case.get("skip_golden", self.SKIP_GOLDEN)) orch_sig = self.CALLABLE.get("orchestration", {}).get("signature", []) # The L2 entry point is `Worker.run(handle, args, cfg)`. Reuse the @@ -1223,7 +1242,7 @@ def _run_and_validate_l2( # noqa: PLR0913 -- threads CLI diagnostic flags + cas worker.run(handle, chip_args, config=config) if not skip_golden: - _compare_outputs(test_args, golden_args, output_names, self.RTOL, self.ATOL) + self.compare_outputs(test_args, golden_args, output_names, params) def _run_and_validate_l3( # noqa: PLR0913 -- threads CLI diagnostic flags + L3 ns context self, @@ -1254,6 +1273,7 @@ def _run_and_validate_l3( # noqa: PLR0913 -- threads CLI diagnostic flags + L3 params = case.get("params", {}) config_dict = case.get("config", {}) + skip_golden = skip_golden or bool(case.get("skip_golden", self.SKIP_GOLDEN)) # Build args test_args = self.generate_args(params) @@ -1314,7 +1334,7 @@ def task_orch(orch, _args, _cfg, _ns=ns, _test_args=test_args, _config=config): worker.run(task_orch) if not skip_golden: - _compare_outputs(test_args, golden_args, all_tensor_names, self.RTOL, self.ATOL) + self.compare_outputs(test_args, golden_args, all_tensor_names, params) finally: rehosted.release() diff --git a/tests/ut/py/test_scene_test_golden_hooks.py b/tests/ut/py/test_scene_test_golden_hooks.py new file mode 100644 index 0000000000..da01d1b7bf --- /dev/null +++ b/tests/ut/py/test_scene_test_golden_hooks.py @@ -0,0 +1,137 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +# ruff: noqa: PLC0415 +"""Focused UT for the golden-validation hooks in scene_test.py. + +Covers the two escapes from the default "allclose every output against +compute_golden" contract: per-case `skip_golden` and a `compare_outputs` +override. +""" + +from __future__ import annotations + +import sys +from unittest.mock import MagicMock, patch + +import torch + +from simpler_setup import SceneTestCase, TaskArgsBuilder, Tensor + +_SCENE_TEST_MOD = sys.modules["simpler_setup.scene_test"] + +_OUTPUT_NAMES = ["y"] + + +class _Recorder(SceneTestCase): + """Level-2 case that records which validation hooks fired.""" + + CALLABLE = {"orchestration": {"signature": []}} + CASES = [] + + def __init__(self): + self.golden_calls = 0 + self.compare_calls = [] + + def generate_args(self, params): + return TaskArgsBuilder(Tensor("y", torch.zeros(4, dtype=torch.float32))) + + def compute_golden(self, args, params): + self.golden_calls += 1 + + def compare_outputs(self, test_args, golden_args, output_names, params): + self.compare_calls.append(list(output_names)) + + +def _drive(inst, case, *, cli_skip=False): + """Run one case through the L2 path with the device-touching steps stubbed.""" + inst._st_level = 2 + with ( + patch.object(_SCENE_TEST_MOD, "_build_chip_task_args", return_value=(object(), list(_OUTPUT_NAMES))), + patch.object(type(inst), "_build_config", return_value=object()), + ): + inst._run_and_validate_l2(MagicMock(), object(), case, skip_golden=cli_skip) + + +def test_default_case_computes_and_compares(): + inst = _Recorder() + _drive(inst, {"name": "c"}) + assert inst.golden_calls == 1 + assert inst.compare_calls == [_OUTPUT_NAMES] + + +def test_per_case_skip_golden_bypasses_both_hooks(): + inst = _Recorder() + _drive(inst, {"name": "c", "skip_golden": True}) + assert inst.golden_calls == 0 + assert inst.compare_calls == [] + + +def test_class_attribute_supplies_the_default(): + class _AllSkipped(_Recorder): + SKIP_GOLDEN = True + + inst = _AllSkipped() + _drive(inst, {"name": "c"}) + assert inst.golden_calls == 0 + assert inst.compare_calls == [] + + +def test_per_case_value_overrides_the_class_attribute(): + class _AllSkipped(_Recorder): + SKIP_GOLDEN = True + + inst = _AllSkipped() + _drive(inst, {"name": "c", "skip_golden": False}) + assert inst.golden_calls == 1 + assert inst.compare_calls == [_OUTPUT_NAMES] + + +def test_cli_skip_golden_wins_over_an_opted_in_case(): + inst = _Recorder() + _drive(inst, {"name": "c", "skip_golden": False}, cli_skip=True) + assert inst.golden_calls == 0 + assert inst.compare_calls == [] + + +def test_compare_outputs_override_replaces_the_default_allclose(): + """A mismatching output passes when the override drops it from the comparison.""" + + class _DropsY(SceneTestCase): + CALLABLE = {"orchestration": {"signature": []}} + CASES = [] + + def generate_args(self, params): + return TaskArgsBuilder(Tensor("y", torch.zeros(4, dtype=torch.float32))) + + def compute_golden(self, args, params): + args.y[:] = 1.0 # deliberately unequal to the un-run test tensor + + def compare_outputs(self, test_args, golden_args, output_names, params): + super().compare_outputs(test_args, golden_args, [n for n in output_names if n != "y"], params) + + _drive(_DropsY(), {"name": "c"}) + + +def test_default_compare_outputs_raises_on_mismatch(): + """Without the override, the same mismatch is caught.""" + + class _Strict(SceneTestCase): + CALLABLE = {"orchestration": {"signature": []}} + CASES = [] + + def generate_args(self, params): + return TaskArgsBuilder(Tensor("y", torch.zeros(4, dtype=torch.float32))) + + def compute_golden(self, args, params): + args.y[:] = 1.0 + + import pytest + + with pytest.raises(AssertionError, match="Golden mismatch on 'y'"): + _drive(_Strict(), {"name": "c"})