Add: scene-test hooks for per-case skip_golden and custom compare

docs/testing.md

@@ -634,6 +634,28 @@ Key fields:
 - `runtime`: which runtime to use
 - `CALLABLE.orchestration.source` / `CALLABLE.incores[].source`: paths relative to the test file
 
+### Output Validation
+
+By default every output tensor is compared against `compute_golden` with
+`torch.allclose` at the class `RTOL` / `ATOL`. Two hooks cover the cases that
+don't fit:
+
+- **`CASES[].skip_golden`** (defaults to the class attribute `SKIP_GOLDEN`,
+  itself `False`) — the case has no host-computable expected output and passes
+  by running clean; `compute_golden` is never called for it. Use it when the
+  output depends on a shape only the device knows, such as the SPMD grid width
+  the runtime resolved. `--skip-golden` on the command line forces this on for
+  every case (benchmark mode).
+- **`compare_outputs(self, test_args, golden_args, output_names, params)`** —
+  override to compare a subset of the outputs, apply a per-tensor tolerance, or
+  derive the valid extent from the output itself (a task that writes one slot
+  per SPMD block reports the grid width it actually ran with, and only that
+  prefix is defined). Call `super().compare_outputs(...)` with the names you
+  still want checked the default way.
+
+Prefer a `compare_outputs` override to `skip_golden`: a self-describing output
+still gets checked, whereas a skipped case only proves the run didn't crash.
+
 ### Case Selection and Manual Cases
 
 `--case` is repeatable and accepts compound forms (useful when a test file declares multiple `SceneTestCase` classes):

examples/a2a3/tensormap_and_ringbuffer/merge_pipeline_barrier/test_merge_pipeline_barrier.py

@@ -18,49 +18,21 @@
 The barrier has two compile-time variants (MERGE_BARRIER_COUNTER in
 merge_pipeline.cpp): a per-slot st_dev poll (O(n) non-cacheable reads) and a
 single-counter st_atomic+dcci arrival + one-ld_dev poll (O(1) read). The kernel
-records each block's per-barrier gap with get_sys_cnt; the hook below prints
-them (ticks at 50 MHz -> us) and skips golden-checking the diagnostic output.
+records each block's per-barrier gap with get_sys_cnt; the `compare_outputs`
+override prints them (ticks at 50 MHz -> us) and leaves the diagnostic 'timing'
+output out of the golden comparison.
 """
 
-import sys
-
 import torch
 from simpler.task_interface import ArgDirection as D
 
-import simpler_setup.scene_test  # noqa: F401
 from simpler_setup import SceneTestCase, TaskArgsBuilder, Tensor, scene_test
 
 TILES = 8
 M = 128
 SIZE = TILES * M  # 1024
 NBLK = 8
 
-# Print the kernel's per-block timing (get_sys_cnt deltas, ticks->us at 50 MHz)
-# and skip golden-checking the diagnostic 'timing' output. Each block's slots are
-# strided by 32 int32 (one cacheline) because scalar st_dev only commits the
-# first 8 int32 of a 128B line.
-_st = sys.modules["simpler_setup.scene_test"]
-_orig_cmp = _st._compare_outputs
-
-
-def _cmp_with_gaps(test_args, golden_args, output_names, rtol, atol):
-    if "timing" in output_names:
-        tm = test_args.timing.reshape(NBLK, 32)[:, :6].to(torch.float64) / 50.0
-        labels = ["segA", "bar1", "segB", "bar2", "segC", "total"]
-        print("[GAP] per-block us (50 MHz counter):")
-        print("  blk " + " ".join(f"{lbl:>7}" for lbl in labels))
-        for b in range(NBLK):
-            print(f"  {b:>3} " + " ".join(f"{tm[b, k]:7.2f}" for k in range(6)))
-        print(
-            f"[GAP] barrier1: max={tm[:, 1].max():.2f} min={tm[:, 1].min():.2f} us | "
-            f"barrier2: max={tm[:, 3].max():.2f} min={tm[:, 3].min():.2f} us"
-        )
-        output_names = [n for n in output_names if n != "timing"]
-    return _orig_cmp(test_args, golden_args, output_names, rtol, atol)
-
-
-_st._compare_outputs = _cmp_with_gaps
-
 
 @scene_test(level=2, runtime="tensormap_and_ringbuffer")
 class TestMergePipelineBarrier(SceneTestCase):
@@ -105,6 +77,23 @@ def generate_args(self, params):
     def compute_golden(self, args, params):
         args.out[:] = 2.0 * args.x + 3.0
 
+    def compare_outputs(self, test_args, golden_args, output_names, params):
+        # 'timing' is a diagnostic output with no golden: print the per-block
+        # gaps, then compare everything else. Each block's slots are strided by
+        # 32 int32 (one cacheline) because scalar st_dev only commits the first
+        # 8 int32 of a 128B line.
+        tm = test_args.timing.reshape(NBLK, 32)[:, :6].to(torch.float64) / 50.0
+        labels = ["segA", "bar1", "segB", "bar2", "segC", "total"]
+        print("[GAP] per-block us (50 MHz counter):")
+        print("  blk " + " ".join(f"{lbl:>7}" for lbl in labels))
+        for b in range(NBLK):
+            print(f"  {b:>3} " + " ".join(f"{tm[b, k]:7.2f}" for k in range(6)))
+        print(
+            f"[GAP] barrier1: max={tm[:, 1].max():.2f} min={tm[:, 1].min():.2f} us | "
+            f"barrier2: max={tm[:, 3].max():.2f} min={tm[:, 3].min():.2f} us"
+        )
+        super().compare_outputs(test_args, golden_args, [n for n in output_names if n != "timing"], params)
+
 
 if __name__ == "__main__":
     SceneTestCase.run_module(__name__)

simpler_setup/scene_test.py

@@ -981,6 +981,13 @@ class SceneTestCase:
     CASES: list[dict] = []
     RTOL: float = 1e-5
     ATOL: float = 1e-5
+    # Class-wide default for `CASES[*]["skip_golden"]`. True means the case has
+    # no host-computable expected output — it passes by running clean, and
+    # `compute_golden` is never called. Use it for cases whose output depends on
+    # a shape only the device knows (e.g. the SPMD grid width the runtime
+    # resolved). Prefer a `compare_outputs` override when the output is
+    # self-describing enough to still be checked.
+    SKIP_GOLDEN: bool = False
     RUNTIME_ENV: dict = {}
 
     def generate_args(self, params) -> TaskArgsBuilder:
@@ -991,6 +998,17 @@ def compute_golden(self, args: TaskArgsBuilder, params) -> None:
         """Compute expected outputs in-place on a cloned TaskArgsBuilder."""
         raise NotImplementedError
 
+    def compare_outputs(self, test_args, golden_args, output_names, params) -> None:
+        """Assert the run's outputs match golden. Raises AssertionError on mismatch.
+
+        The default is `torch.allclose` on every output at the class RTOL/ATOL.
+        Override to compare a subset, to apply a per-tensor tolerance, or when
+        the valid extent of an output is carried in the output itself (a task
+        that writes one slot per SPMD block reports the grid width it actually
+        ran with, and only that prefix is defined).
+        """
+        _compare_outputs(test_args, golden_args, output_names, self.RTOL, self.ATOL)
+
     # ------------------------------------------------------------------
     # Callable compilation
     # ------------------------------------------------------------------
@@ -1168,6 +1186,7 @@ def _run_and_validate_l2(  # noqa: PLR0913 -- threads CLI diagnostic flags + cas
     ):
         params = case.get("params", {})
         config_dict = case.get("config", {})
+        skip_golden = skip_golden or bool(case.get("skip_golden", self.SKIP_GOLDEN))
         orch_sig = self.CALLABLE.get("orchestration", {}).get("signature", [])
 
         # The L2 entry point is `Worker.run(handle, args, cfg)`. Reuse the
@@ -1223,7 +1242,7 @@ def _run_and_validate_l2(  # noqa: PLR0913 -- threads CLI diagnostic flags + cas
                 worker.run(handle, chip_args, config=config)
 
             if not skip_golden:
-                _compare_outputs(test_args, golden_args, output_names, self.RTOL, self.ATOL)
+                self.compare_outputs(test_args, golden_args, output_names, params)
 
     def _run_and_validate_l3(  # noqa: PLR0913 -- threads CLI diagnostic flags + L3 ns context
         self,
@@ -1254,6 +1273,7 @@ def _run_and_validate_l3(  # noqa: PLR0913 -- threads CLI diagnostic flags + L3
 
         params = case.get("params", {})
         config_dict = case.get("config", {})
+        skip_golden = skip_golden or bool(case.get("skip_golden", self.SKIP_GOLDEN))
 
         # Build args
         test_args = self.generate_args(params)
@@ -1314,7 +1334,7 @@ def task_orch(orch, _args, _cfg, _ns=ns, _test_args=test_args, _config=config):
                     worker.run(task_orch)
 
                 if not skip_golden:
-                    _compare_outputs(test_args, golden_args, all_tensor_names, self.RTOL, self.ATOL)
+                    self.compare_outputs(test_args, golden_args, all_tensor_names, params)
         finally:
             rehosted.release()
 

tests/ut/py/test_scene_test_golden_hooks.py

@@ -0,0 +1,137 @@
+# Copyright (c) PyPTO Contributors.
+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+# CANN Open Software License Agreement Version 2.0 (the "License").
+# Please refer to the License for details. You may not use this file except in compliance with the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE in the root of the software repository for the full text of the License.
+# -----------------------------------------------------------------------------------------------------------
+# ruff: noqa: PLC0415
+"""Focused UT for the golden-validation hooks in scene_test.py.
+
+Covers the two escapes from the default "allclose every output against
+compute_golden" contract: per-case `skip_golden` and a `compare_outputs`
+override.
+"""
+
+from __future__ import annotations
+
+import sys
+from unittest.mock import MagicMock, patch
+
+import torch
+
+from simpler_setup import SceneTestCase, TaskArgsBuilder, Tensor
+
+_SCENE_TEST_MOD = sys.modules["simpler_setup.scene_test"]
+
+_OUTPUT_NAMES = ["y"]
+
+
+class _Recorder(SceneTestCase):
+    """Level-2 case that records which validation hooks fired."""
+
+    CALLABLE = {"orchestration": {"signature": []}}
+    CASES = []
+
+    def __init__(self):
+        self.golden_calls = 0
+        self.compare_calls = []
+
+    def generate_args(self, params):
+        return TaskArgsBuilder(Tensor("y", torch.zeros(4, dtype=torch.float32)))
+
+    def compute_golden(self, args, params):
+        self.golden_calls += 1
+
+    def compare_outputs(self, test_args, golden_args, output_names, params):
+        self.compare_calls.append(list(output_names))
+
+
+def _drive(inst, case, *, cli_skip=False):
+    """Run one case through the L2 path with the device-touching steps stubbed."""
+    inst._st_level = 2
+    with (
+        patch.object(_SCENE_TEST_MOD, "_build_chip_task_args", return_value=(object(), list(_OUTPUT_NAMES))),
+        patch.object(type(inst), "_build_config", return_value=object()),
+    ):
+        inst._run_and_validate_l2(MagicMock(), object(), case, skip_golden=cli_skip)
+
+
+def test_default_case_computes_and_compares():
+    inst = _Recorder()
+    _drive(inst, {"name": "c"})
+    assert inst.golden_calls == 1
+    assert inst.compare_calls == [_OUTPUT_NAMES]
+
+
+def test_per_case_skip_golden_bypasses_both_hooks():
+    inst = _Recorder()
+    _drive(inst, {"name": "c", "skip_golden": True})
+    assert inst.golden_calls == 0
+    assert inst.compare_calls == []
+
+
+def test_class_attribute_supplies_the_default():
+    class _AllSkipped(_Recorder):
+        SKIP_GOLDEN = True
+
+    inst = _AllSkipped()
+    _drive(inst, {"name": "c"})
+    assert inst.golden_calls == 0
+    assert inst.compare_calls == []
+
+
+def test_per_case_value_overrides_the_class_attribute():
+    class _AllSkipped(_Recorder):
+        SKIP_GOLDEN = True
+
+    inst = _AllSkipped()
+    _drive(inst, {"name": "c", "skip_golden": False})
+    assert inst.golden_calls == 1
+    assert inst.compare_calls == [_OUTPUT_NAMES]
+
+
+def test_cli_skip_golden_wins_over_an_opted_in_case():
+    inst = _Recorder()
+    _drive(inst, {"name": "c", "skip_golden": False}, cli_skip=True)
+    assert inst.golden_calls == 0
+    assert inst.compare_calls == []
+
+
+def test_compare_outputs_override_replaces_the_default_allclose():
+    """A mismatching output passes when the override drops it from the comparison."""
+
+    class _DropsY(SceneTestCase):
+        CALLABLE = {"orchestration": {"signature": []}}
+        CASES = []
+
+        def generate_args(self, params):
+            return TaskArgsBuilder(Tensor("y", torch.zeros(4, dtype=torch.float32)))
+
+        def compute_golden(self, args, params):
+            args.y[:] = 1.0  # deliberately unequal to the un-run test tensor
+
+        def compare_outputs(self, test_args, golden_args, output_names, params):
+            super().compare_outputs(test_args, golden_args, [n for n in output_names if n != "y"], params)
+
+    _drive(_DropsY(), {"name": "c"})
+
+
+def test_default_compare_outputs_raises_on_mismatch():
+    """Without the override, the same mismatch is caught."""
+
+    class _Strict(SceneTestCase):
+        CALLABLE = {"orchestration": {"signature": []}}
+        CASES = []
+
+        def generate_args(self, params):
+            return TaskArgsBuilder(Tensor("y", torch.zeros(4, dtype=torch.float32)))
+
+        def compute_golden(self, args, params):
+            args.y[:] = 1.0
+
+    import pytest
+
+    with pytest.raises(AssertionError, match="Golden mismatch on 'y'"):
+        _drive(_Strict(), {"name": "c"})