diff --git a/docs/testing.md b/docs/testing.md index 18fa623d7e..b1fec489c7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -634,6 +634,28 @@ Key fields: - `runtime`: which runtime to use - `CALLABLE.orchestration.source` / `CALLABLE.incores[].source`: paths relative to the test file +### Output Validation + +By default every output tensor is compared against `compute_golden` with +`torch.allclose` at the class `RTOL` / `ATOL`. Two hooks cover the cases that +don't fit: + +- **`CASES[].skip_golden`** (defaults to the class attribute `SKIP_GOLDEN`, + itself `False`) — the case has no host-computable expected output and passes + by running clean; `compute_golden` is never called for it. Use it when the + output depends on a shape only the device knows, such as the SPMD grid width + the runtime resolved. `--skip-golden` on the command line forces this on for + every case (benchmark mode). +- **`compare_outputs(self, test_args, golden_args, output_names, params)`** — + override to compare a subset of the outputs, apply a per-tensor tolerance, or + derive the valid extent from the output itself (a task that writes one slot + per SPMD block reports the grid width it actually ran with, and only that + prefix is defined). Call `super().compare_outputs(...)` with the names you + still want checked the default way. + +Prefer a `compare_outputs` override to `skip_golden`: a self-describing output +still gets checked, whereas a skipped case only proves the run didn't crash. + ### Case Selection and Manual Cases `--case` is repeatable and accepts compound forms (useful when a test file declares multiple `SceneTestCase` classes): diff --git a/examples/a2a3/tensormap_and_ringbuffer/merge_pipeline_barrier/test_merge_pipeline_barrier.py b/examples/a2a3/tensormap_and_ringbuffer/merge_pipeline_barrier/test_merge_pipeline_barrier.py index 0a5c3e6554..4f5b8696ca 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/merge_pipeline_barrier/test_merge_pipeline_barrier.py +++ b/examples/a2a3/tensormap_and_ringbuffer/merge_pipeline_barrier/test_merge_pipeline_barrier.py @@ -18,16 +18,14 @@ The barrier has two compile-time variants (MERGE_BARRIER_COUNTER in merge_pipeline.cpp): a per-slot st_dev poll (O(n) non-cacheable reads) and a single-counter st_atomic+dcci arrival + one-ld_dev poll (O(1) read). The kernel -records each block's per-barrier gap with get_sys_cnt; the hook below prints -them (ticks at 50 MHz -> us) and skips golden-checking the diagnostic output. +records each block's per-barrier gap with get_sys_cnt; the `compare_outputs` +override prints them (ticks at 50 MHz -> us) and leaves the diagnostic 'timing' +output out of the golden comparison. """ -import sys - import torch from simpler.task_interface import ArgDirection as D -import simpler_setup.scene_test # noqa: F401 from simpler_setup import SceneTestCase, TaskArgsBuilder, Tensor, scene_test TILES = 8 @@ -35,32 +33,6 @@ SIZE = TILES * M # 1024 NBLK = 8 -# Print the kernel's per-block timing (get_sys_cnt deltas, ticks->us at 50 MHz) -# and skip golden-checking the diagnostic 'timing' output. Each block's slots are -# strided by 32 int32 (one cacheline) because scalar st_dev only commits the -# first 8 int32 of a 128B line. -_st = sys.modules["simpler_setup.scene_test"] -_orig_cmp = _st._compare_outputs - - -def _cmp_with_gaps(test_args, golden_args, output_names, rtol, atol): - if "timing" in output_names: - tm = test_args.timing.reshape(NBLK, 32)[:, :6].to(torch.float64) / 50.0 - labels = ["segA", "bar1", "segB", "bar2", "segC", "total"] - print("[GAP] per-block us (50 MHz counter):") - print(" blk " + " ".join(f"{lbl:>7}" for lbl in labels)) - for b in range(NBLK): - print(f" {b:>3} " + " ".join(f"{tm[b, k]:7.2f}" for k in range(6))) - print( - f"[GAP] barrier1: max={tm[:, 1].max():.2f} min={tm[:, 1].min():.2f} us | " - f"barrier2: max={tm[:, 3].max():.2f} min={tm[:, 3].min():.2f} us" - ) - output_names = [n for n in output_names if n != "timing"] - return _orig_cmp(test_args, golden_args, output_names, rtol, atol) - - -_st._compare_outputs = _cmp_with_gaps - @scene_test(level=2, runtime="tensormap_and_ringbuffer") class TestMergePipelineBarrier(SceneTestCase): @@ -105,6 +77,23 @@ def generate_args(self, params): def compute_golden(self, args, params): args.out[:] = 2.0 * args.x + 3.0 + def compare_outputs(self, test_args, golden_args, output_names, params): + # 'timing' is a diagnostic output with no golden: print the per-block + # gaps, then compare everything else. Each block's slots are strided by + # 32 int32 (one cacheline) because scalar st_dev only commits the first + # 8 int32 of a 128B line. + tm = test_args.timing.reshape(NBLK, 32)[:, :6].to(torch.float64) / 50.0 + labels = ["segA", "bar1", "segB", "bar2", "segC", "total"] + print("[GAP] per-block us (50 MHz counter):") + print(" blk " + " ".join(f"{lbl:>7}" for lbl in labels)) + for b in range(NBLK): + print(f" {b:>3} " + " ".join(f"{tm[b, k]:7.2f}" for k in range(6))) + print( + f"[GAP] barrier1: max={tm[:, 1].max():.2f} min={tm[:, 1].min():.2f} us | " + f"barrier2: max={tm[:, 3].max():.2f} min={tm[:, 3].min():.2f} us" + ) + super().compare_outputs(test_args, golden_args, [n for n in output_names if n != "timing"], params) + if __name__ == "__main__": SceneTestCase.run_module(__name__) diff --git a/simpler_setup/scene_test.py b/simpler_setup/scene_test.py index 216a1671ef..81c9218f8d 100644 --- a/simpler_setup/scene_test.py +++ b/simpler_setup/scene_test.py @@ -981,6 +981,13 @@ class SceneTestCase: CASES: list[dict] = [] RTOL: float = 1e-5 ATOL: float = 1e-5 + # Class-wide default for `CASES[*]["skip_golden"]`. True means the case has + # no host-computable expected output — it passes by running clean, and + # `compute_golden` is never called. Use it for cases whose output depends on + # a shape only the device knows (e.g. the SPMD grid width the runtime + # resolved). Prefer a `compare_outputs` override when the output is + # self-describing enough to still be checked. + SKIP_GOLDEN: bool = False RUNTIME_ENV: dict = {} def generate_args(self, params) -> TaskArgsBuilder: @@ -991,6 +998,17 @@ def compute_golden(self, args: TaskArgsBuilder, params) -> None: """Compute expected outputs in-place on a cloned TaskArgsBuilder.""" raise NotImplementedError + def compare_outputs(self, test_args, golden_args, output_names, params) -> None: + """Assert the run's outputs match golden. Raises AssertionError on mismatch. + + The default is `torch.allclose` on every output at the class RTOL/ATOL. + Override to compare a subset, to apply a per-tensor tolerance, or when + the valid extent of an output is carried in the output itself (a task + that writes one slot per SPMD block reports the grid width it actually + ran with, and only that prefix is defined). + """ + _compare_outputs(test_args, golden_args, output_names, self.RTOL, self.ATOL) + # ------------------------------------------------------------------ # Callable compilation # ------------------------------------------------------------------ @@ -1168,6 +1186,7 @@ def _run_and_validate_l2( # noqa: PLR0913 -- threads CLI diagnostic flags + cas ): params = case.get("params", {}) config_dict = case.get("config", {}) + skip_golden = skip_golden or bool(case.get("skip_golden", self.SKIP_GOLDEN)) orch_sig = self.CALLABLE.get("orchestration", {}).get("signature", []) # The L2 entry point is `Worker.run(handle, args, cfg)`. Reuse the @@ -1223,7 +1242,7 @@ def _run_and_validate_l2( # noqa: PLR0913 -- threads CLI diagnostic flags + cas worker.run(handle, chip_args, config=config) if not skip_golden: - _compare_outputs(test_args, golden_args, output_names, self.RTOL, self.ATOL) + self.compare_outputs(test_args, golden_args, output_names, params) def _run_and_validate_l3( # noqa: PLR0913 -- threads CLI diagnostic flags + L3 ns context self, @@ -1254,6 +1273,7 @@ def _run_and_validate_l3( # noqa: PLR0913 -- threads CLI diagnostic flags + L3 params = case.get("params", {}) config_dict = case.get("config", {}) + skip_golden = skip_golden or bool(case.get("skip_golden", self.SKIP_GOLDEN)) # Build args test_args = self.generate_args(params) @@ -1314,7 +1334,7 @@ def task_orch(orch, _args, _cfg, _ns=ns, _test_args=test_args, _config=config): worker.run(task_orch) if not skip_golden: - _compare_outputs(test_args, golden_args, all_tensor_names, self.RTOL, self.ATOL) + self.compare_outputs(test_args, golden_args, all_tensor_names, params) finally: rehosted.release() diff --git a/tests/ut/py/test_scene_test_golden_hooks.py b/tests/ut/py/test_scene_test_golden_hooks.py new file mode 100644 index 0000000000..da01d1b7bf --- /dev/null +++ b/tests/ut/py/test_scene_test_golden_hooks.py @@ -0,0 +1,137 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +# ruff: noqa: PLC0415 +"""Focused UT for the golden-validation hooks in scene_test.py. + +Covers the two escapes from the default "allclose every output against +compute_golden" contract: per-case `skip_golden` and a `compare_outputs` +override. +""" + +from __future__ import annotations + +import sys +from unittest.mock import MagicMock, patch + +import torch + +from simpler_setup import SceneTestCase, TaskArgsBuilder, Tensor + +_SCENE_TEST_MOD = sys.modules["simpler_setup.scene_test"] + +_OUTPUT_NAMES = ["y"] + + +class _Recorder(SceneTestCase): + """Level-2 case that records which validation hooks fired.""" + + CALLABLE = {"orchestration": {"signature": []}} + CASES = [] + + def __init__(self): + self.golden_calls = 0 + self.compare_calls = [] + + def generate_args(self, params): + return TaskArgsBuilder(Tensor("y", torch.zeros(4, dtype=torch.float32))) + + def compute_golden(self, args, params): + self.golden_calls += 1 + + def compare_outputs(self, test_args, golden_args, output_names, params): + self.compare_calls.append(list(output_names)) + + +def _drive(inst, case, *, cli_skip=False): + """Run one case through the L2 path with the device-touching steps stubbed.""" + inst._st_level = 2 + with ( + patch.object(_SCENE_TEST_MOD, "_build_chip_task_args", return_value=(object(), list(_OUTPUT_NAMES))), + patch.object(type(inst), "_build_config", return_value=object()), + ): + inst._run_and_validate_l2(MagicMock(), object(), case, skip_golden=cli_skip) + + +def test_default_case_computes_and_compares(): + inst = _Recorder() + _drive(inst, {"name": "c"}) + assert inst.golden_calls == 1 + assert inst.compare_calls == [_OUTPUT_NAMES] + + +def test_per_case_skip_golden_bypasses_both_hooks(): + inst = _Recorder() + _drive(inst, {"name": "c", "skip_golden": True}) + assert inst.golden_calls == 0 + assert inst.compare_calls == [] + + +def test_class_attribute_supplies_the_default(): + class _AllSkipped(_Recorder): + SKIP_GOLDEN = True + + inst = _AllSkipped() + _drive(inst, {"name": "c"}) + assert inst.golden_calls == 0 + assert inst.compare_calls == [] + + +def test_per_case_value_overrides_the_class_attribute(): + class _AllSkipped(_Recorder): + SKIP_GOLDEN = True + + inst = _AllSkipped() + _drive(inst, {"name": "c", "skip_golden": False}) + assert inst.golden_calls == 1 + assert inst.compare_calls == [_OUTPUT_NAMES] + + +def test_cli_skip_golden_wins_over_an_opted_in_case(): + inst = _Recorder() + _drive(inst, {"name": "c", "skip_golden": False}, cli_skip=True) + assert inst.golden_calls == 0 + assert inst.compare_calls == [] + + +def test_compare_outputs_override_replaces_the_default_allclose(): + """A mismatching output passes when the override drops it from the comparison.""" + + class _DropsY(SceneTestCase): + CALLABLE = {"orchestration": {"signature": []}} + CASES = [] + + def generate_args(self, params): + return TaskArgsBuilder(Tensor("y", torch.zeros(4, dtype=torch.float32))) + + def compute_golden(self, args, params): + args.y[:] = 1.0 # deliberately unequal to the un-run test tensor + + def compare_outputs(self, test_args, golden_args, output_names, params): + super().compare_outputs(test_args, golden_args, [n for n in output_names if n != "y"], params) + + _drive(_DropsY(), {"name": "c"}) + + +def test_default_compare_outputs_raises_on_mismatch(): + """Without the override, the same mismatch is caught.""" + + class _Strict(SceneTestCase): + CALLABLE = {"orchestration": {"signature": []}} + CASES = [] + + def generate_args(self, params): + return TaskArgsBuilder(Tensor("y", torch.zeros(4, dtype=torch.float32))) + + def compute_golden(self, args, params): + args.y[:] = 1.0 + + import pytest + + with pytest.raises(AssertionError, match="Golden mismatch on 'y'"): + _drive(_Strict(), {"name": "c"})