fix: isolate repeatability grader evidence

Author: cursoragent

src/openenv/validation/graders/runtime/repeatability.py

@@ -110,9 +110,12 @@ def run(self, subject):
     def check(self, subject, evidence):
         problems = []
         original_seed = None
-        for index, sample in enumerate(
-            [evidence] + [replay.evidence for replay in evidence.replays]
-        ):
+        samples = [(0, evidence)] + [
+            (index, replay.evidence)
+            for index, replay in enumerate(evidence.replays, 1)
+            if replay.scope == "seed"
+        ]
+        for index, sample in samples:
             if sample.failure_reason or sample.telemetry_error:
                 problems.append(f"replay {index}: reset or telemetry collection failed")
                 continue
@@ -129,7 +132,7 @@ def check(self, subject, evidence):
                 continue
             if index == 0:
                 original_seed = seed
-            elif evidence.replays[index - 1].scope == "seed" and seed == original_seed:
+            elif seed == original_seed:
                 problems.append(f"replay {index}: scheduled seed was not changed")
             observed = _telemetry(sample).get("seed")
             if (
@@ -213,11 +216,13 @@ def _grade(self, subject, evidence):
         }
         for index, sample in enumerate(samples):
             if sample.failure_reason:
-                return (
-                    CheckStatus.FAIL,
-                    [f"replay {index}: collection failed"],
-                    measured,
-                )
+                if index == 0:
+                    return (
+                        CheckStatus.FAIL,
+                        [f"replay {index}: collection failed"],
+                        measured,
+                    )
+                continue
             if not any(row.operation == "step" for row in sample.exchanges):
                 return (
                     CheckStatus.SKIP,
@@ -229,7 +234,9 @@ def _grade(self, subject, evidence):
                 )
         judged = subject.manifest.capabilities.llm_judged
         required = JUDGED_REPLAYS if judged else 3
-        if len(samples) < required or {row.scope for row in replays} != {
+        if measured["completed_replays"] < required or {
+            row.scope for row in replays
+        } != {
             "session",
             "container",
         }:

tests/test_validation/test_runtime_repeatability.py

@@ -124,6 +124,20 @@ def test_different_seed_schedule_is_required_but_output_need_not_differ(tmp_path
     )
 
 
+def test_seed_control_ignores_unrelated_replay_failure(tmp_path):
+    subject = subject_with_replays(tmp_path)
+    replays = list(subject.runtime_evidence.replays)
+    replays[0] = replace(
+        replays[0],
+        evidence=replace(
+            replays[0].evidence, failure_reason="step failed (TimeoutError)"
+        ),
+    )
+    evidence = replace(subject.runtime_evidence, replays=tuple(replays))
+    result = SeedControlGrader().run(replace(subject, runtime_evidence=evidence))
+    assert result.status is CheckStatus.PASS
+
+
 @pytest.mark.parametrize(
     "field,value", [("trajectory", None), ("schema_version", True)]
 )
@@ -201,6 +215,25 @@ def test_partial_judged_sample_is_incomplete(tmp_path):
     assert result.measured["completed_replays"] == 19
 
 
+def test_failed_replay_is_incomplete(tmp_path):
+    subject = subject_with_replays(tmp_path)
+    replays = list(subject.runtime_evidence.replays)
+    replays[1] = replace(
+        replays[1],
+        evidence=replace(
+            replays[1].evidence, failure_reason="step failed (TimeoutError)"
+        ),
+    )
+    evidence = replace(
+        subject.runtime_evidence,
+        replays=tuple(replays),
+        replay_failure_reason="fresh container replay failed (RuntimeError)",
+    )
+    result = EpisodeDeterminismGrader().run(replace(subject, runtime_evidence=evidence))
+    assert result.status is CheckStatus.SKIP
+    assert result.measured["completed_replays"] == 2
+
+
 @pytest.mark.parametrize(
     "low,bound,status", [(0.5, 0.1, CheckStatus.PASS), (0.0, 0.2, CheckStatus.FAIL)]
 )