Only accept reward.txt written during verification

docs/source/environments/coding_tools.md

@@ -8,6 +8,20 @@ filesystem and shell tools. The tool surface is modeled after
 Each episode creates a fresh E2B sandbox. Reset accepts setup and verify
 commands. Verify commands are used by `submit_solution`.
 
+Reward defaults to `passed_verify_commands / total_verify_commands`. A verify
+command can override this by writing a float to:
+
+```text
+/home/user/logs/verifier/reward.txt
+```
+
+The file is deleted before the verify commands run, so only a verify command
+can set it; anything the agent writes there earlier is discarded. The value
+must be a finite number; a verify command is written by the task author, so
+negative rewards and values above 1 are accepted too. A value that is not a
+finite number is ignored, the pass rate is used, and the reason is recorded in
+`state.reward_override_ignored`.
+
 ## Tools
 
 - `bash(command, timeout=30)`

docs/source/environments/jupyter.md

@@ -86,6 +86,13 @@ can override this by writing a float to:
 /home/user/logs/verifier/reward.txt
 ```
 
+The file is deleted before the verify commands run, so only a verify command
+can set it; anything the agent writes there earlier is discarded. The value
+must be a finite number; a verify command is written by the task author, so
+negative rewards and values above 1 are accepted too. A value that is not a
+finite number is ignored, the pass rate is used, and the reason is recorded in
+`state.reward_override_ignored`.
+
 ## Notes
 
 This first version intentionally keeps sandbox provider selection local to the

docs/source/environments/opencode.md

@@ -25,7 +25,12 @@ with a uniform Task shape:
     pytest invocations, score-file writes)
 
 Reward = `passed_verify / total_verify` unless any `verify` command writes
-a float to `/home/user/logs/verifier/reward.txt` (override).
+a float to `/home/user/logs/verifier/reward.txt` (override). The file is deleted
+before the verify commands run, so a value the agent writes during its run
+is discarded. The override must be a finite number; a verify command is
+written by the task author, so negative rewards and values above 1 are
+accepted too. A value that is not a finite number is ignored in favour of the
+pass rate, with the reason in `RolloutResult.reward_override_ignored`.
 
 ## Quick Start
 

docs/source/environments/pi.md

@@ -22,7 +22,12 @@ with a uniform Task shape:
     pytest invocations, score-file writes)
 
 Reward = `passed_verify / total_verify` unless any `verify` command writes
-a float to `/root/logs/verifier/reward.txt` (override).
+a float to `/root/logs/verifier/reward.txt` (override). The file is deleted
+before the verify commands run, so a value the agent writes during its run
+is discarded. The override must be a finite number; a verify command is
+written by the task author, so negative rewards and values above 1 are
+accepted too. A value that is not a finite number is ignored in favour of the
+pass rate, with the reason in `RolloutResult.reward_override_ignored`.
 
 ## In-process primitive (no HTTP)
 

docs/source/environments/terminus.md

@@ -71,3 +71,10 @@ verify command can override this by writing a float to:
 ```text
 /home/user/logs/verifier/reward.txt
 ```
+
+The file is deleted before the verify commands run, so only a verify command
+can set it; anything the agent writes there earlier is discarded. The value
+must be a finite number; a verify command is written by the task author, so
+negative rewards and values above 1 are accepted too. A value that is not a
+finite number is ignored, the pass rate is used, and the reason is recorded in
+`state.reward_override_ignored`.

envs/coding_tools_env/README.md

@@ -24,6 +24,20 @@ filesystem and shell tools. The tool surface is modeled after
 Each episode creates a fresh E2B sandbox. Reset accepts setup and verify
 commands. Verify commands are used by `submit_solution`.
 
+Reward defaults to `passed_verify_commands / total_verify_commands`. A verify
+command can override this by writing a float to:
+
+```text
+/home/user/logs/verifier/reward.txt
+```
+
+The file is deleted before the verify commands run, so only a verify command
+can set it; anything the agent writes there earlier is discarded. The value
+must be a finite number; a verify command is written by the task author, so
+negative rewards and values above 1 are accepted too. A value that is not a
+finite number is ignored, the pass rate is used, and the reason is recorded in
+`state.reward_override_ignored`.
+
 ## Tools
 
 - `bash(command, timeout=30)`

envs/coding_tools_env/models.py

@@ -44,6 +44,7 @@ class CodingToolsState(State):
     tool_history: list[CommandResult] = Field(default_factory=list)
     submitted: bool = False
     last_reward: float | None = None
+    reward_override_ignored: str | None = None
     last_error: str | None = None
 
 

envs/coding_tools_env/server/coding_tools_env_environment.py

@@ -8,6 +8,7 @@
 
 from __future__ import annotations
 
+import math
 import os
 from typing import Any, Optional
 from uuid import uuid4
@@ -25,6 +26,7 @@
 
 
 REWARD_FILE = "/home/user/logs/verifier/reward.txt"
+_CLEAR_REWARD_FILE = f"rm -f {REWARD_FILE} && test ! -e {REWARD_FILE}"
 
 
 class CodingToolsEnvironment(MCPEnvironment):
@@ -374,6 +376,9 @@ def _run_verify_commands(self) -> dict[str, Any]:
         if not self._sandbox:
             return {"passed": 0, "total": 0, "reward": None}
         self._sandbox.run_shell("mkdir -p /home/user/logs/verifier")
+        # The override is documented as verify-written, so drop anything the
+        # policy wrote there during the episode before verification starts.
+        cleared = self._sandbox.run_shell(_CLEAR_REWARD_FILE).ok
         self._state.verify_results = []
         passed = 0
         for command in self._state.verify_commands:
@@ -389,7 +394,9 @@ def _run_verify_commands(self) -> dict[str, Any]:
             if result.ok:
                 passed += 1
         total = len(self._state.verify_commands)
-        reward = _read_reward_override(self._sandbox)
+        reward, self._state.reward_override_ignored = _read_reward_override(
+            self._sandbox, cleared
+        )
         if reward is None:
             reward = (passed / total) if total else 0.0
         self._state.last_reward = reward
@@ -404,14 +411,35 @@ def _coerce_commands(value: Any) -> list[str]:
     return [str(item) for item in value if str(item).strip()]
 
 
-def _read_reward_override(sandbox: E2BSandbox) -> float | None:
+def _read_reward_override(
+    sandbox: E2BSandbox, cleared: bool
+) -> tuple[float | None, str | None]:
+    if not cleared:
+        return None, "reward file could not be removed before verification"
     result = sandbox.read_file(REWARD_FILE)
     if not result.ok:
-        return None
-    raw = (result.output or "").strip()
+        return None, None
+    return _parse_reward_override(result.output or "")
+
+
+def _parse_reward_override(raw: str) -> tuple[float | None, str | None]:
+    """Parse the reward file written by a verify command.
+
+    The value comes from the task author rather than the agent, so any finite
+    reward is accepted, including negative ones and values above 1. Only a
+    value that would poison training is rejected.
+
+    Returns `(reward, None)` for a finite value, `(None, None)` when the file
+    is empty, and `(None, reason)` when the value is rejected, in which case
+    the caller falls back to the verify pass rate.
+    """
+    raw = raw.strip()
     if not raw:
-        return None
+        return None, None
     try:
-        return float(raw)
+        value = float(raw)
     except ValueError:
-        return None
+        return None, f"not a number: {raw[:40]!r}"
+    if not math.isfinite(value):
+        return None, f"not finite: {raw[:40]!r}"
+    return value, None

envs/jupyter_env/README.md

@@ -101,6 +101,13 @@ can override this by writing a float to:
 /home/user/logs/verifier/reward.txt
 ```
 
+The file is deleted before the verify commands run, so only a verify command
+can set it; anything the agent writes there earlier is discarded. The value
+must be a finite number; a verify command is written by the task author, so
+negative rewards and values above 1 are accepted too. A value that is not a
+finite number is ignored, the pass rate is used, and the reason is recorded in
+`state.reward_override_ignored`.
+
 ## Notes
 
 This first version intentionally keeps sandbox provider selection local to the

envs/jupyter_env/models.py

@@ -40,3 +40,4 @@ class JupyterState(State):
     verify_commands: List[str] = Field(default_factory=list)
     submitted_answer: Optional[str] = None
     last_reward: Optional[float] = None
+    reward_override_ignored: Optional[str] = None

envs/jupyter_env/server/e2b_sandbox.py

@@ -100,6 +100,8 @@ def run_shell(self, command: str, timeout_s: float = 120) -> CellResult:
             f"_result = subprocess.run({command!r}, shell=True, capture_output=True, text=True, timeout={float(timeout_s)!r})\n"
             "print(_result.stdout, end='')\n"
             "if _result.stderr: print(_result.stderr, end='', file=sys.stderr)\n"
+            "if _result.returncode != 0:\n"
+            "    raise SystemExit(_result.returncode)\n"
         )
         return self.run_code(shell_code)
 

envs/jupyter_env/server/jupyter_environment.py

@@ -11,6 +11,7 @@
 """
 
 import logging
+import math
 import os
 from pathlib import Path
 from typing import Any, Iterable, Optional
@@ -29,6 +30,7 @@
 
 log = logging.getLogger(__name__)
 REWARD_FILE = "/home/user/logs/verifier/reward.txt"
+_CLEAR_REWARD_FILE = f"rm -f {REWARD_FILE} && test ! -e {REWARD_FILE}"
 
 
 class JupyterEnvironment(MCPEnvironment):
@@ -387,12 +389,17 @@ def _run_verify_commands(self) -> dict[str, Any]:
             return {"passed": 0, "total": 0, "reward": None}
 
         self._sandbox.run_shell("mkdir -p /home/user/logs/verifier")
+        # The override is documented as verify-written, so drop anything the
+        # policy wrote there during the episode before verification starts.
+        cleared = self._sandbox.run_shell(_CLEAR_REWARD_FILE).success
         verify_results = self._run_shell_commands(self._state.verify_commands)
         self._state.verify_results = verify_results
 
         passed = sum(1 for result in verify_results if result.success)
         total = len(verify_results)
-        reward = _read_reward_override(self._sandbox)
+        reward, self._state.reward_override_ignored = _read_reward_override(
+            self._sandbox, cleared
+        )
         if reward is None and total:
             reward = passed / total
         self._state.last_reward = reward
@@ -441,15 +448,36 @@ def _command_result_from_cell_result(command: str, result):
     )
 
 
-def _read_reward_override(sandbox) -> Optional[float]:
+def _read_reward_override(
+    sandbox, cleared: bool
+) -> tuple[Optional[float], Optional[str]]:
+    if not cleared:
+        return None, "reward file could not be removed before verification"
     result = sandbox.run_shell(f"cat {REWARD_FILE} 2>/dev/null || true")
-    raw = (result.stdout or "").strip()
+    return _parse_reward_override(result.stdout or "")
+
+
+def _parse_reward_override(raw: str) -> tuple[float | None, str | None]:
+    """Parse the reward file written by a verify command.
+
+    The value comes from the task author rather than the agent, so any finite
+    reward is accepted, including negative ones and values above 1. Only a
+    value that would poison training is rejected.
+
+    Returns `(reward, None)` for a finite value, `(None, None)` when the file
+    is empty, and `(None, reason)` when the value is rejected, in which case
+    the caller falls back to the verify pass rate.
+    """
+    raw = raw.strip()
     if not raw:
-        return None
+        return None, None
     try:
-        return float(raw)
+        value = float(raw)
     except ValueError:
-        return None
+        return None, f"not a number: {raw[:40]!r}"
+    if not math.isfinite(value):
+        return None, f"not finite: {raw[:40]!r}"
+    return value, None
 
 
 def _get_notebook_cell_class():

envs/opencode_env/README.md

@@ -38,7 +38,12 @@ with a uniform Task shape:
     pytest invocations, score-file writes)
 
 Reward = `passed_verify / total_verify` unless any `verify` command writes
-a float to `/home/user/logs/verifier/reward.txt` (override).
+a float to `/home/user/logs/verifier/reward.txt` (override). The file is deleted
+before the verify commands run, so a value the agent writes during its run
+is discarded. The override must be a finite number; a verify command is
+written by the task author, so negative rewards and values above 1 are
+accepted too. A value that is not a finite number is ignored in favour of the
+pass rate, with the reason in `RolloutResult.reward_override_ignored`.
 
 ## Quick Start
 

envs/opencode_env/models.py

@@ -57,6 +57,8 @@ class RolloutResult(BaseModel):
 
     # Scalars
     reward: float | None = None
+    # Why reward.txt was present but not used; None when it was used or absent
+    reward_override_ignored: str | None = None
     agent_exit_code: int | None = None
     wall_s: float = 0.0
     mode: str = "transparent_proxy"

envs/opencode_env/server/opencode_environment.py

@@ -22,6 +22,7 @@
 from __future__ import annotations
 
 import json
+import math
 import os
 import time
 from typing import Any, Optional
@@ -50,6 +51,7 @@
 WORKDIR = f"{HOME}/workdir"
 INSTRUCTION_PATH = f"{HOME}/task/instruction.md"
 REWARD_FILE = f"{HOME}/logs/verifier/reward.txt"
+_CLEAR_REWARD_FILE = f"rm -f {REWARD_FILE} && test ! -e {REWARD_FILE}"
 PROXY_LOG = f"{HOME}/logs/agent/proxy.log"
 AGENT_LOG = f"{HOME}/logs/agent/opencode.jsonl"
 VERIFY_TIMEOUT_S = 120
@@ -375,6 +377,12 @@ def _emit(msg: str) -> None:
             # Verify + reward only when the run is clean; a failed setup or
             # agent leaves a half-prepared sandbox, so reward stays None.
             if result.error is None:
+                # The override is documented as verify-written, so drop anything
+                # the agent wrote there during its run before verification starts.
+                cleared = (
+                    self._exec_command(session.sandbox, _CLEAR_REWARD_FILE).exit_code
+                    == 0
+                )
                 verify_passed = 0
                 for i, cmd in enumerate(verify, 1):
                     _emit(f"verify [{i}/{len(verify)}]: {cmd[:80]}")
@@ -384,7 +392,9 @@ def _emit(msg: str) -> None:
                         verify_passed += 1
 
                 # Explicit reward.txt wins; else passed/total of verify.
-                override = self._read_reward(session.sandbox)
+                override, result.reward_override_ignored = self._read_reward(
+                    session.sandbox, cleared
+                )
                 if override is not None:
                     result.reward = override
                 elif verify:
@@ -447,14 +457,12 @@ def _exec_command(self, sandbox: Any, cmd: str, timeout: int = VERIFY_TIMEOUT_S)
                 duration_s=round(time.time() - t, 3),
             )
 
-    def _read_reward(self, sandbox: Any) -> float | None:
-        raw = self._safe_read(sandbox, REWARD_FILE).strip()
-        if not raw:
-            return None
-        try:
-            return float(raw)
-        except ValueError:
-            return None
+    def _read_reward(
+        self, sandbox: Any, cleared: bool
+    ) -> tuple[float | None, str | None]:
+        if not cleared:
+            return None, "reward file could not be removed before verification"
+        return _parse_reward_override(self._safe_read(sandbox, REWARD_FILE))
 
     def _collect_files(
         self, sandbox: Any
@@ -514,3 +522,26 @@ def _safe_read(sandbox: Any, path: str) -> str:
             return sandbox.read_text(path) or ""
         except Exception:
             return ""
+
+
+def _parse_reward_override(raw: str) -> tuple[float | None, str | None]:
+    """Parse the reward file written by a verify command.
+
+    The value comes from the task author rather than the agent, so any finite
+    reward is accepted, including negative ones and values above 1. Only a
+    value that would poison training is rejected.
+
+    Returns `(reward, None)` for a finite value, `(None, None)` when the file
+    is empty, and `(None, reason)` when the value is rejected, in which case
+    the caller falls back to the verify pass rate.
+    """
+    raw = raw.strip()
+    if not raw:
+        return None, None
+    try:
+        value = float(raw)
+    except ValueError:
+        return None, f"not a number: {raw[:40]!r}"
+    if not math.isfinite(value):
+        return None, f"not finite: {raw[:40]!r}"
+    return value, None