diff --git a/docs/source/environments/coding_tools.md b/docs/source/environments/coding_tools.md index fc1b57084b..82e61b9e74 100644 --- a/docs/source/environments/coding_tools.md +++ b/docs/source/environments/coding_tools.md @@ -8,6 +8,20 @@ filesystem and shell tools. The tool surface is modeled after Each episode creates a fresh E2B sandbox. Reset accepts setup and verify commands. Verify commands are used by `submit_solution`. +Reward defaults to `passed_verify_commands / total_verify_commands`. A verify +command can override this by writing a float to: + +```text +/home/user/logs/verifier/reward.txt +``` + +The file is deleted before the verify commands run, so only a verify command +can set it; anything the agent writes there earlier is discarded. The value +must be a finite number; a verify command is written by the task author, so +negative rewards and values above 1 are accepted too. A value that is not a +finite number is ignored, the pass rate is used, and the reason is recorded in +`state.reward_override_ignored`. + ## Tools - `bash(command, timeout=30)` diff --git a/docs/source/environments/jupyter.md b/docs/source/environments/jupyter.md index d5e10e00a0..dfbc64d14f 100644 --- a/docs/source/environments/jupyter.md +++ b/docs/source/environments/jupyter.md @@ -86,6 +86,13 @@ can override this by writing a float to: /home/user/logs/verifier/reward.txt ``` +The file is deleted before the verify commands run, so only a verify command +can set it; anything the agent writes there earlier is discarded. The value +must be a finite number; a verify command is written by the task author, so +negative rewards and values above 1 are accepted too. A value that is not a +finite number is ignored, the pass rate is used, and the reason is recorded in +`state.reward_override_ignored`. + ## Notes This first version intentionally keeps sandbox provider selection local to the diff --git a/docs/source/environments/opencode.md b/docs/source/environments/opencode.md index 8804eee01d..4f299d5aa1 100644 --- a/docs/source/environments/opencode.md +++ b/docs/source/environments/opencode.md @@ -25,7 +25,12 @@ with a uniform Task shape: pytest invocations, score-file writes) Reward = `passed_verify / total_verify` unless any `verify` command writes -a float to `/home/user/logs/verifier/reward.txt` (override). +a float to `/home/user/logs/verifier/reward.txt` (override). The file is deleted +before the verify commands run, so a value the agent writes during its run +is discarded. The override must be a finite number; a verify command is +written by the task author, so negative rewards and values above 1 are +accepted too. A value that is not a finite number is ignored in favour of the +pass rate, with the reason in `RolloutResult.reward_override_ignored`. ## Quick Start diff --git a/docs/source/environments/pi.md b/docs/source/environments/pi.md index a69179844a..0717e66f69 100644 --- a/docs/source/environments/pi.md +++ b/docs/source/environments/pi.md @@ -22,7 +22,12 @@ with a uniform Task shape: pytest invocations, score-file writes) Reward = `passed_verify / total_verify` unless any `verify` command writes -a float to `/root/logs/verifier/reward.txt` (override). +a float to `/root/logs/verifier/reward.txt` (override). The file is deleted +before the verify commands run, so a value the agent writes during its run +is discarded. The override must be a finite number; a verify command is +written by the task author, so negative rewards and values above 1 are +accepted too. A value that is not a finite number is ignored in favour of the +pass rate, with the reason in `RolloutResult.reward_override_ignored`. ## In-process primitive (no HTTP) diff --git a/docs/source/environments/terminus.md b/docs/source/environments/terminus.md index dc1995aa8f..066c166f47 100644 --- a/docs/source/environments/terminus.md +++ b/docs/source/environments/terminus.md @@ -71,3 +71,10 @@ verify command can override this by writing a float to: ```text /home/user/logs/verifier/reward.txt ``` + +The file is deleted before the verify commands run, so only a verify command +can set it; anything the agent writes there earlier is discarded. The value +must be a finite number; a verify command is written by the task author, so +negative rewards and values above 1 are accepted too. A value that is not a +finite number is ignored, the pass rate is used, and the reason is recorded in +`state.reward_override_ignored`. diff --git a/envs/coding_tools_env/README.md b/envs/coding_tools_env/README.md index ddc16191b6..70875147f2 100644 --- a/envs/coding_tools_env/README.md +++ b/envs/coding_tools_env/README.md @@ -24,6 +24,20 @@ filesystem and shell tools. The tool surface is modeled after Each episode creates a fresh E2B sandbox. Reset accepts setup and verify commands. Verify commands are used by `submit_solution`. +Reward defaults to `passed_verify_commands / total_verify_commands`. A verify +command can override this by writing a float to: + +```text +/home/user/logs/verifier/reward.txt +``` + +The file is deleted before the verify commands run, so only a verify command +can set it; anything the agent writes there earlier is discarded. The value +must be a finite number; a verify command is written by the task author, so +negative rewards and values above 1 are accepted too. A value that is not a +finite number is ignored, the pass rate is used, and the reason is recorded in +`state.reward_override_ignored`. + ## Tools - `bash(command, timeout=30)` diff --git a/envs/coding_tools_env/models.py b/envs/coding_tools_env/models.py index 01f32628eb..f2c76d794a 100644 --- a/envs/coding_tools_env/models.py +++ b/envs/coding_tools_env/models.py @@ -44,6 +44,7 @@ class CodingToolsState(State): tool_history: list[CommandResult] = Field(default_factory=list) submitted: bool = False last_reward: float | None = None + reward_override_ignored: str | None = None last_error: str | None = None diff --git a/envs/coding_tools_env/server/coding_tools_env_environment.py b/envs/coding_tools_env/server/coding_tools_env_environment.py index 615e7770fe..c95c45d500 100644 --- a/envs/coding_tools_env/server/coding_tools_env_environment.py +++ b/envs/coding_tools_env/server/coding_tools_env_environment.py @@ -8,6 +8,7 @@ from __future__ import annotations +import math import os from typing import Any, Optional from uuid import uuid4 @@ -25,6 +26,7 @@ REWARD_FILE = "/home/user/logs/verifier/reward.txt" +_CLEAR_REWARD_FILE = f"rm -f {REWARD_FILE} && test ! -e {REWARD_FILE}" class CodingToolsEnvironment(MCPEnvironment): @@ -374,6 +376,9 @@ def _run_verify_commands(self) -> dict[str, Any]: if not self._sandbox: return {"passed": 0, "total": 0, "reward": None} self._sandbox.run_shell("mkdir -p /home/user/logs/verifier") + # The override is documented as verify-written, so drop anything the + # policy wrote there during the episode before verification starts. + cleared = self._sandbox.run_shell(_CLEAR_REWARD_FILE).ok self._state.verify_results = [] passed = 0 for command in self._state.verify_commands: @@ -389,7 +394,9 @@ def _run_verify_commands(self) -> dict[str, Any]: if result.ok: passed += 1 total = len(self._state.verify_commands) - reward = _read_reward_override(self._sandbox) + reward, self._state.reward_override_ignored = _read_reward_override( + self._sandbox, cleared + ) if reward is None: reward = (passed / total) if total else 0.0 self._state.last_reward = reward @@ -404,14 +411,35 @@ def _coerce_commands(value: Any) -> list[str]: return [str(item) for item in value if str(item).strip()] -def _read_reward_override(sandbox: E2BSandbox) -> float | None: +def _read_reward_override( + sandbox: E2BSandbox, cleared: bool +) -> tuple[float | None, str | None]: + if not cleared: + return None, "reward file could not be removed before verification" result = sandbox.read_file(REWARD_FILE) if not result.ok: - return None - raw = (result.output or "").strip() + return None, None + return _parse_reward_override(result.output or "") + + +def _parse_reward_override(raw: str) -> tuple[float | None, str | None]: + """Parse the reward file written by a verify command. + + The value comes from the task author rather than the agent, so any finite + reward is accepted, including negative ones and values above 1. Only a + value that would poison training is rejected. + + Returns `(reward, None)` for a finite value, `(None, None)` when the file + is empty, and `(None, reason)` when the value is rejected, in which case + the caller falls back to the verify pass rate. + """ + raw = raw.strip() if not raw: - return None + return None, None try: - return float(raw) + value = float(raw) except ValueError: - return None + return None, f"not a number: {raw[:40]!r}" + if not math.isfinite(value): + return None, f"not finite: {raw[:40]!r}" + return value, None diff --git a/envs/jupyter_env/README.md b/envs/jupyter_env/README.md index ad5c7c174f..dc5325fa58 100644 --- a/envs/jupyter_env/README.md +++ b/envs/jupyter_env/README.md @@ -101,6 +101,13 @@ can override this by writing a float to: /home/user/logs/verifier/reward.txt ``` +The file is deleted before the verify commands run, so only a verify command +can set it; anything the agent writes there earlier is discarded. The value +must be a finite number; a verify command is written by the task author, so +negative rewards and values above 1 are accepted too. A value that is not a +finite number is ignored, the pass rate is used, and the reason is recorded in +`state.reward_override_ignored`. + ## Notes This first version intentionally keeps sandbox provider selection local to the diff --git a/envs/jupyter_env/models.py b/envs/jupyter_env/models.py index 753773871e..7bbc991256 100644 --- a/envs/jupyter_env/models.py +++ b/envs/jupyter_env/models.py @@ -40,3 +40,4 @@ class JupyterState(State): verify_commands: List[str] = Field(default_factory=list) submitted_answer: Optional[str] = None last_reward: Optional[float] = None + reward_override_ignored: Optional[str] = None diff --git a/envs/jupyter_env/server/e2b_sandbox.py b/envs/jupyter_env/server/e2b_sandbox.py index ca0526c8ce..4f2cfc7b40 100644 --- a/envs/jupyter_env/server/e2b_sandbox.py +++ b/envs/jupyter_env/server/e2b_sandbox.py @@ -100,6 +100,8 @@ def run_shell(self, command: str, timeout_s: float = 120) -> CellResult: f"_result = subprocess.run({command!r}, shell=True, capture_output=True, text=True, timeout={float(timeout_s)!r})\n" "print(_result.stdout, end='')\n" "if _result.stderr: print(_result.stderr, end='', file=sys.stderr)\n" + "if _result.returncode != 0:\n" + " raise SystemExit(_result.returncode)\n" ) return self.run_code(shell_code) diff --git a/envs/jupyter_env/server/jupyter_environment.py b/envs/jupyter_env/server/jupyter_environment.py index bc622ae22b..109010eb06 100644 --- a/envs/jupyter_env/server/jupyter_environment.py +++ b/envs/jupyter_env/server/jupyter_environment.py @@ -11,6 +11,7 @@ """ import logging +import math import os from pathlib import Path from typing import Any, Iterable, Optional @@ -29,6 +30,7 @@ log = logging.getLogger(__name__) REWARD_FILE = "/home/user/logs/verifier/reward.txt" +_CLEAR_REWARD_FILE = f"rm -f {REWARD_FILE} && test ! -e {REWARD_FILE}" class JupyterEnvironment(MCPEnvironment): @@ -387,12 +389,17 @@ def _run_verify_commands(self) -> dict[str, Any]: return {"passed": 0, "total": 0, "reward": None} self._sandbox.run_shell("mkdir -p /home/user/logs/verifier") + # The override is documented as verify-written, so drop anything the + # policy wrote there during the episode before verification starts. + cleared = self._sandbox.run_shell(_CLEAR_REWARD_FILE).success verify_results = self._run_shell_commands(self._state.verify_commands) self._state.verify_results = verify_results passed = sum(1 for result in verify_results if result.success) total = len(verify_results) - reward = _read_reward_override(self._sandbox) + reward, self._state.reward_override_ignored = _read_reward_override( + self._sandbox, cleared + ) if reward is None and total: reward = passed / total self._state.last_reward = reward @@ -441,15 +448,36 @@ def _command_result_from_cell_result(command: str, result): ) -def _read_reward_override(sandbox) -> Optional[float]: +def _read_reward_override( + sandbox, cleared: bool +) -> tuple[Optional[float], Optional[str]]: + if not cleared: + return None, "reward file could not be removed before verification" result = sandbox.run_shell(f"cat {REWARD_FILE} 2>/dev/null || true") - raw = (result.stdout or "").strip() + return _parse_reward_override(result.stdout or "") + + +def _parse_reward_override(raw: str) -> tuple[float | None, str | None]: + """Parse the reward file written by a verify command. + + The value comes from the task author rather than the agent, so any finite + reward is accepted, including negative ones and values above 1. Only a + value that would poison training is rejected. + + Returns `(reward, None)` for a finite value, `(None, None)` when the file + is empty, and `(None, reason)` when the value is rejected, in which case + the caller falls back to the verify pass rate. + """ + raw = raw.strip() if not raw: - return None + return None, None try: - return float(raw) + value = float(raw) except ValueError: - return None + return None, f"not a number: {raw[:40]!r}" + if not math.isfinite(value): + return None, f"not finite: {raw[:40]!r}" + return value, None def _get_notebook_cell_class(): diff --git a/envs/opencode_env/README.md b/envs/opencode_env/README.md index ade85b182f..0bbbddd304 100644 --- a/envs/opencode_env/README.md +++ b/envs/opencode_env/README.md @@ -38,7 +38,12 @@ with a uniform Task shape: pytest invocations, score-file writes) Reward = `passed_verify / total_verify` unless any `verify` command writes -a float to `/home/user/logs/verifier/reward.txt` (override). +a float to `/home/user/logs/verifier/reward.txt` (override). The file is deleted +before the verify commands run, so a value the agent writes during its run +is discarded. The override must be a finite number; a verify command is +written by the task author, so negative rewards and values above 1 are +accepted too. A value that is not a finite number is ignored in favour of the +pass rate, with the reason in `RolloutResult.reward_override_ignored`. ## Quick Start diff --git a/envs/opencode_env/models.py b/envs/opencode_env/models.py index b218c5f78f..1f19c908a3 100644 --- a/envs/opencode_env/models.py +++ b/envs/opencode_env/models.py @@ -57,6 +57,8 @@ class RolloutResult(BaseModel): # Scalars reward: float | None = None + # Why reward.txt was present but not used; None when it was used or absent + reward_override_ignored: str | None = None agent_exit_code: int | None = None wall_s: float = 0.0 mode: str = "transparent_proxy" diff --git a/envs/opencode_env/server/opencode_environment.py b/envs/opencode_env/server/opencode_environment.py index a10290711c..e7e0754455 100644 --- a/envs/opencode_env/server/opencode_environment.py +++ b/envs/opencode_env/server/opencode_environment.py @@ -22,6 +22,7 @@ from __future__ import annotations import json +import math import os import time from typing import Any, Optional @@ -50,6 +51,7 @@ WORKDIR = f"{HOME}/workdir" INSTRUCTION_PATH = f"{HOME}/task/instruction.md" REWARD_FILE = f"{HOME}/logs/verifier/reward.txt" +_CLEAR_REWARD_FILE = f"rm -f {REWARD_FILE} && test ! -e {REWARD_FILE}" PROXY_LOG = f"{HOME}/logs/agent/proxy.log" AGENT_LOG = f"{HOME}/logs/agent/opencode.jsonl" VERIFY_TIMEOUT_S = 120 @@ -375,6 +377,12 @@ def _emit(msg: str) -> None: # Verify + reward only when the run is clean; a failed setup or # agent leaves a half-prepared sandbox, so reward stays None. if result.error is None: + # The override is documented as verify-written, so drop anything + # the agent wrote there during its run before verification starts. + cleared = ( + self._exec_command(session.sandbox, _CLEAR_REWARD_FILE).exit_code + == 0 + ) verify_passed = 0 for i, cmd in enumerate(verify, 1): _emit(f"verify [{i}/{len(verify)}]: {cmd[:80]}") @@ -384,7 +392,9 @@ def _emit(msg: str) -> None: verify_passed += 1 # Explicit reward.txt wins; else passed/total of verify. - override = self._read_reward(session.sandbox) + override, result.reward_override_ignored = self._read_reward( + session.sandbox, cleared + ) if override is not None: result.reward = override elif verify: @@ -447,14 +457,12 @@ def _exec_command(self, sandbox: Any, cmd: str, timeout: int = VERIFY_TIMEOUT_S) duration_s=round(time.time() - t, 3), ) - def _read_reward(self, sandbox: Any) -> float | None: - raw = self._safe_read(sandbox, REWARD_FILE).strip() - if not raw: - return None - try: - return float(raw) - except ValueError: - return None + def _read_reward( + self, sandbox: Any, cleared: bool + ) -> tuple[float | None, str | None]: + if not cleared: + return None, "reward file could not be removed before verification" + return _parse_reward_override(self._safe_read(sandbox, REWARD_FILE)) def _collect_files( self, sandbox: Any @@ -514,3 +522,26 @@ def _safe_read(sandbox: Any, path: str) -> str: return sandbox.read_text(path) or "" except Exception: return "" + + +def _parse_reward_override(raw: str) -> tuple[float | None, str | None]: + """Parse the reward file written by a verify command. + + The value comes from the task author rather than the agent, so any finite + reward is accepted, including negative ones and values above 1. Only a + value that would poison training is rejected. + + Returns `(reward, None)` for a finite value, `(None, None)` when the file + is empty, and `(None, reason)` when the value is rejected, in which case + the caller falls back to the verify pass rate. + """ + raw = raw.strip() + if not raw: + return None, None + try: + value = float(raw) + except ValueError: + return None, f"not a number: {raw[:40]!r}" + if not math.isfinite(value): + return None, f"not finite: {raw[:40]!r}" + return value, None diff --git a/envs/pi_env/README.md b/envs/pi_env/README.md index d4f06f9fce..9cb7c70125 100644 --- a/envs/pi_env/README.md +++ b/envs/pi_env/README.md @@ -35,7 +35,12 @@ with a uniform Task shape: pytest invocations, score-file writes) Reward = `passed_verify / total_verify` unless any `verify` command writes -a float to `/root/logs/verifier/reward.txt` (override). +a float to `/root/logs/verifier/reward.txt` (override). The file is deleted +before the verify commands run, so a value the agent writes during its run +is discarded. The override must be a finite number; a verify command is +written by the task author, so negative rewards and values above 1 are +accepted too. A value that is not a finite number is ignored in favour of the +pass rate, with the reason in `RolloutResult.reward_override_ignored`. ## In-process primitive (no HTTP) diff --git a/envs/pi_env/models.py b/envs/pi_env/models.py index a5daad18b8..0398c03bff 100644 --- a/envs/pi_env/models.py +++ b/envs/pi_env/models.py @@ -57,6 +57,8 @@ class RolloutResult(BaseModel): # Scalars reward: float | None = None + # Why reward.txt was present but not used; None when it was used or absent + reward_override_ignored: str | None = None agent_exit_code: int | None = None wall_s: float = 0.0 mode: str = "transparent_proxy" diff --git a/envs/pi_env/server/pi_environment.py b/envs/pi_env/server/pi_environment.py index afc0d7f111..f7b2d94bab 100644 --- a/envs/pi_env/server/pi_environment.py +++ b/envs/pi_env/server/pi_environment.py @@ -22,6 +22,7 @@ from __future__ import annotations import json +import math import time from typing import Any, Optional from uuid import uuid4 @@ -52,6 +53,7 @@ WORKDIR = f"{HOME}/workdir" INSTRUCTION_PATH = f"{HOME}/task/instruction.md" REWARD_FILE = f"{HOME}/logs/verifier/reward.txt" +_CLEAR_REWARD_FILE = f"rm -f {REWARD_FILE} && test ! -e {REWARD_FILE}" PROXY_LOG = f"{HOME}/logs/agent/proxy.log" AGENT_LOG = f"{HOME}/logs/agent/pi.jsonl" VERIFY_TIMEOUT_S = 120 @@ -373,6 +375,12 @@ def _emit(msg: str) -> None: # Verify + reward only when the run is clean; a failed setup or # agent leaves a half-prepared sandbox, so reward stays None. if result.error is None: + # The override is documented as verify-written, so drop anything + # the agent wrote there during its run before verification starts. + cleared = ( + self._exec_command(session.sandbox, _CLEAR_REWARD_FILE).exit_code + == 0 + ) verify_passed = 0 for i, cmd in enumerate(verify, 1): _emit(f"verify [{i}/{len(verify)}]: {cmd[:80]}") @@ -382,7 +390,9 @@ def _emit(msg: str) -> None: verify_passed += 1 # Explicit reward.txt wins; else passed/total of verify. - override = self._read_reward(session.sandbox) + override, result.reward_override_ignored = self._read_reward( + session.sandbox, cleared + ) if override is not None: result.reward = override elif verify: @@ -445,14 +455,12 @@ def _exec_command(self, sandbox: Any, cmd: str, timeout: int = VERIFY_TIMEOUT_S) duration_s=round(time.time() - t, 3), ) - def _read_reward(self, sandbox: Any) -> float | None: - raw = self._safe_read(sandbox, REWARD_FILE).strip() - if not raw: - return None - try: - return float(raw) - except ValueError: - return None + def _read_reward( + self, sandbox: Any, cleared: bool + ) -> tuple[float | None, str | None]: + if not cleared: + return None, "reward file could not be removed before verification" + return _parse_reward_override(self._safe_read(sandbox, REWARD_FILE)) def _collect_files( self, sandbox: Any @@ -512,3 +520,26 @@ def _safe_read(sandbox: Any, path: str) -> str: return sandbox.read_text(path) or "" except Exception: return "" + + +def _parse_reward_override(raw: str) -> tuple[float | None, str | None]: + """Parse the reward file written by a verify command. + + The value comes from the task author rather than the agent, so any finite + reward is accepted, including negative ones and values above 1. Only a + value that would poison training is rejected. + + Returns `(reward, None)` for a finite value, `(None, None)` when the file + is empty, and `(None, reason)` when the value is rejected, in which case + the caller falls back to the verify pass rate. + """ + raw = raw.strip() + if not raw: + return None, None + try: + value = float(raw) + except ValueError: + return None, f"not a number: {raw[:40]!r}" + if not math.isfinite(value): + return None, f"not finite: {raw[:40]!r}" + return value, None diff --git a/envs/terminus_env/README.md b/envs/terminus_env/README.md index 246ba709ff..6444b278a4 100644 --- a/envs/terminus_env/README.md +++ b/envs/terminus_env/README.md @@ -87,3 +87,10 @@ verify command can override this by writing a float to: ```text /home/user/logs/verifier/reward.txt ``` + +The file is deleted before the verify commands run, so only a verify command +can set it; anything the agent writes there earlier is discarded. The value +must be a finite number; a verify command is written by the task author, so +negative rewards and values above 1 are accepted too. A value that is not a +finite number is ignored, the pass rate is used, and the reason is recorded in +`state.reward_override_ignored`. diff --git a/envs/terminus_env/models.py b/envs/terminus_env/models.py index 3742f8a027..ed9702ad29 100644 --- a/envs/terminus_env/models.py +++ b/envs/terminus_env/models.py @@ -31,4 +31,5 @@ class TerminusState(State): commands: list[CommandResult] = Field(default_factory=list) submitted_answer: str | None = None last_reward: float | None = None + reward_override_ignored: str | None = None last_error: str | None = None diff --git a/envs/terminus_env/server/terminus_env_environment.py b/envs/terminus_env/server/terminus_env_environment.py index c6f9e1c026..596e6b2b85 100644 --- a/envs/terminus_env/server/terminus_env_environment.py +++ b/envs/terminus_env/server/terminus_env_environment.py @@ -8,6 +8,7 @@ from __future__ import annotations +import math import os from typing import Any, Iterable, Optional from uuid import uuid4 @@ -25,6 +26,7 @@ REWARD_FILE = "/home/user/logs/verifier/reward.txt" +_CLEAR_REWARD_FILE = f"rm -f {REWARD_FILE} && test ! -e {REWARD_FILE}" class TerminusEnvironment(MCPEnvironment): @@ -228,11 +230,16 @@ def _run_verify_commands(self) -> dict[str, Any]: return {"passed": 0, "total": 0, "reward": None} self._sandbox.run_shell("mkdir -p /home/user/logs/verifier") + # The override is documented as verify-written, so drop anything the + # policy wrote there during the episode before verification starts. + cleared = self._sandbox.run_shell(_CLEAR_REWARD_FILE).success verify_results = self._run_shell_commands(self._state.verify_commands) self._state.verify_results = verify_results passed = sum(1 for result in verify_results if result.success) total = len(verify_results) - reward = _read_reward_override(self._sandbox) + reward, self._state.reward_override_ignored = _read_reward_override( + self._sandbox, cleared + ) if reward is None and total: reward = passed / total self._state.last_reward = reward @@ -258,12 +265,33 @@ def _format_for_llm(result) -> str: return "\n".join(parts) if parts else "(no output)" -def _read_reward_override(sandbox: E2BSandbox) -> Optional[float]: +def _read_reward_override( + sandbox: E2BSandbox, cleared: bool +) -> tuple[Optional[float], Optional[str]]: + if not cleared: + return None, "reward file could not be removed before verification" result = sandbox.run_shell(f"cat {REWARD_FILE} 2>/dev/null || true") - raw = (result.stdout or "").strip() + return _parse_reward_override(result.stdout or "") + + +def _parse_reward_override(raw: str) -> tuple[float | None, str | None]: + """Parse the reward file written by a verify command. + + The value comes from the task author rather than the agent, so any finite + reward is accepted, including negative ones and values above 1. Only a + value that would poison training is rejected. + + Returns `(reward, None)` for a finite value, `(None, None)` when the file + is empty, and `(None, reason)` when the value is rejected, in which case + the caller falls back to the verify pass rate. + """ + raw = raw.strip() if not raw: - return None + return None, None try: - return float(raw) + value = float(raw) except ValueError: - return None + return None, f"not a number: {raw[:40]!r}" + if not math.isfinite(value): + return None, f"not finite: {raw[:40]!r}" + return value, None diff --git a/tests/envs/test_coding_tools_environment.py b/tests/envs/test_coding_tools_environment.py index b736b8ec8c..57a3963add 100644 --- a/tests/envs/test_coding_tools_environment.py +++ b/tests/envs/test_coding_tools_environment.py @@ -2,6 +2,7 @@ from __future__ import annotations +import pytest from coding_tools_env.models import CodingToolsState from coding_tools_env.server.coding_tools_env_environment import CodingToolsEnvironment from coding_tools_env.server.e2b_sandbox import ToolResult @@ -206,3 +207,101 @@ def test_todo_write_rejects_multiple_in_progress(monkeypatch): ) assert "only one todo item can be in_progress" in _extract_text(obs.result) env._state = CodingToolsState(episode_id="x") + + +REWARD_FILE = "/home/user/logs/verifier/reward.txt" + + +class RewardFileSandbox(FakeSandbox): + """FakeSandbox whose reward file shell commands and file tools can write.""" + + def __init__(self, *, removable: bool = True) -> None: + super().__init__() + self.removable = removable + + def run_shell(self, command: str, timeout_s: float = 30) -> ToolResult: + if command.startswith("echo ") and command.endswith(f" > {REWARD_FILE}"): + self.files[REWARD_FILE] = command[len("echo ") : -len(f" > {REWARD_FILE}")] + return ToolResult(ok=True, output="", error=None, metadata={"exit_code": 0}) + if command.startswith(f"rm -f {REWARD_FILE}"): + if self.removable: + self.files.pop(REWARD_FILE, None) + gone = REWARD_FILE not in self.files + return ToolResult( + ok=gone, + output="", + error=None if gone else "exit_code=1", + metadata={"exit_code": 0 if gone else 1}, + ) + return super().run_shell(command, timeout_s) + + +def _submit(monkeypatch, sandbox, verify_commands, agent_actions=()): + monkeypatch.setenv("E2B_API_KEY", "fake") + monkeypatch.setattr( + "coding_tools_env.server.coding_tools_env_environment.E2BSandbox", + lambda api_key: sandbox, + ) + env = CodingToolsEnvironment() + env.reset(verify=verify_commands) + for tool_name, arguments in agent_actions: + env.step(CallToolAction(tool_name=tool_name, arguments=arguments)) + env.step(CallToolAction(tool_name="submit_solution", arguments={})) + return env.state + + +@pytest.mark.parametrize( + "agent_action", + [ + ("bash", {"command": f"echo 1.0 > {REWARD_FILE}"}), + ("write", {"file_path": REWARD_FILE, "content": "1.0"}), + ], +) +def test_reward_file_written_by_the_agent_is_ignored(monkeypatch, agent_action): + state = _submit( + monkeypatch, RewardFileSandbox(), ["true", "exit 1"], [agent_action] + ) + + assert state.last_reward == 0.5 + assert state.reward_override_ignored is None + + +def test_reward_file_written_by_a_verify_command_overrides_the_pass_rate(monkeypatch): + state = _submit( + monkeypatch, RewardFileSandbox(), [f"echo 0.8 > {REWARD_FILE}", "exit 1"] + ) + + assert state.last_reward == 0.8 + assert state.reward_override_ignored is None + + +@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "passed"]) +def test_invalid_reward_file_falls_back_to_the_pass_rate(monkeypatch, value): + state = _submit( + monkeypatch, RewardFileSandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"] + ) + + assert state.last_reward == 0.5 + assert repr(value) in state.reward_override_ignored + + +@pytest.mark.parametrize("value", ["-0.5", "1.5"]) +def test_reward_file_outside_the_unit_interval_is_kept(monkeypatch, value): + state = _submit( + monkeypatch, RewardFileSandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"] + ) + + assert state.last_reward == float(value) + assert state.reward_override_ignored is None + + +def test_reward_file_that_cannot_be_removed_is_ignored(monkeypatch): + state = _submit( + monkeypatch, + RewardFileSandbox(removable=False), + ["true", "exit 1"], + [("write", {"file_path": REWARD_FILE, "content": "1.0"})], + ) + + assert state.last_reward == 0.5 + assert "could not be removed" in state.reward_override_ignored diff --git a/tests/envs/test_jupyter_environment.py b/tests/envs/test_jupyter_environment.py index eab588a78d..7864151e1f 100644 --- a/tests/envs/test_jupyter_environment.py +++ b/tests/envs/test_jupyter_environment.py @@ -2,8 +2,13 @@ from __future__ import annotations +import contextlib +import io +from types import SimpleNamespace + +import pytest from jupyter_env.models import JupyterState -from jupyter_env.server.e2b_sandbox import CellResult +from jupyter_env.server.e2b_sandbox import CellResult, E2BSandbox from jupyter_env.server.jupyter_environment import JupyterEnvironment from openenv.core.env_server.mcp_types import CallToolAction, ListToolsAction @@ -197,3 +202,141 @@ def test_final_answer_runs_verify_commands(): "test -f answer.py", "exit 1", ] + + +REWARD_FILE = "/home/user/logs/verifier/reward.txt" + + +def _cell(stdout: str = "", success: bool = True) -> CellResult: + return CellResult( + stdout=stdout, + stderr="", + error=None if success else "SystemExit: 1", + error_name=None if success else "SystemExit", + text_results=[], + images=[], + execution_count=1, + success=success, + ) + + +class RewardFileSandbox(FakeSandbox): + """FakeSandbox whose reward file any shell command can write, as in E2B.""" + + def __init__(self, *, removable: bool = True) -> None: + super().__init__() + self.reward_file: str | None = None + self.removable = removable + + def run_shell(self, command: str) -> CellResult: + if command.startswith("echo ") and command.endswith(f" > {REWARD_FILE}"): + self.shell_commands.append(command) + self.reward_file = command[len("echo ") : -len(f" > {REWARD_FILE}")] + return _cell() + if command.startswith(f"rm -f {REWARD_FILE}"): + self.shell_commands.append(command) + if self.removable: + self.reward_file = None + return _cell(success=self.reward_file is None) + if command.startswith(f"cat {REWARD_FILE}"): + self.shell_commands.append(command) + return _cell(stdout=self.reward_file or "") + return super().run_shell(command) + + +def _submit(sandbox, verify_commands, agent_commands=()) -> JupyterState: + env = JupyterEnvironment() + env._sandbox = sandbox + env._state = JupyterState( + episode_id="episode-1", + sandbox_id="fake-sandbox", + verify_commands=verify_commands, + ) + for command in agent_commands: + env.step( + CallToolAction( + tool_name="execute_shell_command", arguments={"command": command} + ) + ) + env.step(CallToolAction(tool_name="final_answer", arguments={"answer": "done"})) + return env.state + + +def test_reward_file_written_by_the_agent_is_ignored(): + state = _submit( + RewardFileSandbox(), + ["true", "exit 1"], + agent_commands=[f"echo 1.0 > {REWARD_FILE}"], + ) + + assert state.last_reward == 0.5 + assert state.reward_override_ignored is None + + +def test_reward_file_written_by_a_verify_command_overrides_the_pass_rate(): + state = _submit(RewardFileSandbox(), [f"echo 0.8 > {REWARD_FILE}", "exit 1"]) + + assert state.last_reward == 0.8 + assert state.reward_override_ignored is None + + +@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "passed"]) +def test_invalid_reward_file_falls_back_to_the_pass_rate(value): + state = _submit(RewardFileSandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"]) + + assert state.last_reward == 0.5 + assert repr(value) in state.reward_override_ignored + + +@pytest.mark.parametrize("value", ["-0.5", "1.5"]) +def test_reward_file_outside_the_unit_interval_is_kept(value): + state = _submit(RewardFileSandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"]) + + assert state.last_reward == float(value) + assert state.reward_override_ignored is None + + +def test_reward_file_that_cannot_be_removed_is_ignored(): + state = _submit( + RewardFileSandbox(removable=False), + ["true", "exit 1"], + agent_commands=[f"echo 1.0 > {REWARD_FILE}"], + ) + + assert state.last_reward == 0.5 + assert "could not be removed" in state.reward_override_ignored + + +class _LocalKernel: + """Runs E2B cell code in-process; an exception becomes execution.error.""" + + def run_code(self, code: str): + stdout, stderr = io.StringIO(), io.StringIO() + error = None + try: + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exec(code, {}) + except BaseException as exc: # SystemExit is reported, not raised, by E2B + error = SimpleNamespace( + name=type(exc).__name__, value=str(exc), traceback="" + ) + return SimpleNamespace( + error=error, + logs=SimpleNamespace( + stdout=[stdout.getvalue()] if stdout.getvalue() else [], + stderr=[stderr.getvalue()] if stderr.getvalue() else [], + ), + results=[], + execution_count=1, + ) + + +@pytest.mark.parametrize( + ("command", "success"), + [("true", True), ("exit 3", False), ("test -e /nonexistent/reward.txt", False)], +) +def test_run_shell_reports_the_exit_code(command, success): + sandbox = E2BSandbox.__new__(E2BSandbox) + sandbox._sbx = _LocalKernel() + + assert sandbox.run_shell(command).success is success diff --git a/tests/envs/test_opencode_reward_override.py b/tests/envs/test_opencode_reward_override.py new file mode 100644 index 0000000000..37e0846d9c --- /dev/null +++ b/tests/envs/test_opencode_reward_override.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: BSD-3-Clause + +"""OpenCode rollouts must only take ``reward.txt`` from verification, not from the agent.""" + +from __future__ import annotations + +import os +import sys +from types import SimpleNamespace + +import pytest + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +for _p in (_REPO_ROOT, os.path.join(_REPO_ROOT, "envs")): + if _p not in sys.path: + sys.path.insert(0, _p) + +from opencode_env.models import RolloutResult # noqa: E402 +from opencode_env.server.opencode_environment import ( # noqa: E402 + OpenCodeEnvironment, + REWARD_FILE, +) + + +class _Sandbox: + """Sandbox whose reward file any command can write, as the agent's shell can.""" + + sandbox_id = "sbx-test" + + def __init__(self, *, removable: bool = True) -> None: + self.reward_file: str | None = None + self.removable = removable + + def exec(self, cmd: str, timeout: int | None = None) -> SimpleNamespace: + exit_code = 0 + if cmd.startswith("echo ") and cmd.endswith(f" > {REWARD_FILE}"): + self.reward_file = cmd[len("echo ") : -len(f" > {REWARD_FILE}")] + elif cmd.startswith(f"rm -f {REWARD_FILE}"): + if self.removable: + self.reward_file = None + exit_code = 0 if self.reward_file is None else 1 + elif cmd == "exit 1": + exit_code = 1 + return SimpleNamespace(exit_code=exit_code, stdout="", stderr="") + + def read_text(self, path: str) -> str: + if path == REWARD_FILE and self.reward_file is not None: + return self.reward_file + "\n" + raise FileNotFoundError(path) + + +class _Session: + def __init__(self, sandbox: _Sandbox, agent_commands: list[str]) -> None: + self.sandbox = sandbox + self._agent_commands = agent_commands + + def start_agent(self) -> None: + pass + + def wait_for_completion(self, timeout_s: float | None = None) -> int: + for cmd in self._agent_commands: + self.sandbox.exec(cmd) + return 0 + + def close(self) -> None: + pass + + +def _rollout(monkeypatch, sandbox, verify, agent_commands=()) -> RolloutResult: + monkeypatch.setenv("E2B_API_KEY", "test") + env = OpenCodeEnvironment() + session = _Session(sandbox, list(agent_commands)) + monkeypatch.setattr( + env, + "_OpenCodeSessionFactory", + lambda **kwargs: SimpleNamespace(create=lambda **kw: session), + ) + monkeypatch.setattr(env, "_E2BSandboxBackend", lambda **kwargs: None) + raw = env._run_rollout_impl( + base_url="http://localhost:8000/v1", + api_key="test", + model="test-model", + instruction="solve the task", + setup=[], + verify=verify, + task_id="t1", + mode="black_box", + disable_thinking=False, + max_tokens_cap=0, + top_logprobs=0, + agent_timeout_s=5, + template="", + ) + return RolloutResult.model_validate_json(raw) + + +def test_reward_file_written_by_the_agent_is_ignored(monkeypatch): + result = _rollout( + monkeypatch, + _Sandbox(), + ["true", "exit 1"], + agent_commands=[f"echo 1.0 > {REWARD_FILE}"], + ) + + assert result.error is None + assert result.reward == 0.5 + assert result.reward_override_ignored is None + + +def test_reward_file_written_by_a_verify_command_overrides_the_pass_rate(monkeypatch): + result = _rollout(monkeypatch, _Sandbox(), [f"echo 0.8 > {REWARD_FILE}", "exit 1"]) + + assert result.reward == 0.8 + assert result.reward_override_ignored is None + + +@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "passed"]) +def test_invalid_reward_file_falls_back_to_the_pass_rate(monkeypatch, value): + result = _rollout( + monkeypatch, _Sandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"] + ) + + assert result.reward == 0.5 + assert repr(value) in result.reward_override_ignored + + +@pytest.mark.parametrize("value", ["-0.5", "1.5"]) +def test_reward_file_outside_the_unit_interval_is_kept(monkeypatch, value): + result = _rollout( + monkeypatch, _Sandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"] + ) + + assert result.reward == float(value) + assert result.reward_override_ignored is None + + +def test_reward_file_that_cannot_be_removed_is_ignored(monkeypatch): + result = _rollout( + monkeypatch, + _Sandbox(removable=False), + ["true", "exit 1"], + agent_commands=[f"echo 1.0 > {REWARD_FILE}"], + ) + + assert result.reward == 0.5 + assert "could not be removed" in result.reward_override_ignored diff --git a/tests/envs/test_pi_reward_override.py b/tests/envs/test_pi_reward_override.py new file mode 100644 index 0000000000..e203fb12d9 --- /dev/null +++ b/tests/envs/test_pi_reward_override.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: BSD-3-Clause + +"""Pi rollouts must only take ``reward.txt`` from verification, not from the agent.""" + +from __future__ import annotations + +import os +import sys +from types import SimpleNamespace + +import pytest + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +for _p in (_REPO_ROOT, os.path.join(_REPO_ROOT, "envs")): + if _p not in sys.path: + sys.path.insert(0, _p) + +from pi_env.models import RolloutResult # noqa: E402 +from pi_env.server.pi_environment import PiEnvironment, REWARD_FILE # noqa: E402 + + +class _Sandbox: + """Sandbox whose reward file any command can write, as the agent's shell can.""" + + sandbox_id = "sbx-test" + + def __init__(self, *, removable: bool = True) -> None: + self.reward_file: str | None = None + self.removable = removable + + def exec(self, cmd: str, timeout: int | None = None) -> SimpleNamespace: + exit_code = 0 + if cmd.startswith("echo ") and cmd.endswith(f" > {REWARD_FILE}"): + self.reward_file = cmd[len("echo ") : -len(f" > {REWARD_FILE}")] + elif cmd.startswith(f"rm -f {REWARD_FILE}"): + if self.removable: + self.reward_file = None + exit_code = 0 if self.reward_file is None else 1 + elif cmd == "exit 1": + exit_code = 1 + return SimpleNamespace(exit_code=exit_code, stdout="", stderr="") + + def read_text(self, path: str) -> str: + if path == REWARD_FILE and self.reward_file is not None: + return self.reward_file + "\n" + raise FileNotFoundError(path) + + +class _Session: + def __init__(self, sandbox: _Sandbox, agent_commands: list[str]) -> None: + self.sandbox = sandbox + self._agent_commands = agent_commands + + def start_agent(self) -> None: + pass + + def wait_for_completion(self, timeout_s: float | None = None) -> int: + for cmd in self._agent_commands: + self.sandbox.exec(cmd) + return 0 + + def close(self) -> None: + pass + + +def _rollout(monkeypatch, sandbox, verify, agent_commands=()) -> RolloutResult: + monkeypatch.setenv("HF_TOKEN", "test") + env = PiEnvironment() + session = _Session(sandbox, list(agent_commands)) + monkeypatch.setattr( + env, + "_PiSessionFactory", + lambda **kwargs: SimpleNamespace(create=lambda **kw: session), + ) + monkeypatch.setattr(env, "_HFSandboxBackend", lambda **kwargs: None) + raw = env._run_rollout_impl( + base_url="http://localhost:8000/v1", + api_key="test", + model="test-model", + instruction="solve the task", + setup=[], + verify=verify, + task_id="t1", + mode="black_box", + disable_thinking=False, + max_tokens_cap=0, + top_logprobs=0, + agent_timeout_s=5, + image="", + ) + return RolloutResult.model_validate_json(raw) + + +def test_reward_file_written_by_the_agent_is_ignored(monkeypatch): + result = _rollout( + monkeypatch, + _Sandbox(), + ["true", "exit 1"], + agent_commands=[f"echo 1.0 > {REWARD_FILE}"], + ) + + assert result.error is None + assert result.reward == 0.5 + assert result.reward_override_ignored is None + + +def test_reward_file_written_by_a_verify_command_overrides_the_pass_rate(monkeypatch): + result = _rollout(monkeypatch, _Sandbox(), [f"echo 0.8 > {REWARD_FILE}", "exit 1"]) + + assert result.reward == 0.8 + assert result.reward_override_ignored is None + + +@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "passed"]) +def test_invalid_reward_file_falls_back_to_the_pass_rate(monkeypatch, value): + result = _rollout( + monkeypatch, _Sandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"] + ) + + assert result.reward == 0.5 + assert repr(value) in result.reward_override_ignored + + +@pytest.mark.parametrize("value", ["-0.5", "1.5"]) +def test_reward_file_outside_the_unit_interval_is_kept(monkeypatch, value): + result = _rollout( + monkeypatch, _Sandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"] + ) + + assert result.reward == float(value) + assert result.reward_override_ignored is None + + +def test_reward_file_that_cannot_be_removed_is_ignored(monkeypatch): + result = _rollout( + monkeypatch, + _Sandbox(removable=False), + ["true", "exit 1"], + agent_commands=[f"echo 1.0 > {REWARD_FILE}"], + ) + + assert result.reward == 0.5 + assert "could not be removed" in result.reward_override_ignored diff --git a/tests/envs/test_terminus_environment.py b/tests/envs/test_terminus_environment.py index baf7532069..f2cc83751c 100644 --- a/tests/envs/test_terminus_environment.py +++ b/tests/envs/test_terminus_environment.py @@ -2,6 +2,7 @@ from __future__ import annotations +import pytest from openenv.core.env_server.mcp_types import CallToolAction, ListToolsAction from terminus_env.models import TerminusState from terminus_env.server.e2b_sandbox import ShellResult @@ -145,3 +146,118 @@ def test_terminal_final_answer_runs_verify_commands(): "test -f answer.txt", "exit 1", ] + + +REWARD_FILE = "/home/user/logs/verifier/reward.txt" + + +class RewardFileSandbox(FakeSandbox): + """FakeSandbox whose reward file any shell command can write, as in E2B.""" + + def __init__(self, *, removable: bool = True) -> None: + super().__init__() + self.reward_file: str | None = None + self.removable = removable + + def run_shell(self, command: str, timeout_s: int = 120) -> ShellResult: + if command.startswith("echo ") and command.endswith(f" > {REWARD_FILE}"): + self.shell_commands.append(command) + self.reward_file = command[len("echo ") : -len(f" > {REWARD_FILE}")] + return ShellResult(stdout="", stderr="", error=None, success=True) + if command.startswith(f"rm -f {REWARD_FILE}"): + self.shell_commands.append(command) + if self.removable: + self.reward_file = None + gone = self.reward_file is None + return ShellResult( + stdout="", + stderr="", + error=None if gone else "SystemExit: 1", + success=gone, + ) + if command.startswith(f"cat {REWARD_FILE}"): + self.shell_commands.append(command) + return ShellResult( + stdout=self.reward_file or "", stderr="", error=None, success=True + ) + return super().run_shell(command, timeout_s) + + +def _submit(sandbox, verify_commands, agent_commands=()) -> TerminusState: + env = TerminusEnvironment() + env._sandbox = sandbox + env._state = TerminusState( + episode_id="episode-1", + sandbox_id="fake-sandbox", + verify_commands=verify_commands, + ) + for command in agent_commands: + env.step(CallToolAction(tool_name="terminal", arguments={"command": command})) + env.step(CallToolAction(tool_name="terminal", arguments={"final_answer": "done"})) + return env.state + + +def test_reward_file_written_by_the_agent_is_ignored(): + state = _submit( + RewardFileSandbox(), + ["true", "exit 1"], + agent_commands=[f"echo 1.0 > {REWARD_FILE}"], + ) + + assert state.last_reward == 0.5 + assert state.reward_override_ignored is None + + +def test_reward_file_is_cleared_again_before_a_second_submission(): + env = TerminusEnvironment() + env._sandbox = RewardFileSandbox() + env._state = TerminusState( + episode_id="episode-1", + sandbox_id="fake-sandbox", + verify_commands=["true", "exit 1"], + ) + + env.step(CallToolAction(tool_name="terminal", arguments={"final_answer": "a"})) + env.step( + CallToolAction( + tool_name="terminal", + arguments={"command": f"echo 1.0 > {REWARD_FILE}"}, + ) + ) + env.step(CallToolAction(tool_name="terminal", arguments={"final_answer": "b"})) + + assert env.state.last_reward == 0.5 + + +def test_reward_file_written_by_a_verify_command_overrides_the_pass_rate(): + state = _submit(RewardFileSandbox(), [f"echo 0.8 > {REWARD_FILE}", "exit 1"]) + + assert state.last_reward == 0.8 + assert state.reward_override_ignored is None + + +@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "passed"]) +def test_invalid_reward_file_falls_back_to_the_pass_rate(value): + state = _submit(RewardFileSandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"]) + + assert state.last_reward == 0.5 + assert repr(value) in state.reward_override_ignored + + +@pytest.mark.parametrize("value", ["-0.5", "1.5"]) +def test_reward_file_outside_the_unit_interval_is_kept(value): + state = _submit(RewardFileSandbox(), [f"echo {value} > {REWARD_FILE}", "exit 1"]) + + assert state.last_reward == float(value) + assert state.reward_override_ignored is None + + +def test_reward_file_that_cannot_be_removed_is_ignored(): + state = _submit( + RewardFileSandbox(removable=False), + ["true", "exit 1"], + agent_commands=[f"echo 1.0 > {REWARD_FILE}"], + ) + + assert state.last_reward == 0.5 + assert "could not be removed" in state.reward_override_ignored