Run coding_tools_env verification outside the notebook kernel

docs/source/environments/coding_tools.md

@@ -22,6 +22,28 @@ negative rewards and values above 1 are accepted too. A value that is not a
 finite number is ignored, the pass rate is used, and the reason is recorded in
 `state.reward_override_ignored`.
 
+Verify commands, the clearing of that file and the read of it run as their own
+sandbox processes through E2B's process API, as `root` in `/home/user`, rather
+than as a `subprocess.run` cell in the code-interpreter kernel. Nothing done to
+the kernel, such as a rebound `subprocess.run`, a changed working directory or
+an edited `os.environ`, reaches verification. This removes the coupling to the
+kernel; it does not isolate verification from the agent. The agent's `bash`
+tool runs as root in the same sandbox, so it can still change anything there,
+including the files a verify command reads and the startup files of the shell
+it runs in. E2B starts each command as a login shell, so a profile the agent
+writes decides what a verify command reports: an `exit 0` or an `EXIT` trap in
+`/root/.bash_profile` makes a failing command look like a passing one. A test
+pins that behaviour, and #1232 tracks verification outside the agent's sandbox,
+which is what closes it.
+
+Only a command that ran and exited decides a verify result. If the sandbox
+itself fails, because it cannot be reached, the command cannot be started, or
+the wait for it ends without an exit status (a verify command that outlives its
+120 s timeout, for example), the error is raised instead: verification stops,
+no reward is produced, and the episode does not finish. A command with no exit
+status may still be running, since E2B's kill signals the command's own process
+and not the children it started.
+
 ## Tools
 
 - `bash(command, timeout=30)`

envs/coding_tools_env/README.md

@@ -38,6 +38,28 @@ negative rewards and values above 1 are accepted too. A value that is not a
 finite number is ignored, the pass rate is used, and the reason is recorded in
 `state.reward_override_ignored`.
 
+Verify commands, the clearing of that file and the read of it run as their own
+sandbox processes through E2B's process API, as `root` in `/home/user`, rather
+than as a `subprocess.run` cell in the code-interpreter kernel. Nothing done to
+the kernel, such as a rebound `subprocess.run`, a changed working directory or
+an edited `os.environ`, reaches verification. This removes the coupling to the
+kernel; it does not isolate verification from the agent. The agent's `bash`
+tool runs as root in the same sandbox, so it can still change anything there,
+including the files a verify command reads and the startup files of the shell
+it runs in. E2B starts each command as a login shell, so a profile the agent
+writes decides what a verify command reports: an `exit 0` or an `EXIT` trap in
+`/root/.bash_profile` makes a failing command look like a passing one. A test
+pins that behaviour, and #1232 tracks verification outside the agent's sandbox,
+which is what closes it.
+
+Only a command that ran and exited decides a verify result. If the sandbox
+itself fails, because it cannot be reached, the command cannot be started, or
+the wait for it ends without an exit status (a verify command that outlives its
+120 s timeout, for example), the error is raised instead: verification stops,
+no reward is produced, and the episode does not finish. A command with no exit
+status may still be running, since E2B's kill signals the command's own process
+and not the children it started.
+
 ## Tools
 
 - `bash(command, timeout=30)`

envs/coding_tools_env/server/coding_tools_env_environment.py

@@ -375,14 +375,17 @@ def _record(
     def _run_verify_commands(self) -> dict[str, Any]:
         if not self._sandbox:
             return {"passed": 0, "total": 0, "reward": None}
-        self._sandbox.run_shell("mkdir -p /home/user/logs/verifier")
+        # Verification runs as separate sandbox processes, not in the
+        # code-interpreter kernel, so nothing done to the kernel changes how
+        # verify commands run or what they report.
+        self._sandbox.run_command("mkdir -p /home/user/logs/verifier")
         # The override is documented as verify-written, so drop anything the
         # policy wrote there during the episode before verification starts.
-        cleared = self._sandbox.run_shell(_CLEAR_REWARD_FILE).ok
+        cleared = self._sandbox.run_command(_CLEAR_REWARD_FILE).ok
         self._state.verify_results = []
         passed = 0
         for command in self._state.verify_commands:
-            result = self._sandbox.run_shell(command, timeout_s=120)
+            result = self._sandbox.run_command(command, timeout_s=120)
             record = CommandResult(
                 tool="verify",
                 ok=result.ok,
@@ -416,10 +419,9 @@ def _read_reward_override(
 ) -> tuple[float | None, str | None]:
     if not cleared:
         return None, "reward file could not be removed before verification"
-    result = sandbox.read_file(REWARD_FILE)
-    if not result.ok:
-        return None, None
-    return _parse_reward_override(result.output or "")
+    result = sandbox.run_command(f"cat {REWARD_FILE} 2>/dev/null || true")
+    # stdout alone: `output` also carries stderr, such as a login profile's.
+    return _parse_reward_override(result.metadata.get("stdout", ""))
 
 
 def _parse_reward_override(raw: str) -> tuple[float | None, str | None]:

envs/coding_tools_env/server/e2b_sandbox.py

@@ -8,6 +8,7 @@
 
 from __future__ import annotations
 
+import contextlib
 import json
 from dataclasses import dataclass
 from typing import Any
@@ -20,6 +21,13 @@
     _E2B_IMPORT_ERROR = _e2b_import_error
     Sandbox = None  # type: ignore[assignment]
 
+# E2B's default code-interpreter template runs its Jupyter server, and so the
+# kernel ``run_shell`` goes through, as root with the notebook in /home/user.
+# ``run_command`` keeps the same user and directory, so verify commands see the
+# same permissions they did when they ran from the kernel.
+_COMMAND_USER = "root"
+_COMMAND_CWD = "/home/user"
+
 
 @dataclass
 class ToolResult:
@@ -29,6 +37,22 @@ class ToolResult:
     metadata: dict[str, Any] | None = None
 
 
+def _command_result(
+    stdout: str | None, stderr: str | None, exit_code: int
+) -> ToolResult:
+    """Describe a command that ran and exited, in the shape `run_shell` uses."""
+    ok = exit_code == 0
+    output = "\n".join(
+        part for part in [(stdout or "").strip(), (stderr or "").strip()] if part
+    )
+    return ToolResult(
+        ok=ok,
+        output=output,
+        error=None if ok else f"exit_code={exit_code}",
+        metadata={"exit_code": exit_code, "stdout": stdout or ""},
+    )
+
+
 class E2BSandbox:
     """One E2B sandbox per OpenEnv episode."""
 
@@ -64,6 +88,69 @@ def run_shell(self, command: str, timeout_s: float = 30) -> ToolResult:
             metadata={"exit_code": rc},
         )
 
+    def run_command(self, command: str, timeout_s: float = 30) -> ToolResult:
+        """
+        Execute a shell command as its own sandbox process, outside the kernel.
+
+        ``run_shell`` sends a ``subprocess.run`` cell to the code-interpreter
+        kernel, so its result depends on the kernel's state: a rebound
+        ``subprocess.run``, a changed working directory, edited ``os.environ``.
+        That is fine for the agent's own ``bash`` tool and wrong for
+        verification. This goes through E2B's process API instead and reports
+        the command's real exit status.
+
+        It removes the coupling to the kernel. It is not an isolation boundary:
+        the agent's shell runs as root in the same sandbox, so it can still
+        change what any process there sees, including this command's login
+        shell.
+
+        Args:
+            command (`str`):
+                Shell command to run.
+            timeout_s (`float`, *optional*, defaults to `30`):
+                Seconds to wait for the command before giving up on it.
+
+        Returns:
+            [`ToolResult`]: `ok` is `True` only for a zero exit status.
+
+        Raises:
+            `Exception`: whatever the SDK raises when the command cannot be
+                started, or when the wait for it ends without an exit status.
+                Only a command that ran and exited is reported as a result:
+                a sandbox that is unreachable, expired or unauthenticated says
+                nothing about the work being verified.
+        """
+        # Started in the background so there is a handle to kill if the wait
+        # fails: E2B keeps a command running when its connection drops.
+        handle = self._sbx.commands.run(
+            command,
+            background=True,
+            user=_COMMAND_USER,
+            cwd=_COMMAND_CWD,
+            timeout=timeout_s,
+        )
+        try:
+            result = handle.wait()
+        except Exception as exc:
+            # The SDK raises CommandExitException for a command that ran and
+            # exited non-zero. It is told apart by its exit code rather than by
+            # class so that this module keeps importing, and the tests keep
+            # driving it, without the SDK installed.
+            exit_code = getattr(exc, "exit_code", None)
+            if exit_code is not None:
+                return _command_result(
+                    getattr(exc, "stdout", ""), getattr(exc, "stderr", ""), exit_code
+                )
+            # No exit status: the command timed out, or the connection to it
+            # broke. Ask E2B to stop it, but a kill signals the command's own
+            # process, so a child of it can stay alive and still write to the
+            # sandbox. Nothing here can call the command finished, so the
+            # failure is raised rather than counted as a verify result.
+            with contextlib.suppress(Exception):
+                handle.kill()
+            raise
+        return _command_result(result.stdout, result.stderr, result.exit_code)
+
     def read_file(
         self, file_path: str, offset: int | None = None, limit: int | None = None
     ) -> ToolResult: