diff --git a/docs/source/environments/jupyter.md b/docs/source/environments/jupyter.md index 36ab988120..bd5f1191f7 100644 --- a/docs/source/environments/jupyter.md +++ b/docs/source/environments/jupyter.md @@ -105,7 +105,43 @@ E2B starts each command as a login shell, so a profile the agent writes decides what a verify command reports: an `exit 0` or an `EXIT` trap in `/root/.bash_profile` makes a failing command look like a passing one. A test pins that behaviour rather than leaving it to be found in a reward curve, and -#1232 tracks verification outside the agent's sandbox, which is what closes it. +`submit_paths` closes that, per environment and per task. + +## Verifying outside the agent's sandbox + +A task that passes `submit_paths` to `reset` is verified in a second sandbox the +agent never touches. The list defaults to empty, and an empty list keeps the +behaviour above, so existing tasks score exactly as before. + +With paths declared, submitting runs this order, which is what makes it a +boundary rather than hardening: + +1. A fresh sandbox is created from the same template. +2. The task's `setup` commands run there, before any of the agent's work exists. +3. The declared paths are copied out of the agent's sandbox into `/submission`, + which is not the working directory and is not on `PATH` or `PYTHONPATH`. +4. The verify commands run, and the reward file is cleared and read there. +5. The sandbox is killed. + +Paths are relative to `/home/user`. An absolute path, one that leaves the +workspace with `..`, a symlink, a socket, a fifo, a device, or a submission over +200 files or 10 MiB is refused, and the reason is recorded in +`state.submission_refused`. Files that a shell or a test runner reads before the +first assertion are left behind even when a declared directory contains them: +the login files a shell reads (`.bash_profile`, `.bash_login`, `.profile`, +`.bashrc`), what Python runs before an import (`sitecustomize.py`, `*.pth`, +`conftest.py`), and every file pytest takes options from (`pytest.ini`, +`.pytest.ini`, `pytest.toml`, `.pytest.toml`, `pyproject.toml`, `setup.cfg`, +`tox.ini`), since any of them can carry `addopts = --co` and report success +without running a test. Declaring one of those exactly still submits it, since +naming it is the task author's decision. + +Because the submission is no longer in the workspace, a task that opts in writes +its verify commands against `/submission`. The task's own test files keep coming +from its `setup` commands, so nothing the agent submits can replace them. + +The cost is one extra sandbox per scored rollout, which is why it is opt-in +rather than the default. Only a command that ran and exited decides a verify result. If the sandbox itself fails, because it cannot be reached, the command cannot be started, or diff --git a/envs/jupyter_env/README.md b/envs/jupyter_env/README.md index ee61ffae90..3bb9631302 100644 --- a/envs/jupyter_env/README.md +++ b/envs/jupyter_env/README.md @@ -120,7 +120,43 @@ E2B starts each command as a login shell, so a profile the agent writes decides what a verify command reports: an `exit 0` or an `EXIT` trap in `/root/.bash_profile` makes a failing command look like a passing one. A test pins that behaviour rather than leaving it to be found in a reward curve, and -#1232 tracks verification outside the agent's sandbox, which is what closes it. +`submit_paths` closes that, per environment and per task. + +## Verifying outside the agent's sandbox + +A task that passes `submit_paths` to `reset` is verified in a second sandbox the +agent never touches. The list defaults to empty, and an empty list keeps the +behaviour above, so existing tasks score exactly as before. + +With paths declared, submitting runs this order, which is what makes it a +boundary rather than hardening: + +1. A fresh sandbox is created from the same template. +2. The task's `setup` commands run there, before any of the agent's work exists. +3. The declared paths are copied out of the agent's sandbox into `/submission`, + which is not the working directory and is not on `PATH` or `PYTHONPATH`. +4. The verify commands run, and the reward file is cleared and read there. +5. The sandbox is killed. + +Paths are relative to `/home/user`. An absolute path, one that leaves the +workspace with `..`, a symlink, a socket, a fifo, a device, or a submission over +200 files or 10 MiB is refused, and the reason is recorded in +`state.submission_refused`. Files that a shell or a test runner reads before the +first assertion are left behind even when a declared directory contains them: +the login files a shell reads (`.bash_profile`, `.bash_login`, `.profile`, +`.bashrc`), what Python runs before an import (`sitecustomize.py`, `*.pth`, +`conftest.py`), and every file pytest takes options from (`pytest.ini`, +`.pytest.ini`, `pytest.toml`, `.pytest.toml`, `pyproject.toml`, `setup.cfg`, +`tox.ini`), since any of them can carry `addopts = --co` and report success +without running a test. Declaring one of those exactly still submits it, since +naming it is the task author's decision. + +Because the submission is no longer in the workspace, a task that opts in writes +its verify commands against `/submission`. The task's own test files keep coming +from its `setup` commands, so nothing the agent submits can replace them. + +The cost is one extra sandbox per scored rollout, which is why it is opt-in +rather than the default. Only a command that ran and exited decides a verify result. If the sandbox itself fails, because it cannot be reached, the command cannot be started, or diff --git a/envs/jupyter_env/models.py b/envs/jupyter_env/models.py index 7bbc991256..ff5e5007f5 100644 --- a/envs/jupyter_env/models.py +++ b/envs/jupyter_env/models.py @@ -38,6 +38,9 @@ class JupyterState(State): setup_results: List[CommandResult] = Field(default_factory=list) verify_results: List[CommandResult] = Field(default_factory=list) verify_commands: List[str] = Field(default_factory=list) + setup_commands: List[str] = Field(default_factory=list) + submit_paths: List[str] = Field(default_factory=list) + submission_refused: List[str] = Field(default_factory=list) submitted_answer: Optional[str] = None last_reward: Optional[float] = None reward_override_ignored: Optional[str] = None diff --git a/envs/jupyter_env/server/e2b_sandbox.py b/envs/jupyter_env/server/e2b_sandbox.py index 9b4c26bc36..8722059003 100644 --- a/envs/jupyter_env/server/e2b_sandbox.py +++ b/envs/jupyter_env/server/e2b_sandbox.py @@ -203,6 +203,10 @@ def write_file(self, filename: str, content: bytes) -> None: """Upload a file into the sandbox filesystem.""" self._sbx.files.write(filename, content) + def read_file(self, filename: str) -> bytes: + """Read a file out of the sandbox filesystem.""" + return self._sbx.files.read(filename, format="bytes") + def kill(self) -> None: """Terminate the sandbox. Safe to call multiple times.""" try: diff --git a/envs/jupyter_env/server/jupyter_environment.py b/envs/jupyter_env/server/jupyter_environment.py index 0e0bfcae9a..963482ef97 100644 --- a/envs/jupyter_env/server/jupyter_environment.py +++ b/envs/jupyter_env/server/jupyter_environment.py @@ -22,9 +22,11 @@ from openenv.core.env_server.types import Action, Observation try: + from . import submission from .e2b_sandbox import E2BSandbox from .notebook_tracker import NotebookTracker except ImportError: # pragma: no cover + from server import submission from server.e2b_sandbox import E2BSandbox from server.notebook_tracker import NotebookTracker @@ -223,7 +225,7 @@ def reset( ) try: - self._sandbox = E2BSandbox(api_key=api_key) + self._sandbox = self._new_sandbox() except Exception as exc: # noqa: BLE001 self._state = self._JupyterState( episode_id=episode_id or str(uuid4()), @@ -251,6 +253,8 @@ def reset( kwargs.get("verify", kwargs.get("verify_scripts", [])) ) self._state.verify_commands = verify_commands + self._state.setup_commands = setup_commands + self._state.submit_paths = _coerce_commands(kwargs.get("submit_paths", [])) if setup_commands: self._sandbox.run_shell("mkdir -p /home/user/logs/verifier") @@ -390,21 +394,75 @@ def _run_shell_commands(self, commands: Iterable[str]): for command in commands ] + def _new_sandbox(self) -> E2BSandbox: + return E2BSandbox(api_key=os.environ["E2B_API_KEY"]) + + def _copy_submission(self, verifier: E2BSandbox) -> list[str]: + """Copy the declared paths into the verifier sandbox, return refusals.""" + refused: list[str] = [] + found: list[submission.SubmissionFile] = [] + for declared in self._state.submit_paths: + reason = submission.check_declared_path(declared) + if reason: + refused.append(reason) + continue + listing = self._sandbox.run_command(submission.listing_command(declared)) + files, refusals = submission.parse_listing(declared, listing.stdout) + found.extend(files) + refused.extend(refusals) + found, capped = submission.apply_caps(found) + refused.extend(capped) + + for entry in found: + content = self._sandbox.read_file( + f"{submission.WORKSPACE}/{entry.relative_path}" + ) + verifier.write_file( + f"{submission.SUBMISSION_DIR}/{entry.relative_path}", content + ) + return refused + + def _verify_in_fresh_sandbox(self) -> dict[str, Any]: + """Verify the submission in a sandbox the agent has never touched. + + The order is what makes it a boundary: the task's setup runs before the + submission arrives, the submission lands outside the workspace, and the + verify commands and the reward file belong to this sandbox alone. + """ + verifier = self._new_sandbox() + try: + setup_results = [ + _command_result_from_cell_result(command, verifier.run_command(command)) + for command in self._state.setup_commands + ] + failed = [result.command for result in setup_results if not result.success] + if failed: + raise RuntimeError( + f"setup failed in the verifier sandbox: {failed[0]!r}" + ) + + self._state.submission_refused = self._copy_submission(verifier) + return self._score(verifier) + finally: + verifier.kill() + def _run_verify_commands(self) -> dict[str, Any]: if not self._sandbox: return {"passed": 0, "total": 0, "reward": None} + if self._state.submit_paths: + return self._verify_in_fresh_sandbox() + return self._score(self._sandbox) + def _score(self, sandbox: E2BSandbox) -> dict[str, Any]: # Verification runs as separate sandbox processes, not in the notebook # kernel, so nothing a cell does to the kernel changes how verify # commands run or what they report. - self._sandbox.run_command("mkdir -p /home/user/logs/verifier") + sandbox.run_command("mkdir -p /home/user/logs/verifier") # The override is documented as verify-written, so drop anything the # policy wrote there during the episode before verification starts. - cleared = self._sandbox.run_command(_CLEAR_REWARD_FILE).success + cleared = sandbox.run_command(_CLEAR_REWARD_FILE).success verify_results = [ - _command_result_from_cell_result( - command, self._sandbox.run_command(command) - ) + _command_result_from_cell_result(command, sandbox.run_command(command)) for command in self._state.verify_commands ] self._state.verify_results = verify_results @@ -412,7 +470,7 @@ def _run_verify_commands(self) -> dict[str, Any]: passed = sum(1 for result in verify_results if result.success) total = len(verify_results) reward, self._state.reward_override_ignored = _read_reward_override( - self._sandbox, cleared + sandbox, cleared ) if reward is None and total: reward = passed / total diff --git a/envs/jupyter_env/server/submission.py b/envs/jupyter_env/server/submission.py new file mode 100644 index 0000000000..316895419b --- /dev/null +++ b/envs/jupyter_env/server/submission.py @@ -0,0 +1,142 @@ +"""Which of the agent's files may be handed to a verifier, and which may not. + +Verification in a fresh sandbox (#1232) copies the paths a task declares in +`submit_paths` out of the agent's sandbox into `/submission` there. The +submission is data: it is not the working directory and not on `PATH` or +`PYTHONPATH`. A task's own tests normally run against it, though, so a file a +shell or a test runner reads on its way to the first assertion is code rather +than data, and those are left behind. +""" + +from dataclasses import dataclass +from typing import List, Optional, Tuple + +SUBMISSION_DIR = "/submission" +WORKSPACE = "/home/user" + +# Read before anything a verify command asserts: a shell's login files, Python's +# import-time hooks, and a test runner's configuration and fixtures. +EXCLUDED_NAMES = frozenset( + { + # A login shell reads the first of these that it can. + ".bash_profile", + ".bash_login", + ".profile", + ".bashrc", + # Python runs these on the way to importing anything. + "sitecustomize.py", + "conftest.py", + # Every one of these can carry `addopts = --co`, which collects the + # tests without running them and reports success. + "pytest.ini", + ".pytest.ini", + "pytest.toml", + ".pytest.toml", + "pyproject.toml", + "setup.cfg", + "tox.ini", + } +) + +# A submission is an answer, not a dataset. +MAX_FILES = 200 +MAX_TOTAL_BYTES = 10 * 1024 * 1024 + + +@dataclass +class SubmissionFile: + """A regular file to copy, as a path relative to the workspace.""" + + relative_path: str + size: int + + +def check_declared_path(path: str) -> Optional[str]: + """Refuse a `submit_paths` entry, or return `None` to accept it. + + Args: + path (`str`): + The entry as the task declared it. + + Returns: + `str` or `None`: why the entry was refused, or `None` when it is usable. + + An excluded name is accepted here on purpose: declaring one exactly is the + task author choosing to submit it, while sweeping a directory that contains + one is an accident, which [`excluded_file`] refuses. + """ + if not path: + return "empty path" + if path.startswith("/"): + return f"not relative to {WORKSPACE}: {path!r}" + if ".." in path.split("/"): + return f"leaves the workspace: {path!r}" + return None + + +def excluded_file(relative_path: str) -> Optional[str]: + """Refuse a file found under a declared directory, or return `None`.""" + name = relative_path.rsplit("/", 1)[-1] + if name in EXCLUDED_NAMES or name.endswith(".pth"): + return f"runs before verification: {relative_path!r}" + return None + + +def listing_command(declared_path: str) -> str: + """Build the command that lists the regular files under a declared path. + + `-printf` is GNU find, which E2B's code-interpreter template has. The type + letter comes before the size so a symlink, socket, fifo or device is + reported rather than read: following one copies whatever it points at. + """ + target = f"{WORKSPACE}/{declared_path}" + return f"find '{target}' -maxdepth 8 -printf '%y\\t%s\\t%P\\n' 2>/dev/null || true" + + +def parse_listing( + declared_path: str, stdout: str +) -> Tuple[List[SubmissionFile], List[str]]: + """Turn one [`listing_command`] result into files to copy and refusals. + + Args: + declared_path (`str`): + The entry being expanded, used to build workspace-relative paths. + stdout (`str`): + Output of [`listing_command`]. + + Returns: + `tuple` with: + - `list[SubmissionFile]`: the regular files to copy. + - `list[str]`: one reason per refused entry. + """ + files: List[SubmissionFile] = [] + refusals: List[str] = [] + for line in stdout.splitlines(): + kind, _, rest = line.partition("\t") + size, _, below = rest.partition("\t") + if kind == "d": + continue + # find prints an empty %P for the path it was given, so an empty one + # means the declared path is itself the file. + relative = f"{declared_path}/{below}" if below else declared_path + if kind != "f": + refusals.append(f"not a regular file: {relative!r}") + continue + reason = excluded_file(relative) if below else None + if reason: + refusals.append(reason) + continue + files.append(SubmissionFile(relative_path=relative, size=int(size))) + if not files and not refusals: + refusals.append(f"nothing to submit at {declared_path!r}") + return files, refusals + + +def apply_caps(files: List[SubmissionFile]) -> Tuple[List[SubmissionFile], List[str]]: + """Keep a submission inside `MAX_FILES` and `MAX_TOTAL_BYTES`.""" + if len(files) > MAX_FILES: + return [], [f"more than {MAX_FILES} files declared: {len(files)}"] + total = sum(entry.size for entry in files) + if total > MAX_TOTAL_BYTES: + return [], [f"submission is larger than {MAX_TOTAL_BYTES} bytes: {total}"] + return files, [] diff --git a/tests/envs/test_jupyter_environment.py b/tests/envs/test_jupyter_environment.py index 1e3a3ae8b7..c3ab4f2eab 100644 --- a/tests/envs/test_jupyter_environment.py +++ b/tests/envs/test_jupyter_environment.py @@ -5,10 +5,13 @@ import contextlib import io import subprocess +import sys +from pathlib import Path from types import SimpleNamespace import pytest from jupyter_env.models import JupyterState +from jupyter_env.server import submission from jupyter_env.server.e2b_sandbox import CellResult, E2BSandbox from jupyter_env.server.jupyter_environment import JupyterEnvironment from openenv.core.env_server.mcp_types import CallToolAction, ListToolsAction @@ -428,10 +431,16 @@ def run(self, cmd, background=None, user=None, cwd=None, timeout=None, **kwargs) "timeout": timeout, } ) + mapped = cmd.replace("/home/user", self.home).replace( + "/submission", f"{self.home}/submission" + ) done = self._run( - ["/bin/bash", "-l", "-c", cmd.replace("/home/user", self.home)], + ["/bin/bash", "-l", "-c", mapped], cwd=(cwd or "").replace("/home/user", self.home) or None, - env={"PATH": "/usr/bin:/bin", "HOME": self.home}, + env={ + "PATH": f"{Path(sys.executable).parent}:/usr/bin:/bin", + "HOME": self.home, + }, capture_output=True, text=True, ) @@ -466,11 +475,42 @@ def kill(self) -> bool: return self._kill_result -def _environment_on_real_sandbox(home, verify_commands): +class _Files: + """E2B's file API over the temporary directory standing in for the sandbox.""" + + def __init__(self, home) -> None: + self.home = str(home) + + def _local(self, path: str) -> Path: + if path.startswith("/submission"): + return Path(self.home) / "submission" / path[len("/submission/") :] + return Path(path.replace("/home/user", self.home, 1)) + + def read(self, path: str, format: str = "text"): + data = self._local(path).read_bytes() + return data if format == "bytes" else data.decode() + + def write(self, path: str, content) -> None: + target = self._local(path) + target.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + target.write_bytes(content) + else: + target.write_text(content) + + +def _sandbox_on(home, sandbox_id="fake-sandbox"): kernel, processes = _Kernel(), _Processes(home) sandbox = E2BSandbox.__new__(E2BSandbox) - sandbox._sbx = SimpleNamespace(run_code=kernel.run_code, commands=processes) - sandbox.sandbox_id = "fake-sandbox" + sandbox._sbx = SimpleNamespace( + run_code=kernel.run_code, commands=processes, files=_Files(home) + ) + sandbox.sandbox_id = sandbox_id + return sandbox, kernel, processes + + +def _environment_on_real_sandbox(home, verify_commands): + sandbox, kernel, processes = _sandbox_on(home) env = JupyterEnvironment() env._sandbox = sandbox env._state = JupyterState( @@ -481,6 +521,24 @@ def _environment_on_real_sandbox(home, verify_commands): return env, kernel, processes +def _environment_with_a_verifier_sandbox( + agent_home, verifier_home, verify_commands, submit_paths, setup_commands=() +): + """An episode whose verification runs in a second, untouched sandbox. + + `/submission` and `/tests` live outside the mapped workspace, so the + verifier's temporary directory is its parent here. + """ + env, kernel, processes = _environment_on_real_sandbox(agent_home, verify_commands) + env._state.submit_paths = list(submit_paths) + env._state.setup_commands = list(setup_commands) + verifier, verifier_kernel, verifier_processes = _sandbox_on( + verifier_home, "fake-verifier-sandbox" + ) + env._new_sandbox = lambda: verifier + return env, processes, verifier, verifier_processes + + def _run_cell(env, code): env.step( CallToolAction(tool_name="add_and_execute_code_cell", arguments={"code": code}) @@ -672,3 +730,259 @@ def test_a_poisoned_login_profile_still_defeats_a_failing_verify_command( # The verify command fails, the profile reports success for it anyway. assert [result.success for result in env.state.verify_results] == [True] assert env.state.last_reward == 1.0 + + +# The cases @zuraiz-anjum ran against `coding_tools_env` on #1232. The task +# ships its own tests from setup; the agent submits an answer and nothing else. +_WRONG_ANSWER = """\ +def add(a, b): + return a - b +""" +_RIGHT_ANSWER = """\ +def add(a, b): + return a + b +""" +_TASK_TESTS = """\ +from solution import add + + +def test_add(): + assert add(2, 3) == 5 +""" +_CHEATING_CONFTEST = """\ +import pytest + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + outcome.get_result().outcome = "passed" +""" + + +def _setup_commands(): + """What the task does in whichever sandbox verification runs in.""" + return [ + f"mkdir -p /home/user/tests && cat > /home/user/tests/test_add.py <<'PYEOF'\n{_TASK_TESTS}PYEOF" + ] + + +def _verify_commands(submission_subdir=""): + source = f"/submission/{submission_subdir}".rstrip("/") + return [ + f"cd /home/user/tests && cp {source}/* . 2>/dev/null; " + "python -m pytest -q -p no:cacheprovider test_add.py" + ] + + +def _write_cell(path, text): + return f"open({str(path)!r}, 'w').write({text!r})" + + +def _submission_episode( + tmp_path, cells, submit_paths, submission_subdir="", setup=None +): + agent_home, verifier_home = tmp_path / "agent", tmp_path / "verifier" + agent_home.mkdir() + verifier_home.mkdir() + env, processes, verifier, verifier_processes = _environment_with_a_verifier_sandbox( + agent_home, + verifier_home, + _verify_commands(submission_subdir), + submit_paths, + setup_commands=_setup_commands() if setup is None else setup, + ) + for code in cells: + _run_cell(env, code) + _submit_answer(env) + return env, agent_home, verifier_home, verifier_processes + + +def test_a_wrong_answer_scores_zero_in_a_fresh_sandbox(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + agent = tmp_path / "agent" + env, _, _, _ = _submission_episode( + tmp_path, + [_write_cell(agent / "solution.py", _WRONG_ANSWER)], + submit_paths=("solution.py",), + ) + + assert env.state.last_reward == 0.0 + assert env.state.submission_refused == [] + + +def test_a_right_answer_scores_one_in_a_fresh_sandbox(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + agent = tmp_path / "agent" + env, _, verifier_home, _ = _submission_episode( + tmp_path, + [_write_cell(agent / "solution.py", _RIGHT_ANSWER)], + submit_paths=("solution.py",), + ) + + assert env.state.last_reward == 1.0 + assert (verifier_home / "submission/solution.py").read_text() == _RIGHT_ANSWER + + +def test_a_cheating_conftest_under_a_declared_directory_is_left_behind( + monkeypatch, tmp_path +): + """The case that takes a wrong answer to 1.0 when the verifier reads a + `conftest.py` the agent wrote.""" + monkeypatch.chdir(tmp_path) + agent = tmp_path / "agent" + env, _, verifier_home, _ = _submission_episode( + tmp_path, + [ + f"import os\nos.makedirs({str(agent / 'work')!r}, exist_ok=True)", + _write_cell(agent / "work/solution.py", _WRONG_ANSWER), + _write_cell(agent / "work/conftest.py", _CHEATING_CONFTEST), + ], + submit_paths=("work",), + submission_subdir="work", + ) + + assert env.state.last_reward == 0.0 + assert any("conftest.py" in reason for reason in env.state.submission_refused) + assert not (verifier_home / "submission/work/conftest.py").exists() + + +def test_a_rewritten_task_test_does_not_replace_the_one_setup_wrote( + monkeypatch, tmp_path +): + """Setup runs before the submission arrives, and the submission lands + outside the workspace, so the task's own test file is what runs.""" + monkeypatch.chdir(tmp_path) + agent = tmp_path / "agent" + env, _, verifier_home, _ = _submission_episode( + tmp_path, + [ + _write_cell(agent / "solution.py", _WRONG_ANSWER), + f"import os\nos.makedirs({str(agent / 'tests')!r}, exist_ok=True)", + _write_cell( + agent / "tests/test_add.py", "def test_add():\n assert True\n" + ), + ], + submit_paths=("solution.py", "tests"), + ) + + assert env.state.last_reward == 0.0 + assert (verifier_home / "tests/test_add.py").read_text() == _TASK_TESTS + assert (verifier_home / "submission/tests/test_add.py").exists() + + +def test_a_poisoned_login_profile_does_not_reach_a_fresh_sandbox(monkeypatch, tmp_path): + """What the fresh sandbox buys over #1233: the poisoned profile is in the + sandbox the verifier never uses.""" + monkeypatch.chdir(tmp_path) + agent_home, verifier_home = tmp_path / "agent", tmp_path / "verifier" + agent_home.mkdir() + verifier_home.mkdir() + env, _, _, _ = _environment_with_a_verifier_sandbox( + agent_home, verifier_home, ["exit 1"], ["answer.txt"] + ) + + _run_cell(env, _write_cell(agent_home / ".bash_profile", _POISONED_PROFILE)) + _run_cell(env, _write_cell(agent_home / "answer.txt", "42\n")) + _submit_answer(env) + + assert [result.success for result in env.state.verify_results] == [False] + assert env.state.last_reward == 0.0 + + +@pytest.mark.parametrize( + "declared", ["/etc/passwd", "../outside.txt", "work/../../outside.txt"] +) +def test_a_path_outside_the_workspace_is_refused(monkeypatch, tmp_path, declared): + monkeypatch.chdir(tmp_path) + agent = tmp_path / "agent" + env, _, _, _ = _submission_episode( + tmp_path, + [_write_cell(agent / "solution.py", _RIGHT_ANSWER)], + submit_paths=("solution.py", declared), + ) + + assert any(declared in reason for reason in env.state.submission_refused) + assert env.state.last_reward == 1.0 + + +def test_a_symlink_is_reported_rather_than_followed(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + agent = tmp_path / "agent" + env, _, verifier_home, _ = _submission_episode( + tmp_path, + [ + _write_cell(agent / "solution.py", _RIGHT_ANSWER), + f"import os\nos.symlink('/etc/passwd', {str(agent / 'secrets.txt')!r})", + ], + submit_paths=("solution.py", "secrets.txt"), + ) + + assert any("not a regular file" in r for r in env.state.submission_refused) + assert not (verifier_home / "submission/secrets.txt").exists() + + +def test_no_declared_paths_keeps_verification_in_the_agent_sandbox( + monkeypatch, tmp_path +): + """The opt-in guarantee: a task that declares nothing scores as it did.""" + home = tmp_path / "home" + home.mkdir() + monkeypatch.chdir(tmp_path) + env, _, _ = _environment_on_real_sandbox(home, ["true", "exit 1"]) + + def no_second_sandbox(): + raise AssertionError("verification started a sandbox it should not have") + + env._new_sandbox = no_second_sandbox + _submit_answer(env) + + assert env.state.last_reward == 0.5 + assert env.state.submission_refused == [] + + +def test_setup_failing_in_the_verifier_sandbox_is_not_a_reward(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + agent_home, verifier_home = tmp_path / "agent", tmp_path / "verifier" + agent_home.mkdir() + verifier_home.mkdir() + env, _, _, _ = _environment_with_a_verifier_sandbox( + agent_home, verifier_home, ["true"], ["answer.txt"], setup_commands=["exit 3"] + ) + + _run_cell(env, _write_cell(agent_home / "answer.txt", "42\n")) + obs = env.step( + CallToolAction(tool_name="final_answer", arguments={"answer": "done"}) + ) + + assert "setup failed in the verifier sandbox" in obs.error.message + assert env.state.verify_results == [] + assert env.state.last_reward is None + + +@pytest.mark.parametrize( + "name", + [ + ".bash_profile", + ".bash_login", + ".profile", + ".bashrc", + "sitecustomize.py", + "answer.pth", + "conftest.py", + "pytest.ini", + ".pytest.ini", + "pytest.toml", + ".pytest.toml", + "pyproject.toml", + "setup.cfg", + "tox.ini", + ], +) +def test_a_file_read_before_the_first_assertion_stays_behind(name): + """@zuraiz-anjum measured the pytest half of this list on #1579: each of + those files can carry `addopts = --co`, which collects without running and + reports success.""" + assert submission.excluded_file(f"work/{name}") + # Naming one exactly is the task author's decision, so it is still allowed. + assert submission.check_declared_path(f"work/{name}") is None