diff --git a/README.md b/README.md index 325920d..b1fb8be 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,8 @@ connected through SSH local forwarding. ``` Harder multi-file cases (exception rename, dataclass field rename, tuple - return + facade, keyword-parameter rename, payload-key rename) are + return + facade, keyword-parameter rename, payload-key rename, + env-var rename) are offline by default. `--live` skips with exit 0 when Ollama is down; that skip is not a quality pass: diff --git a/docs/evaluation-protocol.md b/docs/evaluation-protocol.md index bdfb7ec..abe11b2 100644 --- a/docs/evaluation-protocol.md +++ b/docs/evaluation-protocol.md @@ -65,6 +65,7 @@ The same scorer runs against: | `widen_return_keep_facade` | `local_refactor` | `apply_discount` returns `(discounted, saved)`; `line_total` / `savings` stay ints | | `rename_kwarg_across_files` | `local_refactor` | Rename `send(subject=)` → `send(title=)`; no leftover parameter / `**kwargs` alias | | `rename_payload_key_across_files` | `local_refactor` | Rename payload key `user_id` → `account_id`; no leftover key / `.get()` alias | +| `rename_env_var_across_files` | `local_refactor` | Rename env name `APP_TOKEN` → `APP_SECRET`; no leftover getenv / `.get()` alias | | `implement_clamp` | `local_code` | Implement `clamp` from a spec, no starter file | | `explain_clamp` | `local_explain` | Prose: names the function and bounds; mentions clipping | | `review_login` | `local_review` | Prose first-pass: flags None and missing auth. **Not** an apply | @@ -81,10 +82,12 @@ tell layers apart: - leftover `QuotaError` alias or `years` property → structure fail - leftover `subject=` parameter / keyword alias → structure fail - leftover `user_id` payload key / `.get()` fallback → structure fail +- leftover `APP_TOKEN` env-name / `.get()` fallback → structure fail - exception threshold or greeting format drift → behavior fail - `apply_discount` still returns an int → structure fail; wrong `DISCOUNT_PERCENT` → behavior fail - `send` joins with `|` instead of `:` → behavior fail - `label` prefixes with `user:` instead of `acct:` → behavior fail +- `badge` prefixes with `env:` instead of `tok:` → behavior fail ## Measurement harness @@ -217,7 +220,7 @@ After repeated live runs, a paper may claim: - Layer-conditional rates on this corpus (format vs structure vs behavior). - That a vaguer prompt raises structure failures on the same oracle (`whitespace_extract` vs `whitespace_extract_vague`). -- That leftover type/field/keyword-parameter/payload-key aliases, catch-site drift, +- That leftover type/field/keyword-parameter/payload-key/env-name aliases, catch-site drift, and a helper signature change that breaks a public facade are distinguishable layers on the harder multi-file suite. - That shape-only test generation overstates success relative to diff --git a/docs/phase3-log.md b/docs/phase3-log.md index c4c852c..f77fe90 100644 --- a/docs/phase3-log.md +++ b/docs/phase3-log.md @@ -19,6 +19,7 @@ Cloud-safe stub + apply-gate run: | 2026-09-10 | local_refactor | fixtures | harder multi-file suite (3 cases) | scorer only | Offline corpus + runner; no live rate; Halo not started | | 2026-09-10 | local_refactor | fixtures | harder suite + `rename_kwarg_across_files` | scorer only | Fourth case; leftover `subject=` alias vs colon-join drift; no live rate | | 2026-09-10 | local_refactor | fixtures | harder suite + `rename_payload_key_across_files` | scorer only | Fifth case; leftover `user_id` key / `.get()` alias vs `acct:` prefix drift; no live rate | +| 2026-09-10 | local_refactor | fixtures | harder suite + `rename_env_var_across_files` | scorer only | Sixth case; leftover `APP_TOKEN` getenv / `.get()` alias vs `tok:` prefix drift; no live rate | Columns: diff --git a/src/local_coding_slm/eval/cases.py b/src/local_coding_slm/eval/cases.py index 27996a6..ff6d131 100644 --- a/src/local_coding_slm/eval/cases.py +++ b/src/local_coding_slm/eval/cases.py @@ -35,6 +35,9 @@ RENAME_PAYLOAD_ALIAS, RENAME_PAYLOAD_PARTIAL, RENAME_PAYLOAD_PREFIX, + RENAME_ENV_ALIAS, + RENAME_ENV_PARTIAL, + RENAME_ENV_PREFIX, WIDEN_RETURN_INT, WIDEN_RETURN_PARTIAL, WIDEN_RETURN_WRONG_RATE, @@ -609,4 +612,31 @@ class Fixture: False, "behavior", ), + Fixture( + "rename_env_golden", + "rename_env_var_across_files", + GOLDEN_FOR_CASE["rename_env_var_across_files"], + True, + ), + Fixture( + "rename_env_partial", + "rename_env_var_across_files", + RENAME_ENV_PARTIAL, + False, + "format", + ), + Fixture( + "rename_env_alias", + "rename_env_var_across_files", + RENAME_ENV_ALIAS, + False, + "structure", + ), + Fixture( + "rename_env_prefix", + "rename_env_var_across_files", + RENAME_ENV_PREFIX, + False, + "behavior", + ), ) diff --git a/src/local_coding_slm/eval/cases_harder.py b/src/local_coding_slm/eval/cases_harder.py index 2339631..7f71a33 100644 --- a/src/local_coding_slm/eval/cases_harder.py +++ b/src/local_coding_slm/eval/cases_harder.py @@ -3,7 +3,8 @@ These stay bounded (2–3 files). They isolate failure modes the seed and extended corpus do not: leftover type aliases, catch-site drift, field aliases, a leftover keyword-parameter alias, a leftover payload-key -alias, and a public facade that must survive a helper signature change. +alias, a leftover environment-variable name alias, and a public facade +that must survive a helper signature change. Golden strings are fixtures for the scorer and stub Ollama, not live model dumps. Live rates are not claimed here. @@ -21,6 +22,7 @@ "widen_return_keep_facade", "rename_kwarg_across_files", "rename_payload_key_across_files", + "rename_env_var_across_files", ) @@ -92,6 +94,22 @@ def _string_used(tree: ast.AST, value: str) -> bool: ) +def _assigned_str(tree: ast.AST, name: str) -> str | None: + if not isinstance(tree, ast.Module): + return None + for node in tree.body: + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == name + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ): + return node.value.value + return None + + def _function_returns_tuple(tree: ast.AST, name: str) -> bool: if not isinstance(tree, ast.Module): return False @@ -872,6 +890,123 @@ def _payload_key_renamed(by_path: dict[str, str]) -> str | None: return None +# --- rename env var APP_TOKEN -> APP_SECRET across names and badge --- + +NAMES_SOURCE = """\ +TOKEN_VAR = "APP_TOKEN" + + +def read(env: dict[str, str]) -> str: + return env[TOKEN_VAR] +""" + +BADGE_SOURCE = """\ +from names import TOKEN_VAR, read + + +def badge(name: str) -> str: + return f"tok:{read({TOKEN_VAR: name})}" +""" + +RENAME_ENV_TASK = ( + "Rename the environment variable name APP_TOKEN to APP_SECRET in " + "names.py and badge.py. Update TOKEN_VAR so it is the string " + "APP_SECRET only, and keep read() / badge() using TOKEN_VAR. " + "Do not keep APP_TOKEN as a constant, dict key, getenv/.get " + "fallback, or compatibility alias. Preserve badge('ada') == " + "'tok:ada', badge('') == 'tok:', and read({'APP_SECRET': 'ada'}) " + "== 'ada'. Return two fenced Python files with path comments " + "names.py and badge.py, no prose." +) + +RENAME_ENV_GOLDEN = '''\ +```python +# names.py +TOKEN_VAR = "APP_SECRET" + + +def read(env: dict[str, str]) -> str: + return env[TOKEN_VAR] +``` + +```python +# badge.py +from names import TOKEN_VAR, read + + +def badge(name: str) -> str: + return f"tok:{read({TOKEN_VAR: name})}" +``` +''' + +RENAME_ENV_PARTIAL = '''\ +```python +# names.py +TOKEN_VAR = "APP_SECRET" + + +def read(env: dict[str, str]) -> str: + return env[TOKEN_VAR] +``` +''' + +RENAME_ENV_ALIAS = '''\ +```python +# names.py +TOKEN_VAR = "APP_SECRET" + + +def read(env: dict[str, str]) -> str: + return env.get(TOKEN_VAR) or env.get("APP_TOKEN", "") +``` + +```python +# badge.py +from names import TOKEN_VAR, read + + +def badge(name: str) -> str: + return f"tok:{read({TOKEN_VAR: name}) or read({'APP_TOKEN': name})}" +``` +''' + +RENAME_ENV_PREFIX = '''\ +```python +# names.py +TOKEN_VAR = "APP_SECRET" + + +def read(env: dict[str, str]) -> str: + return env[TOKEN_VAR] +``` + +```python +# badge.py +from names import TOKEN_VAR, read + + +def badge(name: str) -> str: + return f"env:{read({TOKEN_VAR: name})}" +``` +''' + + +def _env_var_renamed(by_path: dict[str, str]) -> str | None: + try: + names = ast.parse(by_path.get("names.py", "")) + badge = ast.parse(by_path.get("badge.py", "")) + except SyntaxError as exc: + return f"unparseable Python: {exc.msg}" + if _assigned_str(names, "TOKEN_VAR") != "APP_SECRET": + return "names.py must assign TOKEN_VAR = 'APP_SECRET'" + for path, tree in (("names.py", names), ("badge.py", badge)): + if _string_used(tree, "APP_TOKEN") or _name_used(tree, "APP_TOKEN"): + return f"{path} still references APP_TOKEN; do not keep an env-name alias" + if not _name_used(badge, "TOKEN_VAR") and not _string_used(badge, "APP_SECRET"): + return "badge.py must use TOKEN_VAR or APP_SECRET" + return None + + HARDER_CASES: tuple[EvalCase, ...] = ( EvalCase( id="rename_exception_across_files", @@ -962,6 +1097,24 @@ def _payload_key_renamed(by_path: dict[str, str]) -> str | None: ), max_tokens=1200, ), + EvalCase( + id="rename_env_var_across_files", + tool="local_refactor", + task=RENAME_ENV_TASK, + files=( + {"path": "names.py", "content": NAMES_SOURCE}, + {"path": "badge.py", "content": BADGE_SOURCE}, + ), + required_paths=("names.py", "badge.py"), + required_top_level=("read", "badge"), + extra_structure=_env_var_renamed, + behavior=( + BehaviorCheck("badge", "badge", ("ada",), "tok:ada"), + BehaviorCheck("badge", "badge", ("",), "tok:"), + BehaviorCheck("names", "read", ({"APP_SECRET": "ada"},), "ada"), + ), + max_tokens=1200, + ), ) HARDER_GOLDEN = { @@ -970,6 +1123,7 @@ def _payload_key_renamed(by_path: dict[str, str]) -> str | None: "widen_return_keep_facade": WIDEN_RETURN_GOLDEN, "rename_kwarg_across_files": RENAME_KWARG_GOLDEN, "rename_payload_key_across_files": RENAME_PAYLOAD_GOLDEN, + "rename_env_var_across_files": RENAME_ENV_GOLDEN, } HARDER_OBSERVED_FIRST = { @@ -978,4 +1132,5 @@ def _payload_key_renamed(by_path: dict[str, str]) -> str | None: "widen_return_keep_facade": WIDEN_RETURN_PARTIAL, "rename_kwarg_across_files": RENAME_KWARG_PARTIAL, "rename_payload_key_across_files": RENAME_PAYLOAD_PARTIAL, + "rename_env_var_across_files": RENAME_ENV_PARTIAL, } diff --git a/tests/test_eval_harder.py b/tests/test_eval_harder.py index f300b21..3206474 100644 --- a/tests/test_eval_harder.py +++ b/tests/test_eval_harder.py @@ -21,7 +21,7 @@ def _fixture(name: str): class HarderMultifileCorpusTests(unittest.TestCase): def test_harder_cases_span_more_than_one_file(self) -> None: - self.assertEqual(len(HARDER_CASE_IDS), 5) + self.assertEqual(len(HARDER_CASE_IDS), 6) for case_id in HARDER_CASE_IDS: with self.subTest(case_id=case_id): case = CASES_BY_ID[case_id] @@ -36,6 +36,7 @@ def test_goldens_pass_all_four_layers(self) -> None: "widen_return_golden", "rename_kwarg_golden", "rename_payload_golden", + "rename_env_golden", ): with self.subTest(fixture=name): fixture = _fixture(name) @@ -51,6 +52,7 @@ def test_alias_and_int_return_are_structure_not_behavior(self) -> None: "widen_return_int", "rename_kwarg_alias", "rename_payload_alias", + "rename_env_alias", ): with self.subTest(fixture=name): fixture = _fixture(name) @@ -66,6 +68,7 @@ def test_threshold_format_and_rate_are_behavior(self) -> None: "widen_return_wrong_rate", "rename_kwarg_separator", "rename_payload_prefix", + "rename_env_prefix", ): with self.subTest(fixture=name): fixture = _fixture(name) @@ -99,6 +102,7 @@ def test_partial_files_are_format(self) -> None: "widen_return_partial", "rename_kwarg_partial", "rename_payload_partial", + "rename_env_partial", ): with self.subTest(fixture=name): fixture = _fixture(name)