From 7517ea0cd47634a9c7380e3204931b0e0c878b76 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 02:33:47 -0600
Subject: [PATCH 1/4] Run a cost-capped smoke eval on main when a prompt,
 charter or model changes

Adds the Smoke eval workflow, --max-cost, --subset and --precision off to evals/run.ts, and evals/smoke.ts to compare a run with evals/baseline-smoke.json and scan evals/out for secrets before it's uploaded.

Refs #52
---
 .github/workflows/smoke-eval.yml |  76 +++++++++++++
 CHANGELOG.md                     |   1 +
 README.md                        |  11 +-
 evals/cases.ts                   |   9 ++
 evals/compare.ts                 |   5 +-
 evals/run.ts                     |  99 +++++++++++++----
 evals/smoke.ts                   | 150 +++++++++++++++++++++++++
 test/evals.test.ts               |   9 ++
 test/smoke-eval.test.ts          | 182 +++++++++++++++++++++++++++++++
 9 files changed, 519 insertions(+), 23 deletions(-)
 create mode 100644 .github/workflows/smoke-eval.yml
 create mode 100644 evals/smoke.ts
 create mode 100644 test/smoke-eval.test.ts

diff --git a/.github/workflows/smoke-eval.yml b/.github/workflows/smoke-eval.yml
new file mode 100644
index 0000000..8a2acf6
--- /dev/null
+++ b/.github/workflows/smoke-eval.yml
@@ -0,0 +1,76 @@
+# A small, cost-capped eval on main whenever a prompt, a charter, a model or the guidance changes, so a regression shows
+# before a release reaches the repositories that use the action. The full eval stays on demand (README, Evals).
+#
+# It runs the smoke subset in evals/cases.ts with the production guidance, stops and fails past $8, and compares the run
+# with evals/baseline-smoke.json in the job summary: it fails when caught drops by more than 2 or the must-not case is
+# flagged. With no baseline yet, the run is uploaded as the proposed one and the job passes with a notice.
+#
+# It uses the organization's Bedrock key in the code-review environment, which only the default branch can use. Never
+# add a pull_request trigger: the key would then run code from any pull request.
+name: Smoke eval
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'prompts/**'
+      - 'reviewers/**'
+      - 'src/config.ts'
+      - 'guidance/**'
+  workflow_dispatch:
+
+permissions: {}
+
+# A run costs money, so a newer push waits for the running one instead of cancelling it.
+concurrency:
+  group: smoke-eval
+  cancel-in-progress: false
+
+jobs:
+  smoke-eval:
+    name: Smoke eval
+    runs-on: ubuntu-latest
+    environment: code-review
+    timeout-minutes: 90
+    permissions:
+      contents: read
+      # The eval reads the issues a case's pull request closes.
+      issues: read
+    env:
+      LABEL: smoke-${{ github.sha }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version: 24
+      - run: npm ci --ignore-scripts --no-audit --no-fund
+
+      # The precision judge's Opus calls are left out: the gate reads caught and the must-not case only.
+      - name: Run the smoke subset
+        env:
+          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
+          GH_TOKEN: ${{ github.token }}
+        run: node evals/run.ts --subset smoke --guidance full --precision off --max-cost 8 --label "$LABEL"
+
+      - name: Compare with the baseline
+        run: node evals/smoke.ts check "evals/out/$LABEL.json"
+
+      # Artifacts on a public repository are public: nothing is uploaded unless the scan passes.
+      - name: Scan the outputs for secrets
+        id: scan
+        if: ${{ !cancelled() }}
+        env:
+          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
+          GH_TOKEN: ${{ github.token }}
+        run: node evals/smoke.ts scan evals/out
+
+      - name: Upload the outputs
+        if: ${{ !cancelled() && steps.scan.outcome == 'success' }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: smoke-eval
+          path: evals/out/
+          retention-days: 30
+          if-no-files-found: ignore
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 49cf6ab..acd43f3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ### Added
 
+- A cost-capped smoke eval: the Smoke eval workflow runs 4 eval cases, among them the must-not case and a large diff, on every push to `main` that changes a prompt, a charter, `src/config.ts` or the guidance, and by hand, in the `code-review` environment and never on a pull request. It compares the run with `evals/baseline-smoke.json` in the job summary, fails when caught drops by more than 2 or the must-not case is flagged, and uploads `evals/out` once a secret scan passes. Until a baseline is committed, it passes with a notice and uploads the run as the proposed one. `node evals/run.ts` gains `--max-cost <usd>`, which stops and fails the run once it costs more, `--subset <name>` for a named set of cases, and `--precision off`; `--only` now fails on a case that doesn't exist ([#52](https://github.com/NCodeGroup/code-review/issues/52)).
 - Evals measure precision: each posted blocker and should-fix that matches no expected concern is judged `real`, `not-real` or `unsure` by Opus reading the checkout, with verdicts cached in `evals/verdicts.json`. Each output also reports findings per case (mean, max) and the review's cost per case. `npm run eval:compare` compares two runs, marking recall differences of 2 or less as noise, and `npm run eval:confirm` adds confirmed verdicts to `cases.json` as expected or must-not-flag concerns ([#51](https://github.com/NCodeGroup/code-review/issues/51)).
 - A per-review cost ceiling guards against a runaway review: $8 plus $4 for each part of the diff, overridable with `CODE_REVIEW_MAX_RUN_USD`. Reviewer calls not started once it is reached are skipped and shown as `skipped: run budget` in the check run, synthesis still runs on what was found, and the summary warns that the review is incomplete ([#50](https://github.com/NCodeGroup/code-review/issues/50)).
 - A verification stage, off by default: with `CODE_REVIEW_VERIFICATION=on`, a Sonnet call with the checkout proves or refutes each merged blocker and should-fix, a file's findings at a time at up to $0.50 a call. Confirmed findings get the code path in their evidence, refuted ones are dropped and listed with the reason under "Refuted by verification" in the check run and in the trace, and ones it can't settle become questions one severity lower. The summary names the refuted findings, and verification turns a request for changes with nothing serious left into a comment, never an approval. Nits aren't verified, and the calls show as `verification` in the cost table. It stays off until the eval shows a gain, and the eval report now counts posted blockers and should-fixes and refuted findings ([#41](https://github.com/NCodeGroup/code-review/issues/41)).
diff --git a/README.md b/README.md
index 14d8cd6..3b4075e 100644
--- a/README.md
+++ b/README.md
@@ -170,7 +170,16 @@ The cases come from review comments the learned guidance has read too, so the ev
 
 **Precision.** Every posted blocker and should-fix that matches no expected concern goes to an Opus judge, which reads the case's checkout, locked down like the reviewers, with the finding passed as data, and answers `real`, `not-real` or `unsure`: is it a real defect the maintainer would act on? A finding that raises a must-not-flag concern counts as not real without a call. Precision is (matched + real) ÷ all posted blockers and should-fixes, with `unsure` counted apart. Verdicts are cached by case and finding fingerprint (file, severity and the title's words) in `evals/verdicts.json`, so a rerun judges only findings it hasn't seen; commit it with `cases.json` so every run is judged the same way. Each `evals/out/<label>.json` has the precision counts per case and in `summary.precision`, findings per case (`summary.findingsPerCase`: mean, max) and the review's own cost, without the eval's judges (`reviewCostUsd`, and `summary.reviewCostPerCase`).
 
-**Comparing two setups.** `npm run eval:compare -- <a.json> <b.json>` (or bare labels from `evals/out/`) prints recall, precision, findings per pull request and cost for both runs, then each case's differences. A recall difference of 2 or less is marked `(noise)`, within one run's variance; the other differences aren't marked, since their variance hasn't been measured. Outputs from before precision show `—` for it.
+**Comparing two setups.** `npm run eval:compare -- <a.json> <b.json>` (or bare labels from `evals/out/`) prints recall, precision, findings per pull request and cost for both runs, then each case's differences. A recall difference of 2 or less is marked `(noise)`, within one run's variance; the other differences aren't marked, since their variance hasn't been measured. Outputs from before precision, or from a run with `--precision off`, which skips the precision judge's Opus calls, show `—` for it.
+
+**Capping the cost.** `--max-cost <usd>` stops a run once its cost, judges included, passes that many dollars: the case that crossed it finishes, the rest don't start, the output covers the cases that ran, and the run fails. `--subset <name>` runs a named set of cases from `evals/cases.ts` instead of `--only id,id`; a name that matches no case fails the run.
+
+**Smoke eval.** The [Smoke eval](.github/workflows/smoke-eval.yml) workflow runs on every push to `main` that changes `prompts/`, `reviewers/`, `src/config.ts` or `guidance/`, and by hand (`gh workflow run smoke-eval.yml`). It runs in the `code-review` environment with the organization's Bedrock key, so it never runs on a pull request. It runs the `smoke` subset, 4 cases with the must-not case and one large diff, with the production guidance, `--precision off` and `--max-cost 8`. `node evals/smoke.ts check` then compares the run with `evals/baseline-smoke.json`, posts the comparison to the job summary, and fails when caught drops by more than 2, when the must-not case is flagged, or when the two cover different cases. `evals/out` is scanned for secrets before it's uploaded as the `smoke-eval` artifact, since artifacts on a public repository are public. With no baseline yet, the run passes with a notice, failing only on a flagged must-not case. To adopt a run as the baseline, the first time or after an intended change in what the smoke cases catch or a change to the subset, take `proposed-baseline-smoke.json` from that run's artifact, which is the run without its agent calls and raw findings, and commit it in a pull request:
+
+```powershell
+gh run download <run id> -n smoke-eval -D evals/out/smoke
+Copy-Item evals/out/smoke/proposed-baseline-smoke.json evals/baseline-smoke.json
+```
 
 **Growing the set from verdicts.** `npm run eval:confirm` lists the `real` and `not-real` verdicts nobody has confirmed yet (`--verdict real`, `--verdict unsure` or `--sample 5` of each narrow it), each with the command that confirms it. `npm run eval:confirm -- --case <id> --fingerprint <fp>` adds a `real` finding to its case in `cases.json` as an expected concern, and a `not-real` one as a must-not-flag concern, and marks the verdict confirmed. `--as expect` or `--as must-not` overrides the judge, and is needed for an `unsure` one. Review the new concern's wording in `cases.json` before committing it. In PowerShell, quote the separator: `npm run eval:confirm '--' --case <id> --fingerprint <fp>`.
 
diff --git a/evals/cases.ts b/evals/cases.ts
index e2b799f..271fc26 100644
--- a/evals/cases.ts
+++ b/evals/cases.ts
@@ -16,3 +16,12 @@ export interface EvalCase {
   mustNotFlag: Concern[];
   source: string;
 }
+
+// Named subsets of evals/cases.json, for node evals/run.ts --subset <name>.
+// smoke: what .github/workflows/smoke-eval.yml runs on main when a prompt, a charter, a model or the guidance changes,
+// under --max-cost 8: the must-not case (an accepted decision the reviewers must not flag again), one large diff (43
+// files, about 2,600 changed lines), and two small ones in C# and TypeScript. Changing it needs a new
+// evals/baseline-smoke.json.
+export const subsets: Record<string, readonly string[]> = {
+  smoke: ['accepted-no-backfill', 'heap-allocation-key-export', 'secret-buffer-length', 'test-gap-newest-summary-selection'],
+};
diff --git a/evals/compare.ts b/evals/compare.ts
index 1068be0..2d5a4cb 100644
--- a/evals/compare.ts
+++ b/evals/compare.ts
@@ -19,7 +19,7 @@ export interface EvalOutput {
   cases: {
     id: string;
     verdicts: { caught: boolean; caughtRaw: boolean }[];
-    forbidden: { raisedBy: string[] }[];
+    forbidden: { concern?: string; raisedBy: string[] }[];
     findings: unknown[];
     precision?: PrecisionCounts;
     reviewCostUsd?: number;
@@ -113,7 +113,8 @@ export function compare(a: EvalOutput, b: EvalOutput, names: [string, string]):
   const notes = [`A recall difference of ${noiseLimit} or less is within one run's variance: (noise). Other differences aren't flagged; their variance hasn't been measured.`];
   const onlyIn = (from: Map<string, unknown>, other: Map<string, unknown>) => [...from.keys()].filter((id) => !other.has(id));
   if (onlyIn(ca, cb).length || onlyIn(cb, ca).length) notes.push(`The runs cover different cases; totals aren't comparable. Only in ${names[0]}: ${onlyIn(ca, cb).join(', ') || 'none'}. Only in ${names[1]}: ${onlyIn(cb, ca).join(', ') || 'none'}.`);
-  if (ta.precision === undefined || tb.precision === undefined) notes.push('A run without precision fields predates the precision judge: — marks what it lacks, and its review cost includes the eval\'s judges.');
+  if (ta.precision === undefined || tb.precision === undefined) notes.push('— marks the precision a run lacks: it predates the precision judge or ran with --precision off.');
+  if (a.summary.reviewCostUsd === undefined || b.summary.reviewCostUsd === undefined) notes.push('A run without reviewCostUsd predates it: its review cost includes the eval\'s judges.');
   return `${overall}\n\n${perCase}\n\n${notes.join('\n')}\n`;
 }
 
diff --git a/evals/run.ts b/evals/run.ts
index 7587a3b..3b694b4 100644
--- a/evals/run.ts
+++ b/evals/run.ts
@@ -1,11 +1,16 @@
 // Replays the eval cases through the review pipeline and scores them: each expected concern must be raised, and no
 // must-not-flag concern may be. Runs locally on Bedrock with AWS_BEARER_TOKEN_BEDROCK.
 //
-//   node evals/run.ts [--only id,id] [--tiers "conventions=haiku"] [--label name] [--guidance full]
+//   node evals/run.ts [--only id,id | --subset smoke] [--tiers "conventions=haiku"] [--label name] [--guidance full]
+//                     [--precision off] [--max-cost 8]
 //
 // Writes evals/out/<label>.json with every finding, the judge's verdicts, precision, findings per case and the cost.
 // Unmatched blockers and should-fixes are judged for precision (evals/precision.ts); verdicts are cached in
-// evals/verdicts.json. npm run eval:compare compares two outputs; npm run eval:confirm adds verdicts to the cases.
+// evals/verdicts.json. --precision off skips that judge and its Opus calls. npm run eval:compare compares two outputs;
+// npm run eval:confirm adds verdicts to the cases. --subset names a set of cases in evals/cases.ts.
+//
+// --max-cost stops the run once its cost, judges included, passes that many dollars: the case that crossed it finishes,
+// the rest don't start, the output covers the cases that ran, and the run fails.
 //
 // A concern with a minSeverity counts as raised only by a finding at that severity or above: an expected should-fix
 // raised as a nit is missed, and a must-not concern at should-fix may still be raised as a nit.
@@ -19,7 +24,7 @@ import { verificationOn } from '../src/config.ts';
 import { closedIssues, gather, splitDiff, type PullRequest } from '../src/gather.ts';
 import { startProxy } from '../src/proxy.ts';
 import { loadCharters, review, severityRank, type Finding } from '../src/review.ts';
-import type { Concern, EvalCase } from './cases.ts';
+import { subsets, type Concern, type EvalCase } from './cases.ts';
 import { judgePosted, loadVerdicts, percent, realityJudge, saveVerdicts, spread, totalPrecision, type CasePrecision } from './precision.ts';
 
 interface Verdict { concern: string; caught: boolean; caughtRaw: boolean; by: string[] }
@@ -99,15 +104,58 @@ async function raised<F extends Finding>(run: AgentRun, concern: Concern, findin
   return by;
 }
 
+// The --max-cost value in dollars, or undefined when the flag is absent. A flag without a positive number fails before
+// anything is spent, instead of running uncapped.
+export function parseMaxCost(value: string | undefined): number | undefined {
+  if (value === undefined) return undefined;
+  const text = value.trim();
+  if (!/^\d+(\.\d+)?$/.test(text) || Number(text) <= 0) throw new Error(`--max-cost takes a positive number of dollars, such as 8, not ${JSON.stringify(value.slice(0, 40))}.`);
+  return Number(text);
+}
+
+// The cases --only or --subset names, in cases.json's order. A name that matches no case fails, so a typo or a renamed
+// case can't quietly shrink the run.
+export function selectCases(all: EvalCase[], only: string | undefined, subset: string | undefined): EvalCase[] {
+  if (only !== undefined && subset !== undefined) throw new Error('Use --only or --subset, not both.');
+  let ids: readonly string[] | undefined = only?.split(',').map((id) => id.trim()).filter(Boolean);
+  if (subset !== undefined) {
+    ids = Object.hasOwn(subsets, subset) ? subsets[subset] : undefined;
+    if (!ids) throw new Error(`No subset named ${JSON.stringify(subset)}; evals/cases.ts has ${Object.keys(subsets).join(', ')}.`);
+  }
+  if (!ids) return all;
+  const missing = ids.filter((id) => !all.some((c) => c.id === id));
+  if (missing.length) throw new Error(`No eval case named ${missing.join(', ')}.`);
+  return all.filter((c) => ids.includes(c.id));
+}
+
+// Runs the cases in order, each returning what it cost, and stops once the total passes maxCostUsd: the case that
+// crossed it finishes, since a review stopped partway has nothing to score, and the rest don't start.
+export async function runCapped<T>(
+  items: readonly T[],
+  runOne: (item: T) => Promise<number>,
+  maxCostUsd?: number,
+): Promise<{ costUsd: number; ran: number; exceeded: boolean }> {
+  let costUsd = 0, ran = 0;
+  for (const item of items) {
+    costUsd += await runOne(item);
+    ran++;
+    if (maxCostUsd !== undefined && costUsd > maxCostUsd) return { costUsd, ran, exceeded: true };
+  }
+  return { costUsd, ran, exceeded: false };
+}
+
 // How the output names a finding that raised a concern.
 const raisedLabel = (finding: Finding & { reviewers?: string[] }) => `${finding.file}:${finding.line} (${(finding.reviewers ?? []).join(', ')})`;
 
 async function main(): Promise<void> {
+  const maxCost = parseMaxCost(process.argv.includes('--max-cost') ? (arg('max-cost') ?? '') : undefined);
+  const precisionSetting = arg('precision') ?? 'on';
+  if (precisionSetting !== 'on' && precisionSetting !== 'off') throw new Error(`--precision takes on or off, not ${JSON.stringify(precisionSetting.slice(0, 40))}.`);
+  const precisionOn = precisionSetting === 'on';
   const key = process.env['CODE_REVIEW_BEDROCK_KEY'] ?? process.env['AWS_BEARER_TOKEN_BEDROCK'];
   if (!key) throw new Error('Set AWS_BEARER_TOKEN_BEDROCK to run the evals on Bedrock.');
   if (arg('tiers')) process.env['CODE_REVIEW_TIERS'] = arg('tiers');
-  const only = arg('only')?.split(',');
-  const cases = (JSON.parse(readFileSync(path.join(root, 'evals', 'cases.json'), 'utf8')) as EvalCase[]).filter((c) => !only || only.includes(c.id));
+  const cases = selectCases(JSON.parse(readFileSync(path.join(root, 'evals', 'cases.json'), 'utf8')) as EvalCase[], arg('only'), arg('subset'));
   const label = arg('label') ?? new Date().toISOString().replace(/[:.]/g, '-');
   const charters = loadCharters(root);
   // The held-out guidance was trained without the eval cases' own pull requests and decision records, so a catch
@@ -125,8 +173,9 @@ async function main(): Promise<void> {
   const findingsPerCase: number[] = [];
   const reviewCosts: number[] = [];
 
+  let capped: Awaited<ReturnType<typeof runCapped>>;
   try {
-    for (const c of cases) {
+    capped = await runCapped(cases, async (c) => {
       const repo = clone(c.repository);
       const head = worktree(repo, `${c.id}-head`, c.head);
       const base = worktree(repo, `${c.id}-base`, c.base);
@@ -164,16 +213,20 @@ async function main(): Promise<void> {
       }
 
       const diffs = splitDiff(gathered.diff);
-      const precision = await judgePosted({
-        caseId: c.id,
-        findings: outcome.findings,
-        matched,
-        mustNot,
-        cache: verdictCache,
-        judge: realityJudge({ ...run, checkout: head }, { title: c.title, diffFor: (file) => diffs.get(file) }, calls),
-      });
-      saveVerdicts(verdictFile, verdictCache);
-      precisions.push(precision);
+      const precision = precisionOn
+        ? await judgePosted({
+          caseId: c.id,
+          findings: outcome.findings,
+          matched,
+          mustNot,
+          cache: verdictCache,
+          judge: realityJudge({ ...run, checkout: head }, { title: c.title, diffFor: (file) => diffs.get(file) }, calls),
+        })
+        : undefined;
+      if (precision) {
+        saveVerdicts(verdictFile, verdictCache);
+        precisions.push(precision);
+      }
       findingsPerCase.push(outcome.findings.length);
       const reviewCost = outcome.calls.reduce((sum, call) => sum + call.costUsd, 0);
       reviewCosts.push(reviewCost);
@@ -191,25 +244,31 @@ async function main(): Promise<void> {
         ...verdicts.map((v) => (v.caught ? 'caught' : v.caughtRaw ? 'dropped by synthesis' : 'missed')),
         ...forbidden.map((f) => (f.raisedBy.length ? 'FLAGGED (must not)' : 'stayed quiet')),
       ].join(', ');
-      console.log(`${c.id}: ${status}; ${outcome.findings.length} findings, precision ${percent(precision.precision)} (${precision.unsure} unsure), $${caseCost.toFixed(2)}`);
+      const precisionText = precision ? `, precision ${percent(precision.precision)} (${precision.unsure} unsure)` : '';
+      console.log(`${c.id}: ${status}; ${outcome.findings.length} findings${precisionText}, $${caseCost.toFixed(2)}`);
       report.push({ id: c.id, verdicts, forbidden, precision, findings: outcome.findings, refuted: outcome.refuted, raw, calls, reviewCostUsd: reviewCost, costUsd: caseCost });
-    }
+      return caseCost;
+    }, maxCost);
   } finally {
     await proxy.close();
   }
 
-  const precision = totalPrecision(precisions);
+  const precision = precisionOn ? totalPrecision(precisions) : undefined;
   const summary = {
     label, tiers: process.env['CODE_REVIEW_TIERS'] ?? '', verification: verificationOn() ? 'on' : 'off', expected, caught, caughtRaw, violations,
     findings: findingCount, blockersAndShouldFixes: seriousCount, refuted: refutedCount,
     precision, findingsPerCase: spread(findingsPerCase), reviewCostUsd: reviewCosts.reduce((a, b) => a + b, 0),
     reviewCostPerCase: spread(reviewCosts), costUsd: cost,
+    ...(maxCost === undefined ? {} : { maxCostUsd: maxCost, casesRun: capped.ran, casesSelected: cases.length }),
   };
   console.log(`\nCaught ${caught}/${expected} (reviewers found ${caughtRaw}/${expected}); must-not violations ${violations}; ${findingCount} findings in all, ${seriousCount} blockers and should-fixes, ${refutedCount} refuted by verification; $${cost.toFixed(2)}`);
-  console.log(`Precision ${percent(precision.precision)}: ${precision.matched} matched + ${precision.real} real of ${precision.posted} blockers and should-fixes; ${precision.unsure} unsure, ${precision.notReal} not real, ${precision.mustNot} must-not`);
+  if (precision) console.log(`Precision ${percent(precision.precision)}: ${precision.matched} matched + ${precision.real} real of ${precision.posted} blockers and should-fixes; ${precision.unsure} unsure, ${precision.notReal} not real, ${precision.mustNot} must-not`);
   console.log(`Findings per case: mean ${summary.findingsPerCase.mean.toFixed(1)}, max ${summary.findingsPerCase.max}; review cost per case: mean $${summary.reviewCostPerCase.mean.toFixed(2)}, max $${summary.reviewCostPerCase.max.toFixed(2)}`);
   mkdirSync(path.join(root, 'evals', 'out'), { recursive: true });
   writeFileSync(path.join(root, 'evals', 'out', `${label}.json`), JSON.stringify({ summary, cases: report }, null, 2));
+  if (capped.exceeded) {
+    throw new Error(`Stopped over --max-cost: the run cost $${capped.costUsd.toFixed(2)}, more than $${maxCost?.toFixed(2)}, after ${capped.ran} of ${cases.length} cases. evals/out/${label}.json has the cases that ran.`);
+  }
 }
 
 if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
diff --git a/evals/smoke.ts b/evals/smoke.ts
new file mode 100644
index 0000000..a55f273
--- /dev/null
+++ b/evals/smoke.ts
@@ -0,0 +1,150 @@
+// The smoke eval's gate, run by .github/workflows/smoke-eval.yml after node evals/run.ts --subset smoke.
+//
+//   node evals/smoke.ts check <run.json> [--baseline <file>]
+//     Compares the run with evals/baseline-smoke.json in eval:compare's table and fails when caught drops by more than
+//     2, when a must-not-flag concern was raised, or when the two cover different cases. With no baseline yet it fails
+//     only on a must-not-flag concern, with a notice. Writes the table to $GITHUB_STEP_SUMMARY (stdout outside
+//     Actions), and the run, trimmed to what compare reads, to proposed-baseline-smoke.json beside it.
+//   node evals/smoke.ts scan <dir>
+//     Fails when a file under <dir> holds one of the job's secrets or anything shaped like a credential, before the
+//     workflow uploads the folder: artifacts on a public repository are public.
+
+import { appendFileSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { assertNoSecrets } from '../src/outputs.ts';
+import { compare, noiseLimit, readOutput, table, type EvalOutput } from './compare.ts';
+
+export const baselineFile = 'evals/baseline-smoke.json';
+export const proposedName = 'proposed-baseline-smoke.json';
+// The environment variables that may hold a secret of the smoke eval's job.
+export const secretVariables = ['AWS_BEARER_TOKEN_BEDROCK', 'CODE_REVIEW_BEDROCK_KEY', 'GH_TOKEN', 'GITHUB_TOKEN'];
+
+export interface SmokeVerdict { failures: string[]; notices: string[]; markdown: string }
+
+const caseIds = (output: EvalOutput) => output.cases.map((c) => c.id).sort();
+
+// The run alone, for when there is no baseline to compare it with.
+function runTable(run: EvalOutput): string {
+  const rows = run.cases.map((c) => [
+    c.id,
+    `${c.verdicts.filter((v) => v.caught).length}/${c.verdicts.length}`,
+    String(c.forbidden.filter((f) => f.raisedBy.length).length),
+    String(c.findings.length),
+    `$${(c.reviewCostUsd ?? c.costUsd).toFixed(2)}`,
+  ]);
+  const s = run.summary;
+  return `${table([['Case', 'caught', 'must-not raised', 'findings', 'review cost'], ...rows])}\n\nCaught ${s.caught}/${s.expected}; must-not violations ${s.violations}; review cost $${(s.reviewCostUsd ?? s.costUsd).toFixed(2)}, with judges $${s.costUsd.toFixed(2)}.\n`;
+}
+
+// Model output reaches the summary only inside a code block, which a run of backticks could otherwise close.
+const fenced = (text: string) => `\`\`\`text\n${text.replace(/`{3,}/g, (ticks) => "'".repeat(ticks.length)).trimEnd()}\n\`\`\`\n`;
+
+export function smokeCheck(run: EvalOutput, baseline: EvalOutput | undefined): SmokeVerdict {
+  const failures: string[] = [];
+  const notices: string[] = [];
+  for (const c of run.cases) {
+    for (const f of c.forbidden.filter((x) => x.raisedBy.length)) {
+      failures.push(`${c.id} raised a must-not-flag concern (by ${f.raisedBy.join('; ')})${f.concern ? `: ${f.concern}` : '.'}`);
+    }
+  }
+  if (baseline) {
+    const [ours, theirs] = [caseIds(run), caseIds(baseline)];
+    if (ours.join(',') !== theirs.join(',')) {
+      failures.push(`The run covers ${ours.join(', ')} but ${baselineFile} covers ${theirs.join(', ')}, so caught can't be compared. Adopt this run's proposed baseline once it looks right.`);
+    } else {
+      const drop = baseline.summary.caught - run.summary.caught;
+      if (drop > noiseLimit) failures.push(`Caught dropped by ${drop}, from ${baseline.summary.caught}/${baseline.summary.expected} to ${run.summary.caught}/${run.summary.expected}: more than the ${noiseLimit} a run varies by.`);
+    }
+  } else {
+    notices.push(`There is no ${baselineFile} yet, so caught wasn't compared. This run is the proposed baseline: ${proposedName} in the smoke-eval artifact. The README's Smoke eval section says how to adopt it.`);
+  }
+  const status = failures.length ? '**Failed.**' : notices.length ? '**Passed** with no baseline to compare with.' : '**Passed.**';
+  const messages = [...failures, ...notices];
+  const body = baseline ? compare(baseline, run, ['baseline', 'this run']) : runTable(run);
+  const markdown = `## Smoke eval\n\n${status}\n\n${messages.length ? fenced(messages.join('\n')) + '\n' : ''}${fenced(body)}`;
+  return { failures, notices, markdown };
+}
+
+// The run with only what compare and the check read, for checking in as evals/baseline-smoke.json: no agent calls,
+// raw reviewer findings or refutations.
+export function trimmed(output: EvalOutput): EvalOutput {
+  return {
+    summary: output.summary,
+    cases: output.cases.map(({ id, verdicts, forbidden, findings, precision, reviewCostUsd, costUsd }) => ({
+      id, verdicts, forbidden, findings,
+      ...(precision === undefined ? {} : { precision }),
+      ...(reviewCostUsd === undefined ? {} : { reviewCostUsd }),
+      costUsd,
+    })),
+  };
+}
+
+// Scans every file under dir and returns how many there were; a missing dir has none. Anything but a regular file or a
+// folder fails, so a link can't take the upload outside what was scanned.
+export function scanDir(dir: string, knownSecrets: string[]): number {
+  let entries;
+  try {
+    entries = readdirSync(dir, { recursive: true, withFileTypes: true });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 0;
+    throw error;
+  }
+  let files = 0;
+  for (const entry of entries) {
+    const file = path.relative(dir, path.join(entry.parentPath, entry.name));
+    if (entry.isDirectory()) continue;
+    if (!entry.isFile()) throw new Error(`${file} is not a regular file; only files and folders are uploaded.`);
+    assertNoSecrets(readFileSync(path.join(dir, file), 'utf8'), knownSecrets, file);
+    files++;
+  }
+  return files;
+}
+
+// Text for a workflow command's message, which ends at a line break.
+export const commandData = (text: string) => text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
+
+function readBaseline(file: string): EvalOutput | undefined {
+  try {
+    return readOutput(file);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
+    throw error;
+  }
+}
+
+function main(): void {
+  const [command, target] = process.argv.slice(2);
+  const annotate = (kind: 'notice' | 'error', message: string) =>
+    console.log(process.env['GITHUB_ACTIONS'] === 'true' ? `::${kind} title=Smoke eval::${commandData(message)}` : `${kind}: ${message}`);
+  if (command === 'check' && target) {
+    const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+    const index = process.argv.indexOf('--baseline');
+    const baselinePath = index >= 0 && process.argv[index + 1] ? path.resolve(process.argv[index + 1] ?? '') : path.join(root, baselineFile);
+    const runFile = path.resolve(target);
+    const run = readOutput(runFile);
+    const verdict = smokeCheck(run, readBaseline(baselinePath));
+    writeFileSync(path.join(path.dirname(runFile), proposedName), `${JSON.stringify(trimmed(run), null, 2)}\n`);
+    const summaryFile = process.env['GITHUB_STEP_SUMMARY'];
+    if (summaryFile) appendFileSync(summaryFile, verdict.markdown);
+    else process.stdout.write(verdict.markdown);
+    for (const notice of verdict.notices) annotate('notice', notice);
+    for (const failure of verdict.failures) annotate('error', failure);
+    if (verdict.failures.length) process.exitCode = 1;
+  } else if (command === 'scan' && target) {
+    const secrets = secretVariables.flatMap((name) => process.env[name] ?? []);
+    const files = scanDir(path.resolve(target), secrets);
+    console.log(`Scanned ${files} files in ${target}: no secrets.`);
+  } else {
+    throw new Error('Usage: node evals/smoke.ts check <run.json> [--baseline <file>] | scan <dir>');
+  }
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+  try {
+    main();
+  } catch (error) {
+    console.error((error as Error).message);
+    process.exitCode = 1;
+  }
+}
diff --git a/test/evals.test.ts b/test/evals.test.ts
index 6f525cb..4e5ea59 100644
--- a/test/evals.test.ts
+++ b/test/evals.test.ts
@@ -227,6 +227,15 @@ test('compare reads an output from before precision was measured, and says which
   assert.match(text, /^two\s+—\s+0\/1\s+—/m);
   assert.match(text, /Only in old: none\. Only in new: two\./);
   assert.match(text, /predates the precision judge/);
+  assert.match(text, /reviewCostUsd predates it: its review cost includes the eval's judges/);
+});
+
+test('compare says a run without precision may have run with --precision off, and keeps its review cost apart', () => {
+  const off = output([caseOutput('one', 1, 3, 4, 2)], false);
+  const text = compare(off, output([caseOutput('one', 1, 3, 4, 2, 0.5)], true), ['off', 'on']);
+  assert.match(text, /^Precision\s+—\s+50%\s+—$/m);
+  assert.match(text, /ran with --precision off/);
+  assert.doesNotMatch(text, /reviewCostUsd predates it/);
 });
 
 test('outputPath takes a file path as given and a bare label from evals/out', () => {
diff --git a/test/smoke-eval.test.ts b/test/smoke-eval.test.ts
new file mode 100644
index 0000000..6206022
--- /dev/null
+++ b/test/smoke-eval.test.ts
@@ -0,0 +1,182 @@
+import assert from 'node:assert/strict';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { test } from 'node:test';
+import type { EvalCase } from '../evals/cases.ts';
+import type { EvalOutput } from '../evals/compare.ts';
+import { parseMaxCost, runCapped, selectCases } from '../evals/run.ts';
+import { commandData, scanDir, smokeCheck, trimmed } from '../evals/smoke.ts';
+import { SecretFoundError } from '../src/outputs.ts';
+
+const root = path.resolve(import.meta.dirname, '..');
+const allCases = JSON.parse(readFileSync(path.join(root, 'evals', 'cases.json'), 'utf8')) as EvalCase[];
+
+test('parseMaxCost reads a positive number of dollars and is absent without the flag', () => {
+  assert.equal(parseMaxCost(undefined), undefined);
+  assert.equal(parseMaxCost('8'), 8);
+  assert.equal(parseMaxCost(' 2.5 '), 2.5);
+});
+
+test('parseMaxCost refuses a flag without a positive number, instead of running uncapped', () => {
+  for (const value of ['', '0', '-1', 'eight', '8usd', 'Infinity']) {
+    assert.throws(() => parseMaxCost(value), /--max-cost takes a positive number of dollars/, value);
+  }
+});
+
+// A runner whose cases cost what the table says, recording the ones it started.
+function fakeCases(costs: number[]): { started: number[]; runOne: (index: number) => Promise<number> } {
+  const started: number[] = [];
+  return { started, runOne: async (index) => { started.push(index); return costs[index] ?? 0; } };
+}
+
+test('runCapped stops after the case that passes the cap, and does not start the rest', async () => {
+  const { started, runOne } = fakeCases([3, 3, 3]);
+  const result = await runCapped([0, 1, 2], runOne, 5);
+  assert.deepEqual(started, [0, 1]);
+  assert.deepEqual(result, { costUsd: 6, ran: 2, exceeded: true });
+});
+
+test('runCapped counts a run whose last case passes the cap as exceeded', async () => {
+  const { started, runOne } = fakeCases([3, 3, 3]);
+  assert.deepEqual(await runCapped([0, 1, 2], runOne, 8), { costUsd: 9, ran: 3, exceeded: true });
+  assert.deepEqual(started, [0, 1, 2]);
+});
+
+test('runCapped runs every case when the total stays at or under the cap, or without one', async () => {
+  assert.deepEqual(await runCapped([0, 1], fakeCases([4, 4]).runOne, 8), { costUsd: 8, ran: 2, exceeded: false });
+  assert.deepEqual(await runCapped([0, 1], fakeCases([40, 40]).runOne), { costUsd: 80, ran: 2, exceeded: false });
+});
+
+test('the smoke subset is 4 existing cases, with the must-not case', () => {
+  const smoke = selectCases(allCases, undefined, 'smoke');
+  assert.equal(smoke.length, 4);
+  assert.ok(smoke.some((c) => c.mustNotFlag.length > 0 && c.expect.length === 0), 'a must-not case');
+  assert.ok(smoke.some((c) => c.id === 'heap-allocation-key-export'), 'the large-diff case');
+});
+
+test('selectCases fails on a case or subset that does not exist, and on --only with --subset', () => {
+  assert.throws(() => selectCases(allCases, 'accepted-no-backfill,no-such-case', undefined), /No eval case named no-such-case/);
+  assert.throws(() => selectCases(allCases, undefined, 'nightly'), /No subset named "nightly"/);
+  assert.throws(() => selectCases(allCases, undefined, '__proto__'), /No subset named/);
+  assert.throws(() => selectCases(allCases, 'accepted-no-backfill', 'smoke'), /--only or --subset, not both/);
+  assert.deepEqual(selectCases(allCases, 'secret-buffer-length, accepted-no-backfill', undefined).map((c) => c.id), ['secret-buffer-length', 'accepted-no-backfill']);
+  assert.equal(selectCases(allCases, undefined, undefined).length, allCases.length);
+});
+
+const smokeCase = (id: string, caught: number, expected: number, raisedBy: string[] = []): EvalOutput['cases'][number] => ({
+  id,
+  verdicts: Array.from({ length: expected }, (_, i) => ({ caught: i < caught, caughtRaw: i < caught })),
+  forbidden: id === 'must-not' ? [{ concern: 'Do not ask for a backfill.', raisedBy }] : [],
+  findings: [{}, {}],
+  reviewCostUsd: 1,
+  costUsd: 1.25,
+});
+
+function smokeOutput(cases: EvalOutput['cases']): EvalOutput {
+  return {
+    summary: {
+      expected: cases.reduce((n, c) => n + c.verdicts.length, 0),
+      caught: cases.reduce((n, c) => n + c.verdicts.filter((v) => v.caught).length, 0),
+      caughtRaw: 0,
+      violations: cases.filter((c) => c.forbidden.some((f) => f.raisedBy.length)).length,
+      costUsd: cases.length * 1.25,
+      reviewCostUsd: cases.length,
+    },
+    cases,
+  };
+}
+
+const baseline = smokeOutput([smokeCase('must-not', 0, 0), smokeCase('large', 4, 4), smokeCase('small', 2, 2)]);
+
+test('smokeCheck passes a drop in caught of 2, within a run\'s variance, and shows the comparison', () => {
+  const verdict = smokeCheck(smokeOutput([smokeCase('must-not', 0, 0), smokeCase('large', 2, 4), smokeCase('small', 2, 2)]), baseline);
+  assert.deepEqual(verdict.failures, []);
+  assert.deepEqual(verdict.notices, []);
+  assert.match(verdict.markdown, /^## Smoke eval\n\n\*\*Passed\.\*\*/);
+  assert.match(verdict.markdown, /^Recall \(caught\)\s+6\/6 \(100%\)\s+4\/6 \(67%\)\s+-2 \(noise\)$/m);
+});
+
+test('smokeCheck fails when caught drops by more than 2', () => {
+  const verdict = smokeCheck(smokeOutput([smokeCase('must-not', 0, 0), smokeCase('large', 1, 4), smokeCase('small', 2, 2)]), baseline);
+  assert.deepEqual(verdict.failures, ['Caught dropped by 3, from 6/6 to 3/6: more than the 2 a run varies by.']);
+  assert.match(verdict.markdown, /\*\*Failed\.\*\*/);
+});
+
+test('smokeCheck fails when the must-not case is flagged, with or without a baseline', () => {
+  const run = smokeOutput([smokeCase('must-not', 0, 0, ['src/A.cs:3 (architecture)']), smokeCase('large', 4, 4), smokeCase('small', 2, 2)]);
+  for (const against of [baseline, undefined]) {
+    const verdict = smokeCheck(run, against);
+    assert.deepEqual(verdict.failures, ['must-not raised a must-not-flag concern (by src/A.cs:3 (architecture)): Do not ask for a backfill.']);
+  }
+});
+
+test('smokeCheck fails when the run and the baseline cover different cases', () => {
+  const verdict = smokeCheck(smokeOutput([smokeCase('must-not', 0, 0), smokeCase('large', 4, 4), smokeCase('other', 2, 2)]), baseline);
+  assert.equal(verdict.failures.length, 1);
+  assert.match(verdict.failures[0] ?? '', /covers large, must-not, other but evals\/baseline-smoke\.json covers large, must-not, small/);
+});
+
+test('smokeCheck passes with a notice and the run alone when there is no baseline yet', () => {
+  const verdict = smokeCheck(smokeOutput([smokeCase('must-not', 0, 0), smokeCase('large', 1, 4)]), undefined);
+  assert.deepEqual(verdict.failures, []);
+  assert.equal(verdict.notices.length, 1);
+  assert.match(verdict.notices[0] ?? '', /no evals\/baseline-smoke\.json yet.*proposed-baseline-smoke\.json in the smoke-eval artifact/);
+  assert.match(verdict.markdown, /\*\*Passed\*\* with no baseline/);
+  assert.match(verdict.markdown, /^large\s+1\/4\s+0\s+2\s+\$1\.00$/m);
+  assert.match(verdict.markdown, /Caught 1\/4; must-not violations 0; review cost \$2\.00, with judges \$2\.50\./);
+});
+
+test('smokeCheck keeps model text inside its code block', () => {
+  const run = smokeOutput([smokeCase('must-not', 0, 0, ['```\n# Injected']), smokeCase('large', 4, 4), smokeCase('small', 2, 2)]);
+  const { markdown } = smokeCheck(run, baseline);
+  assert.equal(markdown.match(/```/g)?.length, 4, 'two code blocks, each opened and closed once');
+  assert.match(markdown, /'''/);
+});
+
+test('trimmed keeps what compare reads and drops the agent calls and raw findings', () => {
+  const full = smokeOutput([{ ...smokeCase('large', 1, 1), raw: [{ title: 'r' }], calls: [{ role: 'judge' }], refuted: [] } as EvalOutput['cases'][number]]);
+  const [only] = trimmed(full).cases;
+  assert.deepEqual(Object.keys(only ?? {}).sort(), ['costUsd', 'findings', 'forbidden', 'id', 'reviewCostUsd', 'verdicts']);
+});
+
+test('commandData keeps a message on one workflow command line', () => {
+  assert.equal(commandData('50% done\r\n::error::x'), '50%25 done%0D%0A::error::x');
+});
+
+test('scanDir scans every file under the folder and fails on a secret, naming the file but not the secret', () => {
+  const dir = mkdtempSync(path.join(tmpdir(), 'smoke-scan-'));
+  try {
+    mkdirSync(path.join(dir, 'nested'));
+    writeFileSync(path.join(dir, 'run.json'), '{"summary":{}}');
+    writeFileSync(path.join(dir, 'nested', 'b.json'), '{"finding":"A plain finding."}');
+    assert.equal(scanDir(dir, ['known-secret-value']), 2);
+
+    writeFileSync(path.join(dir, 'nested', 'b.json'), '{"leak":"known-secret-value"}');
+    assert.throws(() => scanDir(dir, ['known-secret-value']), (error: Error) => error instanceof SecretFoundError && error.message.includes(path.join('nested', 'b.json')) && !error.message.includes('known-secret-value'));
+
+    writeFileSync(path.join(dir, 'nested', 'b.json'), '{"leak":"ghs_0123456789abcdefghijABCDEFGHIJ"}');
+    assert.throws(() => scanDir(dir, []), SecretFoundError);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test('scanDir finds nothing to scan in a folder that does not exist', () => {
+  assert.equal(scanDir(path.join(tmpdir(), 'smoke-scan-missing-folder', 'out'), []), 0);
+});
+
+test('the smoke eval workflow runs only on main and by hand, in the code-review environment, with pinned actions', () => {
+  const workflow = readFileSync(path.join(root, '.github', 'workflows', 'smoke-eval.yml'), 'utf8').replace(/\r\n/g, '\n');
+  const triggers = workflow.slice(workflow.indexOf('\non:'), workflow.indexOf('\npermissions:'));
+  assert.doesNotMatch(triggers, /pull_request|issue_comment|workflow_run/);
+  assert.match(triggers, /push:\n\s+branches: \[main\]/);
+  assert.match(triggers, /workflow_dispatch/);
+  for (const glob of ['prompts/**', 'reviewers/**', 'src/config.ts', 'guidance/**']) assert.ok(triggers.includes(`'${glob}'`), glob);
+  assert.match(workflow, /^permissions: \{\}$/m);
+  assert.match(workflow, /^\s+environment: code-review$/m);
+  assert.match(workflow, /--subset smoke .*--max-cost 8/);
+  const uses = workflow.match(/uses: .*/g) ?? [];
+  assert.ok(uses.length > 0);
+  for (const line of uses) assert.match(line, /^uses: [\w.-]+\/[\w.-]+@[0-9a-f]{40} # v\d+\.\d+\.\d+$/, line);
+});

From 25d1086076b70e38a9b29ed0930fd326ae736a53 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 02:51:41 -0600
Subject: [PATCH 2/4] Test the smoke check's file handling, the scan's link
 refusal and --precision off

Moves the check command's file reading and writing into checkFiles and baselinePathFrom, and the per-case precision decision into casePrecision and parsePrecision, so each can be tested; names the --max-cost argument before parsing it.
---
 evals/run.ts            |  38 ++++++++-----
 evals/smoke.ts          |  22 +++++---
 test/smoke-eval.test.ts | 117 ++++++++++++++++++++++++++++++++++++++--
 3 files changed, 154 insertions(+), 23 deletions(-)

diff --git a/evals/run.ts b/evals/run.ts
index 3b694b4..5de7597 100644
--- a/evals/run.ts
+++ b/evals/run.ts
@@ -113,6 +113,18 @@ export function parseMaxCost(value: string | undefined): number | undefined {
   return Number(text);
 }
 
+// Whether the precision judge runs: --precision on, the default, or off.
+export function parsePrecision(value: string | undefined): boolean {
+  const setting = value ?? 'on';
+  if (setting !== 'on' && setting !== 'off') throw new Error(`--precision takes on or off, not ${JSON.stringify(setting.slice(0, 40))}.`);
+  return setting === 'on';
+}
+
+// One case's precision, or undefined with --precision off, when the judge isn't asked about any finding.
+export function casePrecision(precisionOn: boolean, input: Parameters<typeof judgePosted>[0]): Promise<CasePrecision | undefined> {
+  return precisionOn ? judgePosted(input) : Promise.resolve(undefined);
+}
+
 // The cases --only or --subset names, in cases.json's order. A name that matches no case fails, so a typo or a renamed
 // case can't quietly shrink the run.
 export function selectCases(all: EvalCase[], only: string | undefined, subset: string | undefined): EvalCase[] {
@@ -148,10 +160,10 @@ export async function runCapped<T>(
 const raisedLabel = (finding: Finding & { reviewers?: string[] }) => `${finding.file}:${finding.line} (${(finding.reviewers ?? []).join(', ')})`;
 
 async function main(): Promise<void> {
-  const maxCost = parseMaxCost(process.argv.includes('--max-cost') ? (arg('max-cost') ?? '') : undefined);
-  const precisionSetting = arg('precision') ?? 'on';
-  if (precisionSetting !== 'on' && precisionSetting !== 'off') throw new Error(`--precision takes on or off, not ${JSON.stringify(precisionSetting.slice(0, 40))}.`);
-  const precisionOn = precisionSetting === 'on';
+  // A --max-cost with no value reaches parseMaxCost as '' and fails, instead of running uncapped.
+  const maxCostArg = process.argv.includes('--max-cost') ? (arg('max-cost') ?? '') : undefined;
+  const maxCost = parseMaxCost(maxCostArg);
+  const precisionOn = parsePrecision(arg('precision'));
   const key = process.env['CODE_REVIEW_BEDROCK_KEY'] ?? process.env['AWS_BEARER_TOKEN_BEDROCK'];
   if (!key) throw new Error('Set AWS_BEARER_TOKEN_BEDROCK to run the evals on Bedrock.');
   if (arg('tiers')) process.env['CODE_REVIEW_TIERS'] = arg('tiers');
@@ -213,16 +225,14 @@ async function main(): Promise<void> {
       }
 
       const diffs = splitDiff(gathered.diff);
-      const precision = precisionOn
-        ? await judgePosted({
-          caseId: c.id,
-          findings: outcome.findings,
-          matched,
-          mustNot,
-          cache: verdictCache,
-          judge: realityJudge({ ...run, checkout: head }, { title: c.title, diffFor: (file) => diffs.get(file) }, calls),
-        })
-        : undefined;
+      const precision = await casePrecision(precisionOn, {
+        caseId: c.id,
+        findings: outcome.findings,
+        matched,
+        mustNot,
+        cache: verdictCache,
+        judge: realityJudge({ ...run, checkout: head }, { title: c.title, diffFor: (file) => diffs.get(file) }, calls),
+      });
       if (precision) {
         saveVerdicts(verdictFile, verdictCache);
         precisions.push(precision);
diff --git a/evals/smoke.ts b/evals/smoke.ts
index a55f273..a428bf7 100644
--- a/evals/smoke.ts
+++ b/evals/smoke.ts
@@ -113,18 +113,28 @@ function readBaseline(file: string): EvalOutput | undefined {
   }
 }
 
+// The baseline check's arguments: --baseline <file>, or evals/baseline-smoke.json under root.
+export function baselinePathFrom(argv: readonly string[], root: string): string {
+  const index = argv.indexOf('--baseline');
+  const value = index >= 0 ? argv[index + 1] : undefined;
+  return value ? path.resolve(value) : path.join(root, baselineFile);
+}
+
+// Reads the run and the baseline, a missing one as none, checks them, and writes the proposed baseline beside the run.
+export function checkFiles(runFile: string, baselinePath: string): SmokeVerdict {
+  const run = readOutput(runFile);
+  const verdict = smokeCheck(run, readBaseline(baselinePath));
+  writeFileSync(path.join(path.dirname(runFile), proposedName), `${JSON.stringify(trimmed(run), null, 2)}\n`);
+  return verdict;
+}
+
 function main(): void {
   const [command, target] = process.argv.slice(2);
   const annotate = (kind: 'notice' | 'error', message: string) =>
     console.log(process.env['GITHUB_ACTIONS'] === 'true' ? `::${kind} title=Smoke eval::${commandData(message)}` : `${kind}: ${message}`);
   if (command === 'check' && target) {
     const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
-    const index = process.argv.indexOf('--baseline');
-    const baselinePath = index >= 0 && process.argv[index + 1] ? path.resolve(process.argv[index + 1] ?? '') : path.join(root, baselineFile);
-    const runFile = path.resolve(target);
-    const run = readOutput(runFile);
-    const verdict = smokeCheck(run, readBaseline(baselinePath));
-    writeFileSync(path.join(path.dirname(runFile), proposedName), `${JSON.stringify(trimmed(run), null, 2)}\n`);
+    const verdict = checkFiles(path.resolve(target), baselinePathFrom(process.argv, root));
     const summaryFile = process.env['GITHUB_STEP_SUMMARY'];
     if (summaryFile) appendFileSync(summaryFile, verdict.markdown);
     else process.stdout.write(verdict.markdown);
diff --git a/test/smoke-eval.test.ts b/test/smoke-eval.test.ts
index 6206022..84be4f9 100644
--- a/test/smoke-eval.test.ts
+++ b/test/smoke-eval.test.ts
@@ -1,12 +1,15 @@
 import assert from 'node:assert/strict';
-import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { test } from 'node:test';
 import type { EvalCase } from '../evals/cases.ts';
 import type { EvalOutput } from '../evals/compare.ts';
-import { parseMaxCost, runCapped, selectCases } from '../evals/run.ts';
-import { commandData, scanDir, smokeCheck, trimmed } from '../evals/smoke.ts';
+import type { RealityJudge } from '../evals/precision.ts';
+import { casePrecision, parseMaxCost, parsePrecision, runCapped, selectCases } from '../evals/run.ts';
+import { baselinePathFrom, checkFiles, commandData, proposedName, scanDir, smokeCheck, trimmed } from '../evals/smoke.ts';
+import type { Finding } from '../src/review.ts';
 import { SecretFoundError } from '../src/outputs.ts';
 
 const root = path.resolve(import.meta.dirname, '..');
@@ -24,6 +27,24 @@ test('parseMaxCost refuses a flag without a positive number, instead of running
   }
 });
 
+test('parsePrecision is on by default and refuses anything but on or off', () => {
+  assert.equal(parsePrecision(undefined), true);
+  assert.equal(parsePrecision('on'), true);
+  assert.equal(parsePrecision('off'), false);
+  assert.throws(() => parsePrecision('no'), /--precision takes on or off/);
+});
+
+test('casePrecision with precision off asks the judge nothing and has no precision', async () => {
+  const asked: string[] = [];
+  const judge: RealityJudge = async (f) => { asked.push(f.title); return { verdict: 'real', reason: 'r' }; };
+  const finding: Finding = { severity: 'should-fix', file: 'src/a.cs', line: 10, title: 'Unmatched', problem: 'p', consequence: 'c', fix: 'f', evidence: 'e', confidence: 0.9 };
+  const input = { caseId: 'case', findings: [finding], matched: new Set<Finding>(), mustNot: new Set<Finding>(), cache: {}, judge };
+  assert.equal(await casePrecision(false, input), undefined);
+  assert.deepEqual(asked, []);
+  assert.equal((await casePrecision(true, input))?.real, 1);
+  assert.deepEqual(asked, ['Unmatched']);
+});
+
 // A runner whose cases cost what the table says, recording the ones it started.
 function fakeCases(costs: number[]): { started: number[]; runOne: (index: number) => Promise<number> } {
   const started: number[] = [];
@@ -166,6 +187,96 @@ test('scanDir finds nothing to scan in a folder that does not exist', () => {
   assert.equal(scanDir(path.join(tmpdir(), 'smoke-scan-missing-folder', 'out'), []), 0);
 });
 
+// A link to a file, or on Windows without the symlink privilege a junction to a folder, outside the scanned folder.
+function linkOutside(dir: string, outside: string): boolean {
+  try {
+    symlinkSync(path.join(outside, 'secret.txt'), path.join(dir, 'link.txt'), 'file');
+    return true;
+  } catch {
+    try {
+      symlinkSync(outside, path.join(dir, 'link'), 'junction');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}
+
+test('scanDir refuses a link, so the upload cannot reach a file outside what was scanned', (t) => {
+  const dir = mkdtempSync(path.join(tmpdir(), 'smoke-scan-'));
+  const outside = mkdtempSync(path.join(tmpdir(), 'smoke-outside-'));
+  try {
+    writeFileSync(path.join(outside, 'secret.txt'), 'known-secret-value');
+    writeFileSync(path.join(dir, 'run.json'), '{}');
+    if (!linkOutside(dir, outside)) {
+      t.skip('this system cannot create a link');
+      return;
+    }
+    assert.throws(() => scanDir(dir, ['known-secret-value']), /is not a regular file/);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+    rmSync(outside, { recursive: true, force: true });
+  }
+});
+
+test('baselinePathFrom honors --baseline and otherwise reads evals/baseline-smoke.json', () => {
+  assert.equal(baselinePathFrom(['node', 'smoke.ts', 'check', 'run.json', '--baseline', 'other.json'], '/repo'), path.resolve('other.json'));
+  assert.equal(baselinePathFrom(['node', 'smoke.ts', 'check', 'run.json'], '/repo'), path.join('/repo', 'evals', 'baseline-smoke.json'));
+  assert.equal(baselinePathFrom(['node', 'smoke.ts', 'check', 'run.json', '--baseline'], '/repo'), path.join('/repo', 'evals', 'baseline-smoke.json'));
+});
+
+const regressed = smokeOutput([smokeCase('must-not', 0, 0), smokeCase('large', 1, 4), smokeCase('small', 2, 2)]);
+
+test('checkFiles with no baseline file records no failure and writes the proposed baseline beside the run', () => {
+  const dir = mkdtempSync(path.join(tmpdir(), 'smoke-check-'));
+  try {
+    const runFile = path.join(dir, 'run.json');
+    writeFileSync(runFile, JSON.stringify(regressed));
+    const verdict = checkFiles(runFile, path.join(dir, 'missing-baseline.json'));
+    assert.deepEqual(verdict.failures, []);
+    assert.equal(verdict.notices.length, 1);
+    assert.deepEqual(JSON.parse(readFileSync(path.join(dir, proposedName), 'utf8')), trimmed(regressed));
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test('checkFiles records a failure for a run that regressed from the baseline file', () => {
+  const dir = mkdtempSync(path.join(tmpdir(), 'smoke-check-'));
+  try {
+    const runFile = path.join(dir, 'run.json');
+    const baselineFile = path.join(dir, 'baseline.json');
+    writeFileSync(runFile, JSON.stringify(regressed));
+    writeFileSync(baselineFile, JSON.stringify(baseline));
+    assert.deepEqual(checkFiles(runFile, baselineFile).failures, ['Caught dropped by 3, from 6/6 to 3/6: more than the 2 a run varies by.']);
+    assert.ok(existsSync(path.join(dir, proposedName)));
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// Runs the real command once per outcome, since the exit code it sets is what fails the workflow's step.
+test('smoke.ts check exits 1 on a regression from --baseline and 0 without a baseline', () => {
+  const dir = mkdtempSync(path.join(tmpdir(), 'smoke-cli-'));
+  try {
+    const runFile = path.join(dir, 'run.json');
+    const baselineFile = path.join(dir, 'baseline.json');
+    writeFileSync(runFile, JSON.stringify(regressed));
+    writeFileSync(baselineFile, JSON.stringify(baseline));
+    const check = (...extra: string[]) => spawnSync(process.execPath, [path.join(root, 'evals', 'smoke.ts'), 'check', runFile, ...extra], {
+      encoding: 'utf8', env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' },
+    });
+    const failed = check('--baseline', baselineFile);
+    assert.equal(failed.status, 1, failed.stderr);
+    assert.match(failed.stdout, /error: Caught dropped by 3/);
+    const passed = check('--baseline', path.join(dir, 'missing.json'));
+    assert.equal(passed.status, 0, passed.stderr);
+    assert.match(passed.stdout, /notice: There is no evals\/baseline-smoke\.json yet/);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
 test('the smoke eval workflow runs only on main and by hand, in the code-review environment, with pinned actions', () => {
   const workflow = readFileSync(path.join(root, '.github', 'workflows', 'smoke-eval.yml'), 'utf8').replace(/\r\n/g, '\n');
   const triggers = workflow.slice(workflow.indexOf('\non:'), workflow.indexOf('\npermissions:'));

From 2da07ece0954d3c76f8bf5736f8cf38969b3f43b Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 10:11:50 -0600
Subject: [PATCH 3/4] Test the --max-cost stop: move the cap's outcome and the
 flag's reading out of main()

capOutcome decides what the summary records and the error that fails an over-budget run; maxCostFrom reads the flag from argv. The unknown-case error points at evals/cases.json, and checkFiles is checked on a run that matches its baseline.
---
 evals/run.ts            | 38 ++++++++++++++++++++++++++++++--------
 test/smoke-eval.test.ts | 24 ++++++++++++++++++++++--
 2 files changed, 52 insertions(+), 10 deletions(-)

diff --git a/evals/run.ts b/evals/run.ts
index 10794d6..51f771f 100644
--- a/evals/run.ts
+++ b/evals/run.ts
@@ -113,6 +113,31 @@ export function parseMaxCost(value: string | undefined): number | undefined {
   return Number(text);
 }
 
+// The --max-cost value in argv. A --max-cost with no value reaches parseMaxCost as '' and fails, instead of running
+// uncapped.
+export function maxCostFrom(argv: readonly string[]): number | undefined {
+  const index = argv.indexOf('--max-cost');
+  const value = index >= 0 ? (argv[index + 1] ?? '') : undefined;
+  return parseMaxCost(value);
+}
+
+// What a capped run's summary records about the cap, and the error that fails the run once the cap was passed. A run
+// without --max-cost records nothing.
+export function capOutcome(
+  maxCostUsd: number | undefined,
+  capped: { costUsd: number; ran: number; exceeded: boolean },
+  casesSelected: number,
+  label: string,
+): { record?: { maxCostUsd: number; casesRun: number; casesSelected: number }; error?: string } {
+  if (maxCostUsd === undefined) return {};
+  const record = { maxCostUsd, casesRun: capped.ran, casesSelected };
+  if (!capped.exceeded) return { record };
+  return {
+    record,
+    error: `Stopped over --max-cost: the run cost $${capped.costUsd.toFixed(2)}, more than $${maxCostUsd.toFixed(2)}, after ${capped.ran} of ${casesSelected} cases. evals/out/${label}.json has the cases that ran.`,
+  };
+}
+
 // Whether the precision judge runs: --precision on, the default, or off.
 export function parsePrecision(value: string | undefined): boolean {
   const setting = value ?? 'on';
@@ -136,7 +161,7 @@ export function selectCases(all: EvalCase[], only: string | undefined, subset: s
   }
   if (!ids) return all;
   const missing = ids.filter((id) => !all.some((c) => c.id === id));
-  if (missing.length) throw new Error(`No eval case named ${missing.join(', ')}.`);
+  if (missing.length) throw new Error(`No eval case named ${missing.join(', ')}; evals/cases.json lists the case ids.`);
   return all.filter((c) => ids.includes(c.id));
 }
 
@@ -160,9 +185,7 @@ export async function runCapped<T>(
 const raisedLabel = (finding: Finding & { reviewers?: string[] }) => `${finding.file}:${finding.line} (${(finding.reviewers ?? []).join(', ')})`;
 
 async function main(): Promise<void> {
-  // A --max-cost with no value reaches parseMaxCost as '' and fails, instead of running uncapped.
-  const maxCostArg = process.argv.includes('--max-cost') ? (arg('max-cost') ?? '') : undefined;
-  const maxCost = parseMaxCost(maxCostArg);
+  const maxCost = maxCostFrom(process.argv);
   const precisionOn = parsePrecision(arg('precision'));
   const key = process.env['CODE_REVIEW_BEDROCK_KEY'] ?? process.env['AWS_BEARER_TOKEN_BEDROCK'];
   if (!key) throw new Error('Set AWS_BEARER_TOKEN_BEDROCK to run the evals on Bedrock.');
@@ -264,21 +287,20 @@ async function main(): Promise<void> {
   }
 
   const precision = precisionOn ? totalPrecision(precisions) : undefined;
+  const cap = capOutcome(maxCost, capped, cases.length, label);
   const summary = {
     label, tiers: process.env['CODE_REVIEW_TIERS'] ?? '', verification: verificationOn() ? 'on' : 'off', expected, caught, caughtRaw, violations,
     findings: findingCount, blockersAndShouldFixes: seriousCount, refuted: refutedCount,
     precision, findingsPerCase: spread(findingsPerCase), reviewCostUsd: reviewCosts.reduce((a, b) => a + b, 0),
     reviewCostPerCase: spread(reviewCosts), costUsd: cost,
-    ...(maxCost === undefined ? {} : { maxCostUsd: maxCost, casesRun: capped.ran, casesSelected: cases.length }),
+    ...cap.record,
   };
   console.log(`\nCaught ${caught}/${expected} (reviewers found ${caughtRaw}/${expected}); must-not violations ${violations}; ${findingCount} findings in all, ${seriousCount} blockers and should-fixes, ${refutedCount} refuted by verification; $${cost.toFixed(2)}`);
   if (precision) console.log(`Precision ${percent(precision.precision)}: ${precision.matched} matched + ${precision.real} real of ${precision.posted} blockers and should-fixes; ${precision.unsure} unsure, ${precision.notReal} not real, ${precision.mustNot} must-not`);
   console.log(`Findings per case: mean ${summary.findingsPerCase.mean.toFixed(1)}, max ${summary.findingsPerCase.max}; review cost per case: mean $${summary.reviewCostPerCase.mean.toFixed(2)}, max $${summary.reviewCostPerCase.max.toFixed(2)}`);
   mkdirSync(path.join(root, 'evals', 'out'), { recursive: true });
   writeFileSync(path.join(root, 'evals', 'out', `${label}.json`), JSON.stringify({ summary, cases: report }, null, 2));
-  if (capped.exceeded) {
-    throw new Error(`Stopped over --max-cost: the run cost $${capped.costUsd.toFixed(2)}, more than $${maxCost?.toFixed(2)}, after ${capped.ran} of ${cases.length} cases. evals/out/${label}.json has the cases that ran.`);
-  }
+  if (cap.error) throw new Error(cap.error);
 }
 
 if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
diff --git a/test/smoke-eval.test.ts b/test/smoke-eval.test.ts
index 84be4f9..cf078a4 100644
--- a/test/smoke-eval.test.ts
+++ b/test/smoke-eval.test.ts
@@ -7,7 +7,7 @@ import { test } from 'node:test';
 import type { EvalCase } from '../evals/cases.ts';
 import type { EvalOutput } from '../evals/compare.ts';
 import type { RealityJudge } from '../evals/precision.ts';
-import { casePrecision, parseMaxCost, parsePrecision, runCapped, selectCases } from '../evals/run.ts';
+import { capOutcome, casePrecision, maxCostFrom, parseMaxCost, parsePrecision, runCapped, selectCases } from '../evals/run.ts';
 import { baselinePathFrom, checkFiles, commandData, proposedName, scanDir, smokeCheck, trimmed } from '../evals/smoke.ts';
 import type { Finding } from '../src/review.ts';
 import { SecretFoundError } from '../src/outputs.ts';
@@ -27,6 +27,13 @@ test('parseMaxCost refuses a flag without a positive number, instead of running
   }
 });
 
+test('maxCostFrom reads --max-cost from argv, fails on a bare flag, and is absent without it', () => {
+  assert.equal(maxCostFrom(['node', 'run.ts', '--subset', 'smoke', '--max-cost', '8']), 8);
+  assert.equal(maxCostFrom(['node', 'run.ts', '--subset', 'smoke']), undefined);
+  assert.throws(() => maxCostFrom(['node', 'run.ts', '--max-cost']), /--max-cost takes a positive number of dollars/);
+  assert.throws(() => maxCostFrom(['node', 'run.ts', '--max-cost', '--label', 'x']), /--max-cost takes a positive number of dollars/);
+});
+
 test('parsePrecision is on by default and refuses anything but on or off', () => {
   assert.equal(parsePrecision(undefined), true);
   assert.equal(parsePrecision('on'), true);
@@ -69,6 +76,18 @@ test('runCapped runs every case when the total stays at or under the cap, or wit
   assert.deepEqual(await runCapped([0, 1], fakeCases([40, 40]).runOne), { costUsd: 80, ran: 2, exceeded: false });
 });
 
+test('capOutcome fails a run that passed the cap, and records the cap and the cases run', () => {
+  assert.deepEqual(capOutcome(5, { costUsd: 6, ran: 2, exceeded: true }, 3, 'smoke'), {
+    record: { maxCostUsd: 5, casesRun: 2, casesSelected: 3 },
+    error: 'Stopped over --max-cost: the run cost $6.00, more than $5.00, after 2 of 3 cases. evals/out/smoke.json has the cases that ran.',
+  });
+});
+
+test('capOutcome passes a run within the cap, and records nothing without one', () => {
+  assert.deepEqual(capOutcome(8, { costUsd: 8, ran: 2, exceeded: false }, 2, 'smoke'), { record: { maxCostUsd: 8, casesRun: 2, casesSelected: 2 } });
+  assert.deepEqual(capOutcome(undefined, { costUsd: 80, ran: 2, exceeded: false }, 2, 'smoke'), {});
+});
+
 test('the smoke subset is 4 existing cases, with the must-not case', () => {
   const smoke = selectCases(allCases, undefined, 'smoke');
   assert.equal(smoke.length, 4);
@@ -241,7 +260,7 @@ test('checkFiles with no baseline file records no failure and writes the propose
   }
 });
 
-test('checkFiles records a failure for a run that regressed from the baseline file', () => {
+test('checkFiles records a failure for a run that regressed from the baseline file, and none for a run that matches it', () => {
   const dir = mkdtempSync(path.join(tmpdir(), 'smoke-check-'));
   try {
     const runFile = path.join(dir, 'run.json');
@@ -250,6 +269,7 @@ test('checkFiles records a failure for a run that regressed from the baseline fi
     writeFileSync(baselineFile, JSON.stringify(baseline));
     assert.deepEqual(checkFiles(runFile, baselineFile).failures, ['Caught dropped by 3, from 6/6 to 3/6: more than the 2 a run varies by.']);
     assert.ok(existsSync(path.join(dir, proposedName)));
+    assert.deepEqual(checkFiles(baselineFile, baselineFile).failures, []);
   } finally {
     rmSync(dir, { recursive: true, force: true });
   }

From dae21e6acbe34e36783df5ce426a15bc971e675f Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 10:55:11 -0600
Subject: [PATCH 4/4] Fail on an --only that names no case

selectCases dropped empty ids, so --only '' or --only ',' ran zero cases and passed.
---
 CHANGELOG.md            | 2 +-
 evals/run.ts            | 5 +++--
 test/smoke-eval.test.ts | 4 +++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d9e018c..d830d4e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ### Added
 
-- A cost-capped smoke eval: the Smoke eval workflow runs 4 eval cases, among them the must-not case and a large diff, on every push to `main` that changes a prompt, a charter, `src/config.ts` or the guidance, and by hand, in the `code-review` environment and never on a pull request. It compares the run with `evals/baseline-smoke.json` in the job summary, fails when caught drops by more than 2 or the must-not case is flagged, and uploads `evals/out` once a secret scan passes. Until a baseline is committed, it passes with a notice and uploads the run as the proposed one. `node evals/run.ts` gains `--max-cost <usd>`, which stops and fails the run once it costs more, `--subset <name>` for a named set of cases, and `--precision off`; `--only` now fails on a case that doesn't exist ([#52](https://github.com/NCodeGroup/code-review/issues/52)).
+- A cost-capped smoke eval: the Smoke eval workflow runs 4 eval cases, among them the must-not case and a large diff, on every push to `main` that changes a prompt, a charter, `src/config.ts` or the guidance, and by hand, in the `code-review` environment and never on a pull request. It compares the run with `evals/baseline-smoke.json` in the job summary, fails when caught drops by more than 2 or the must-not case is flagged, and uploads `evals/out` once a secret scan passes. Until a baseline is committed, it passes with a notice and uploads the run as the proposed one. `node evals/run.ts` gains `--max-cost <usd>`, which stops and fails the run once it costs more, `--subset <name>` for a named set of cases, and `--precision off`; `--only` now fails on a case that doesn't exist or when it names none ([#52](https://github.com/NCodeGroup/code-review/issues/52)).
 - Evals measure precision: each posted blocker and should-fix that matches no expected concern is judged `real`, `not-real` or `unsure` by Opus reading the checkout, with verdicts cached in `evals/verdicts.json`. Each output also reports findings per case (mean, max) and the review's cost per case. `npm run eval:compare` compares two runs, marking recall differences of 2 or less as noise, and `npm run eval:confirm` adds confirmed verdicts to `cases.json` as expected or must-not-flag concerns ([#51](https://github.com/NCodeGroup/code-review/issues/51)).
 - A finding whose fix is a mechanical edit of up to five lines, all of them added by the pull request, carries a GitHub suggestion block on a comment that covers those lines, so it applies with one click; any other fix stays in prose. The hidden marker on each inline comment also names its reviewers, and comments posted before still count as already posted ([#54](https://github.com/NCodeGroup/code-review/issues/54)).
 - A per-review cost ceiling guards against a runaway review: $8 plus $4 for each part of the diff, overridable with `CODE_REVIEW_MAX_RUN_USD`. Reviewer calls not started once it is reached are skipped and shown as `skipped: run budget` in the check run, synthesis still runs on what was found, and the summary warns that the review is incomplete ([#50](https://github.com/NCodeGroup/code-review/issues/50)).
diff --git a/evals/run.ts b/evals/run.ts
index c1d3a1a..585eb8b 100644
--- a/evals/run.ts
+++ b/evals/run.ts
@@ -166,11 +166,12 @@ export function casePrecision(precisionOn: boolean, input: Parameters<typeof jud
   return precisionOn ? judgePosted(input) : Promise.resolve(undefined);
 }
 
-// The cases --only or --subset names, in cases.json's order. A name that matches no case fails, so a typo or a renamed
-// case can't quietly shrink the run.
+// The cases --only or --subset names, in cases.json's order. A name that matches no case fails, and so does an --only
+// that names none, so a typo, a renamed case or an unset shell variable can't quietly shrink the run.
 export function selectCases(all: EvalCase[], only: string | undefined, subset: string | undefined): EvalCase[] {
   if (only !== undefined && subset !== undefined) throw new Error('Use --only or --subset, not both.');
   let ids: readonly string[] | undefined = only?.split(',').map((id) => id.trim()).filter(Boolean);
+  if (ids?.length === 0) throw new Error('--only names no case; give it case ids separated by commas.');
   if (subset !== undefined) {
     ids = Object.hasOwn(subsets, subset) ? subsets[subset] : undefined;
     if (!ids) throw new Error(`No subset named ${JSON.stringify(subset)}; evals/cases.ts has ${Object.keys(subsets).join(', ')}.`);
diff --git a/test/smoke-eval.test.ts b/test/smoke-eval.test.ts
index cf078a4..3cb9b22 100644
--- a/test/smoke-eval.test.ts
+++ b/test/smoke-eval.test.ts
@@ -95,11 +95,13 @@ test('the smoke subset is 4 existing cases, with the must-not case', () => {
   assert.ok(smoke.some((c) => c.id === 'heap-allocation-key-export'), 'the large-diff case');
 });
 
-test('selectCases fails on a case or subset that does not exist, and on --only with --subset', () => {
+test('selectCases fails on a case or subset that does not exist, on an empty --only, and on --only with --subset', () => {
   assert.throws(() => selectCases(allCases, 'accepted-no-backfill,no-such-case', undefined), /No eval case named no-such-case/);
   assert.throws(() => selectCases(allCases, undefined, 'nightly'), /No subset named "nightly"/);
   assert.throws(() => selectCases(allCases, undefined, '__proto__'), /No subset named/);
   assert.throws(() => selectCases(allCases, 'accepted-no-backfill', 'smoke'), /--only or --subset, not both/);
+  assert.throws(() => selectCases(allCases, '', undefined), /--only names no case/);
+  assert.throws(() => selectCases(allCases, ' , ', undefined), /--only names no case/);
   assert.deepEqual(selectCases(allCases, 'secret-buffer-length, accepted-no-backfill', undefined).map((c) => c.id), ['secret-buffer-length', 'accepted-no-backfill']);
   assert.equal(selectCases(allCases, undefined, undefined).length, allCases.length);
 });