Run a cost-capped smoke eval when a prompt, charter or model changes

.github/workflows/smoke-eval.yml

@@ -0,0 +1,76 @@
+# A small, cost-capped eval on main whenever a prompt, a charter, a model or the guidance changes, so a regression shows
+# before a release reaches the repositories that use the action. The full eval stays on demand (README, Evals).
+#
+# It runs the smoke subset in evals/cases.ts with the production guidance, stops and fails past $8, and compares the run
+# with evals/baseline-smoke.json in the job summary: it fails when caught drops by more than 2 or the must-not case is
+# flagged. With no baseline yet, the run is uploaded as the proposed one and the job passes with a notice.
+#
+# It uses the organization's Bedrock key in the code-review environment, which only the default branch can use. Never
+# add a pull_request trigger: the key would then run code from any pull request.
+name: Smoke eval
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'prompts/**'
+      - 'reviewers/**'
+      - 'src/config.ts'
+      - 'guidance/**'
+  workflow_dispatch:
+
+permissions: {}
+
+# A run costs money, so a newer push waits for the running one instead of cancelling it.
+concurrency:
+  group: smoke-eval
+  cancel-in-progress: false
+
+jobs:
+  smoke-eval:
+    name: Smoke eval
+    runs-on: ubuntu-latest
+    environment: code-review
+    timeout-minutes: 90
+    permissions:
+      contents: read
+      # The eval reads the issues a case's pull request closes.
+      issues: read
+    env:
+      LABEL: smoke-${{ github.sha }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version: 24
+      - run: npm ci --ignore-scripts --no-audit --no-fund
+
+      # The precision judge's Opus calls are left out: the gate reads caught and the must-not case only.
+      - name: Run the smoke subset
+        env:
+          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
+          GH_TOKEN: ${{ github.token }}
+        run: node evals/run.ts --subset smoke --guidance full --precision off --max-cost 8 --label "$LABEL"
+
+      - name: Compare with the baseline
+        run: node evals/smoke.ts check "evals/out/$LABEL.json"
+
+      # Artifacts on a public repository are public: nothing is uploaded unless the scan passes.
+      - name: Scan the outputs for secrets
+        id: scan
+        if: ${{ !cancelled() }}
+        env:
+          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
+          GH_TOKEN: ${{ github.token }}
+        run: node evals/smoke.ts scan evals/out
+
+      - name: Upload the outputs
+        if: ${{ !cancelled() && steps.scan.outcome == 'success' }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: smoke-eval
+          path: evals/out/
+          retention-days: 30
+          if-no-files-found: ignore

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ### Added
 
+- A cost-capped smoke eval: the Smoke eval workflow runs 4 eval cases, among them the must-not case and a large diff, on every push to `main` that changes a prompt, a charter, `src/config.ts` or the guidance, and by hand, in the `code-review` environment and never on a pull request. It compares the run with `evals/baseline-smoke.json` in the job summary, fails when caught drops by more than 2 or the must-not case is flagged, and uploads `evals/out` once a secret scan passes. Until a baseline is committed, it passes with a notice and uploads the run as the proposed one. `node evals/run.ts` gains `--max-cost <usd>`, which stops and fails the run once it costs more, `--subset <name>` for a named set of cases, and `--precision off`; `--only` now fails on a case that doesn't exist or when it names none ([#52](https://github.com/NCodeGroup/code-review/issues/52)).
 - Per-repository settings in `.github/code-review/`, read from the default branch's checkout only, so a pull request that edits them is reviewed with the default branch's copy: `config.yml` (`sensitive:`, `skip:`, `policies:`, `tiers:`), `guidance.md` rules for the context stage, and `reviewers/*.md` project reviewers that join the roster as `project:<name>`. Files `skip:` leaves out are listed in the check run as skipped by config, and a pull request whose every file it skips runs no model; policies reach every reviewer's conventions within a budget of their own, and a setting the engine can't use, including a `tiers:` name that matches no reviewer, is a warning in the check run, never a failed review ([#43](https://github.com/NCodeGroup/code-review/issues/43)).
 - Shadow mode for a new reviewer or prompt: a charter with `shadow: true` in its frontmatter, or named in `CODE_REVIEW_SHADOW`, runs as usual, but its findings go only to the trace and a collapsed "Shadow findings (not posted)" section of the check run, never to synthesis, the threads, the summary or the verdict. A `prompts/<name>.shadow.md` runs beside the live prompt on the share of reviews `CODE_REVIEW_SHADOW_RATE` sets (0 by default), sampled by pull request and head, with both outputs in the trace. The evals score shadow reviewers apart from the live ones ([#53](https://github.com/NCodeGroup/code-review/issues/53)).
 - `/review teach <rule>`, `/review teach F3 <rule>` and `/review teach org <rule>` turn a correction into lasting guidance: with no model call, the publish step opens an issue labelled `review-guidance`, in this repository or, for `org`, in NCodeGroup/code-review, quoting the rule as data with a link to the comment and, for `F3`, the finding's file, line and text, and replies with a link to it. It creates the label when it's missing and uses a second app token that can write only issues, so a review never asks for issues write; when the app lacks that permission the reply says so. `npm run train` copies the rules of `teach org` issues closed as completed into `training/feedback.md` before it generalizes. Writers only, top-level comments only, no forks, as for `/review` ([#45](https://github.com/NCodeGroup/code-review/issues/45)).

README.md

@@ -391,7 +391,16 @@ A case with a `reviewed` head is a [re-review](#re-reviews): it carries the find
 
 **Precision.** Every posted blocker and should-fix that matches no expected concern goes to an Opus judge, which reads the case's checkout, locked down like the reviewers, with the finding passed as data, and answers `real`, `not-real` or `unsure`: is it a real defect the maintainer would act on? A finding that raises a must-not-flag concern counts as not real without a call. Precision is (matched + real) ÷ all posted blockers and should-fixes, with `unsure` counted apart. Verdicts are cached by case and finding fingerprint (file, severity and the title's words) in `evals/verdicts.json`, so a rerun judges only findings it hasn't seen; commit it with `cases.json` so every run is judged the same way. Each `evals/out/<label>.json` has the precision counts per case and in `summary.precision`, findings per case (`summary.findingsPerCase`: mean, max) and the review's own cost, without the eval's judges (`reviewCostUsd`, and `summary.reviewCostPerCase`).
 
-**Comparing two setups.** `npm run eval:compare -- <a.json> <b.json>` (or bare labels from `evals/out/`) prints recall, precision, findings per pull request and cost for both runs, then each case's differences. A recall difference of 2 or less is marked `(noise)`, within one run's variance; the other differences aren't marked, since their variance hasn't been measured. Outputs from before precision show `—` for it.
+**Comparing two setups.** `npm run eval:compare -- <a.json> <b.json>` (or bare labels from `evals/out/`) prints recall, precision, findings per pull request and cost for both runs, then each case's differences. A recall difference of 2 or less is marked `(noise)`, within one run's variance; the other differences aren't marked, since their variance hasn't been measured. Outputs from before precision, or from a run with `--precision off`, which skips the precision judge's Opus calls, show `—` for it.
+
+**Capping the cost.** `--max-cost <usd>` stops a run once its cost, judges included, passes that many dollars: the case that crossed it finishes, the rest don't start, the output covers the cases that ran, and the run fails. `--subset <name>` runs a named set of cases from `evals/cases.ts` instead of `--only id,id`; a name that matches no case fails the run.
+
+**Smoke eval.** The [Smoke eval](.github/workflows/smoke-eval.yml) workflow runs on every push to `main` that changes `prompts/`, `reviewers/`, `src/config.ts` or `guidance/`, and by hand (`gh workflow run smoke-eval.yml`). It runs in the `code-review` environment with the organization's Bedrock key, so it never runs on a pull request. It runs the `smoke` subset, 4 cases with the must-not case and one large diff, with the production guidance, `--precision off` and `--max-cost 8`. `node evals/smoke.ts check` then compares the run with `evals/baseline-smoke.json`, posts the comparison to the job summary, and fails when caught drops by more than 2, when the must-not case is flagged, or when the two cover different cases. `evals/out` is scanned for secrets before it's uploaded as the `smoke-eval` artifact, since artifacts on a public repository are public. With no baseline yet, the run passes with a notice, failing only on a flagged must-not case. To adopt a run as the baseline, the first time or after an intended change in what the smoke cases catch or a change to the subset, take `proposed-baseline-smoke.json` from that run's artifact, which is the run without its agent calls and raw findings, and commit it in a pull request:
+
+```powershell
+gh run download <run id> -n smoke-eval -D evals/out/smoke
+Copy-Item evals/out/smoke/proposed-baseline-smoke.json evals/baseline-smoke.json
+```
 
 **Growing the set from verdicts.** `npm run eval:confirm` lists the `real` and `not-real` verdicts nobody has confirmed yet (`--verdict real`, `--verdict unsure` or `--sample 5` of each narrow it), each with the command that confirms it. `npm run eval:confirm -- --case <id> --fingerprint <fp>` adds a `real` finding to its case in `cases.json` as an expected concern, and a `not-real` one as a must-not-flag concern, and marks the verdict confirmed. `--as expect` or `--as must-not` overrides the judge, and is needed for an `unsure` one. Review the new concern's wording in `cases.json` before committing it. In PowerShell, quote the separator: `npm run eval:confirm '--' --case <id> --fingerprint <fp>`.
 

evals/cases.ts

@@ -22,3 +22,12 @@ export interface EvalCase {
   mustNotFlag: Concern[];
   source: string;
 }
+
+// Named subsets of evals/cases.json, for node evals/run.ts --subset <name>.
+// smoke: what .github/workflows/smoke-eval.yml runs on main when a prompt, a charter, a model or the guidance changes,
+// under --max-cost 8: the must-not case (an accepted decision the reviewers must not flag again), one large diff (43
+// files, about 2,600 changed lines), and two small ones in C# and TypeScript. Changing it needs a new
+// evals/baseline-smoke.json.
+export const subsets: Record<string, readonly string[]> = {
+  smoke: ['accepted-no-backfill', 'heap-allocation-key-export', 'secret-buffer-length', 'test-gap-newest-summary-selection'],
+};

evals/compare.ts

@@ -19,7 +19,7 @@ export interface EvalOutput {
   cases: {
     id: string;
     verdicts: { caught: boolean; caughtRaw: boolean }[];
-    forbidden: { raisedBy: string[] }[];
+    forbidden: { concern?: string; raisedBy: string[] }[];
     findings: unknown[];
     precision?: PrecisionCounts;
     reviewCostUsd?: number;
@@ -113,7 +113,8 @@ export function compare(a: EvalOutput, b: EvalOutput, names: [string, string]):
   const notes = [`A recall difference of ${noiseLimit} or less is within one run's variance: (noise). Other differences aren't flagged; their variance hasn't been measured.`];
   const onlyIn = (from: Map<string, unknown>, other: Map<string, unknown>) => [...from.keys()].filter((id) => !other.has(id));
   if (onlyIn(ca, cb).length || onlyIn(cb, ca).length) notes.push(`The runs cover different cases; totals aren't comparable. Only in ${names[0]}: ${onlyIn(ca, cb).join(', ') || 'none'}. Only in ${names[1]}: ${onlyIn(cb, ca).join(', ') || 'none'}.`);
-  if (ta.precision === undefined || tb.precision === undefined) notes.push('A run without precision fields predates the precision judge: — marks what it lacks, and its review cost includes the eval\'s judges.');
+  if (ta.precision === undefined || tb.precision === undefined) notes.push('— marks the precision a run lacks: it predates the precision judge or ran with --precision off.');
+  if (a.summary.reviewCostUsd === undefined || b.summary.reviewCostUsd === undefined) notes.push('A run without reviewCostUsd predates it: its review cost includes the eval\'s judges.');
   return `${overall}\n\n${perCase}\n\n${notes.join('\n')}\n`;
 }