From 565f2bfae772544dbc9fbfab9924ae4c0803ba98 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 16:55:20 -0600
Subject: [PATCH 01/10] Plan step 1: Fail the smoke check when a committed
 baseline is missing

---
 evals/smoke.ts          | 13 ++++++++-----
 test/smoke-eval.test.ts | 21 +++++++++++++++++++++
 2 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/evals/smoke.ts b/evals/smoke.ts
index a428bf7..dcf8683 100644
--- a/evals/smoke.ts
+++ b/evals/smoke.ts
@@ -3,8 +3,9 @@
 //   node evals/smoke.ts check <run.json> [--baseline <file>]
 //     Compares the run with evals/baseline-smoke.json in eval:compare's table and fails when caught drops by more than
 //     2, when a must-not-flag concern was raised, or when the two cover different cases. With no baseline yet it fails
-//     only on a must-not-flag concern, with a notice. Writes the table to $GITHUB_STEP_SUMMARY (stdout outside
-//     Actions), and the run, trimmed to what compare reads, to proposed-baseline-smoke.json beside it.
+//     only on a must-not-flag concern, with a notice. Once one was committed, a missing one fails, naming the commit
+//     that deleted it. Writes the table to $GITHUB_STEP_SUMMARY (stdout outside Actions), and the run, trimmed to what
+//     compare reads, to proposed-baseline-smoke.json beside it.
 //   node evals/smoke.ts scan <dir>
 //     Fails when a file under <dir> holds one of the job's secrets or anything shaped like a credential, before the
 //     workflow uploads the folder: artifacts on a public repository are public.
@@ -40,7 +41,7 @@ function runTable(run: EvalOutput): string {
 // Model output reaches the summary only inside a code block, which a run of backticks could otherwise close.
 const fenced = (text: string) => `\`\`\`text\n${text.replace(/`{3,}/g, (ticks) => "'".repeat(ticks.length)).trimEnd()}\n\`\`\`\n`;
 
-export function smokeCheck(run: EvalOutput, baseline: EvalOutput | undefined): SmokeVerdict {
+export function smokeCheck(run: EvalOutput, baseline: EvalOutput | undefined, deletedIn?: string): SmokeVerdict {
   const failures: string[] = [];
   const notices: string[] = [];
   for (const c of run.cases) {
@@ -56,6 +57,8 @@ export function smokeCheck(run: EvalOutput, baseline: EvalOutput | undefined): S
       const drop = baseline.summary.caught - run.summary.caught;
       if (drop > noiseLimit) failures.push(`Caught dropped by ${drop}, from ${baseline.summary.caught}/${baseline.summary.expected} to ${run.summary.caught}/${run.summary.expected}: more than the ${noiseLimit} a run varies by.`);
     }
+  } else if (deletedIn) {
+    failures.push(`${baselineFile} is missing, but commit ${deletedIn} deleted it: restore it, or adopt a new baseline from this run's ${proposedName} in a pull request.`);
   } else {
     notices.push(`There is no ${baselineFile} yet, so caught wasn't compared. This run is the proposed baseline: ${proposedName} in the smoke-eval artifact. The README's Smoke eval section says how to adopt it.`);
   }
@@ -121,9 +124,9 @@ export function baselinePathFrom(argv: readonly string[], root: string): string
 }
 
 // Reads the run and the baseline, a missing one as none, checks them, and writes the proposed baseline beside the run.
-export function checkFiles(runFile: string, baselinePath: string): SmokeVerdict {
+export function checkFiles(runFile: string, baselinePath: string, deletedIn?: string): SmokeVerdict {
   const run = readOutput(runFile);
-  const verdict = smokeCheck(run, readBaseline(baselinePath));
+  const verdict = smokeCheck(run, readBaseline(baselinePath), deletedIn);
   writeFileSync(path.join(path.dirname(runFile), proposedName), `${JSON.stringify(trimmed(run), null, 2)}\n`);
   return verdict;
 }
diff --git a/test/smoke-eval.test.ts b/test/smoke-eval.test.ts
index f2a6e4e..00d0fe6 100644
--- a/test/smoke-eval.test.ts
+++ b/test/smoke-eval.test.ts
@@ -193,6 +193,13 @@ test('smokeCheck passes with a notice and the run alone when there is no baselin
   assert.match(verdict.markdown, /Caught 1\/4; must-not violations 0; review cost \$2\.00, with judges \$2\.50\./);
 });
 
+test('smokeCheck fails when the baseline was committed once and is missing now', () => {
+  const verdict = smokeCheck(smokeOutput([smokeCase('must-not', 0, 0), smokeCase('large', 4, 4)]), undefined, 'abc1234');
+  assert.deepEqual(verdict.failures, ["evals/baseline-smoke.json is missing, but commit abc1234 deleted it: restore it, or adopt a new baseline from this run's proposed-baseline-smoke.json in a pull request."]);
+  assert.deepEqual(verdict.notices, []);
+  assert.match(verdict.markdown, /\*\*Failed\.\*\*/);
+});
+
 test('smokeCheck keeps model text inside its code block', () => {
   const run = smokeOutput([smokeCase('must-not', 0, 0, ['```\n# Injected']), smokeCase('large', 4, 4), smokeCase('small', 2, 2)]);
   const { markdown } = smokeCheck(run, baseline);
@@ -286,6 +293,20 @@ test('checkFiles with no baseline file records no failure and writes the propose
   }
 });
 
+test('checkFiles fails a run whose missing baseline a commit deleted, and still writes the proposed baseline', () => {
+  const dir = mkdtempSync(path.join(tmpdir(), 'smoke-check-'));
+  try {
+    const runFile = path.join(dir, 'run.json');
+    writeFileSync(runFile, JSON.stringify(regressed));
+    const verdict = checkFiles(runFile, path.join(dir, 'missing-baseline.json'), 'abc1234');
+    assert.equal(verdict.failures.length, 1);
+    assert.match(verdict.failures[0] ?? '', /commit abc1234 deleted it/);
+    assert.ok(existsSync(path.join(dir, proposedName)));
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
 test('checkFiles records a failure for a run that regressed from the baseline file, and none for a run that matches it', () => {
   const dir = mkdtempSync(path.join(tmpdir(), 'smoke-check-'));
   try {

From 2d3f8cf1e91e5e7c59d11cb9d9aeb23b93a2627f Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 16:56:03 -0600
Subject: [PATCH 02/10] Plan step 2: Read the baseline's history in the smoke
 check

---
 evals/smoke.ts          | 16 ++++++++++++++--
 test/smoke-eval.test.ts | 35 +++++++++++++++++++++++++++++++++--
 2 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/evals/smoke.ts b/evals/smoke.ts
index dcf8683..21afb20 100644
--- a/evals/smoke.ts
+++ b/evals/smoke.ts
@@ -10,7 +10,8 @@
 //     Fails when a file under <dir> holds one of the job's secrets or anything shaped like a credential, before the
 //     workflow uploads the folder: artifacts on a public repository are public.
 
-import { appendFileSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import { appendFileSync, existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { assertNoSecrets } from '../src/outputs.ts';
@@ -131,13 +132,24 @@ export function checkFiles(runFile: string, baselinePath: string, deletedIn?: st
   return verdict;
 }
 
+// The last commit that changed baselineFile in root's history, or undefined when none did. A shallow checkout can't
+// tell, so it throws rather than pass a deleted baseline.
+export function lastBaselineCommit(root: string): string | undefined {
+  const git = (...args: string[]) => execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+  if (git('rev-parse', '--is-shallow-repository') === 'true') throw new Error(`The checkout is shallow, so it can't tell whether ${baselineFile} was deleted; check out with fetch-depth: 0.`);
+  return git('log', '-1', '--format=%h', '--', baselineFile) || undefined;
+}
+
 function main(): void {
   const [command, target] = process.argv.slice(2);
   const annotate = (kind: 'notice' | 'error', message: string) =>
     console.log(process.env['GITHUB_ACTIONS'] === 'true' ? `::${kind} title=Smoke eval::${commandData(message)}` : `${kind}: ${message}`);
   if (command === 'check' && target) {
     const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
-    const verdict = checkFiles(path.resolve(target), baselinePathFrom(process.argv, root));
+    const baselinePath = baselinePathFrom(process.argv, root);
+    // Only the committed baseline has a history; a --baseline file is the caller's.
+    const deletedIn = baselinePath === path.join(root, baselineFile) && !existsSync(baselinePath) ? lastBaselineCommit(root) : undefined;
+    const verdict = checkFiles(path.resolve(target), baselinePath, deletedIn);
     const summaryFile = process.env['GITHUB_STEP_SUMMARY'];
     if (summaryFile) appendFileSync(summaryFile, verdict.markdown);
     else process.stdout.write(verdict.markdown);
diff --git a/test/smoke-eval.test.ts b/test/smoke-eval.test.ts
index 00d0fe6..8bbbc6f 100644
--- a/test/smoke-eval.test.ts
+++ b/test/smoke-eval.test.ts
@@ -1,14 +1,15 @@
 import assert from 'node:assert/strict';
-import { spawnSync } from 'node:child_process';
+import { execFileSync, spawnSync } from 'node:child_process';
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { test } from 'node:test';
+import { pathToFileURL } from 'node:url';
 import type { EvalCase } from '../evals/cases.ts';
 import type { EvalOutput } from '../evals/compare.ts';
 import type { CasePrecision, RealityJudge } from '../evals/precision.ts';
 import { capOutcome, caseIssues, casePrecision, casePullRequest, maxCostFrom, parseMaxCost, parsePrecision, runCapped, selectCases, tallyCase, type CaseTally } from '../evals/run.ts';
-import { baselinePathFrom, checkFiles, commandData, proposedName, scanDir, smokeCheck, trimmed } from '../evals/smoke.ts';
+import { baselinePathFrom, checkFiles, commandData, lastBaselineCommit, proposedName, scanDir, smokeCheck, trimmed } from '../evals/smoke.ts';
 import type { CallRecord } from '../src/agent.ts';
 import type { Finding } from '../src/review.ts';
 import { SecretFoundError } from '../src/outputs.ts';
@@ -307,6 +308,36 @@ test('checkFiles fails a run whose missing baseline a commit deleted, and still
   }
 });
 
+// Runs git, since what the check reads is a real repository's history.
+test('lastBaselineCommit names the commit that deleted the baseline, none before one was committed, and fails on a shallow checkout', () => {
+  const dir = mkdtempSync(path.join(tmpdir(), 'smoke-history-'));
+  // Without the machine's own git configuration, so a default branch name, hooks or signing can't change the result.
+  const git = (cwd: string, ...args: string[]) => execFileSync('git', ['-C', cwd, '-c', 'user.name=t', '-c', 'user.email=t@example.com', '-c', 'commit.gpgsign=false', '-c', 'init.defaultBranch=main', ...args], {
+    encoding: 'utf8',
+    env: { PATH: process.env['PATH'] ?? '', ...(process.env['SystemRoot'] ? { SystemRoot: process.env['SystemRoot'] } : {}), HOME: dir, USERPROFILE: dir, XDG_CONFIG_HOME: dir, GIT_CONFIG_NOSYSTEM: '1' },
+  });
+  try {
+    const repo = path.join(dir, 'repo');
+    mkdirSync(path.join(repo, 'evals'), { recursive: true });
+    git(repo, 'init', '-q');
+    writeFileSync(path.join(repo, 'README.md'), 'r\n');
+    git(repo, 'add', '.');
+    git(repo, 'commit', '-q', '-m', 'first');
+    assert.equal(lastBaselineCommit(repo), undefined);
+    writeFileSync(path.join(repo, 'evals', 'baseline-smoke.json'), '{}\n');
+    git(repo, 'add', '.');
+    git(repo, 'commit', '-q', '-m', 'adopt');
+    git(repo, 'rm', '-q', 'evals/baseline-smoke.json');
+    git(repo, 'commit', '-q', '-m', 'delete');
+    assert.equal(lastBaselineCommit(repo), git(repo, 'rev-parse', '--short', 'HEAD').trim());
+    const shallow = path.join(dir, 'shallow');
+    git(dir, 'clone', '-q', '--depth', '1', pathToFileURL(repo).href, shallow);
+    assert.throws(() => lastBaselineCommit(shallow), /The checkout is shallow/);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
 test('checkFiles records a failure for a run that regressed from the baseline file, and none for a run that matches it', () => {
   const dir = mkdtempSync(path.join(tmpdir(), 'smoke-check-'));
   try {

From a871ff36b064bca8e8cd21bbfa9bceb77853283c Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 16:56:41 -0600
Subject: [PATCH 03/10] Plan step 3: Check out the full history and test the
 workflow's order

---
 .github/workflows/smoke-eval.yml |  5 +++-
 README.md                        |  2 +-
 test/smoke-eval.test.ts          | 46 ++++++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/smoke-eval.yml b/.github/workflows/smoke-eval.yml
index 8a2acf6..03c5cef 100644
--- a/.github/workflows/smoke-eval.yml
+++ b/.github/workflows/smoke-eval.yml
@@ -3,7 +3,8 @@
 #
 # It runs the smoke subset in evals/cases.ts with the production guidance, stops and fails past $8, and compares the run
 # with evals/baseline-smoke.json in the job summary: it fails when caught drops by more than 2 or the must-not case is
-# flagged. With no baseline yet, the run is uploaded as the proposed one and the job passes with a notice.
+# flagged. Until a baseline is first committed, the run is uploaded as the proposed one and the job passes with a notice;
+# once one has been, a run without it fails.
 #
 # It uses the organization's Bedrock key in the code-review environment, which only the default branch can use. Never
 # add a pull_request trigger: the key would then run code from any pull request.
@@ -41,6 +42,8 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
+          # The full history, so the check can tell a deleted baseline from one never committed.
+          fetch-depth: 0
           persist-credentials: false
       - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
         with:
diff --git a/README.md b/README.md
index 9aac71d..af33ecc 100644
--- a/README.md
+++ b/README.md
@@ -488,7 +488,7 @@ Each case's pull request is built as the review step builds it: `author` and `au
 
 **Capping the cost.** `--max-cost <usd>` stops a run once its cost, judges included, passes that many dollars: the case that crossed it finishes, the rest don't start, the output covers the cases that ran, and the run fails. `--subset <name>` runs a named set of cases from `evals/cases.ts` instead of `--only id,id`; a name that matches no case fails the run.
 
-**Smoke eval.** The [Smoke eval](.github/workflows/smoke-eval.yml) workflow runs on every push to `main` that changes `prompts/`, `reviewers/`, `src/config.ts` or `guidance/`, and by hand (`gh workflow run smoke-eval.yml`). It runs in the `code-review` environment with the organization's Bedrock key, so it never runs on a pull request. It runs the `smoke` subset, 4 cases with the must-not case and one large diff, with the production guidance, `--precision off` and `--max-cost 8`. `node evals/smoke.ts check` then compares the run with `evals/baseline-smoke.json`, posts the comparison to the job summary, and fails when caught drops by more than 2, when the must-not case is flagged, or when the two cover different cases. `evals/out` is scanned for secrets before it's uploaded as the `smoke-eval` artifact, since artifacts on a public repository are public. With no baseline yet, the run passes with a notice, failing only on a flagged must-not case. To adopt a run as the baseline, the first time or after an intended change in what the smoke cases catch or a change to the subset, take `proposed-baseline-smoke.json` from that run's artifact, which is the run without its agent calls and raw findings, and commit it in a pull request:
+**Smoke eval.** The [Smoke eval](.github/workflows/smoke-eval.yml) workflow runs on every push to `main` that changes `prompts/`, `reviewers/`, `src/config.ts` or `guidance/`, and by hand (`gh workflow run smoke-eval.yml`). It runs in the `code-review` environment with the organization's Bedrock key, so it never runs on a pull request. It runs the `smoke` subset, 4 cases with the must-not case and one large diff, with the production guidance, `--precision off` and `--max-cost 8`. `node evals/smoke.ts check` then compares the run with `evals/baseline-smoke.json`, posts the comparison to the job summary, and fails when caught drops by more than 2, when the must-not case is flagged, or when the two cover different cases. `evals/out` is scanned for secrets before it's uploaded as the `smoke-eval` artifact, since artifacts on a public repository are public. Until `evals/baseline-smoke.json` is first committed, the run passes with a notice, failing only on a flagged must-not case. Once it has been, a run without it fails and names the commit that deleted it, so deleting the file can't turn the gate off; the workflow checks out the full history so the check can tell, and the check fails on a shallow checkout. To adopt a run as the baseline, the first time or after an intended change in what the smoke cases catch or a change to the subset, take `proposed-baseline-smoke.json` from that run's artifact, which is the run without its agent calls and raw findings, and commit it in a pull request:
 
 ```powershell
 gh run download <run id> -n smoke-eval -D evals/out/smoke
diff --git a/test/smoke-eval.test.ts b/test/smoke-eval.test.ts
index 8bbbc6f..9100c72 100644
--- a/test/smoke-eval.test.ts
+++ b/test/smoke-eval.test.ts
@@ -390,6 +390,52 @@ test('the smoke eval workflow runs only on main and by hand, in the code-review
   for (const line of uses) assert.match(line, /^uses: [\w.-]+\/[\w.-]+@[0-9a-f]{40} # v\d+\.\d+\.\d+$/, line);
 });
 
+// The steps of the smoke eval workflow, read just far enough to see each step's name, id, if, uses and run.
+interface WorkflowStep { name: string; id: string; if: string; uses: string; run: string; text: string }
+function workflowSteps(): WorkflowStep[] {
+  const lines = readFileSync(path.join(root, '.github', 'workflows', 'smoke-eval.yml'), 'utf8').replace(/\r\n/g, '\n').split('\n');
+  const steps: WorkflowStep[] = [];
+  for (const line of lines.slice(lines.indexOf('    steps:') + 1)) {
+    if (line.startsWith('      - ')) steps.push({ name: '', id: '', if: '', uses: '', run: '', text: '' });
+    const step = steps.at(-1);
+    if (!step || line.trimStart().startsWith('#')) continue;
+    step.text += `${line}\n`;
+    const field = /^ {6}(?:- | {2})([\w-]+): ?(.*)$/.exec(line);
+    if (!field) continue;
+    const [, key = '', value = ''] = field;
+    if (key === 'name' || key === 'id' || key === 'if' || key === 'uses' || key === 'run') step[key] = value;
+  }
+  return steps;
+}
+
+test('the smoke eval workflow uploads the outputs only after the secret scan passes', () => {
+  const steps = workflowSteps();
+  const scan = steps.findIndex((s) => s.id === 'scan');
+  const upload = steps.findIndex((s) => s.uses.startsWith('actions/upload-artifact@'));
+  assert.ok(scan >= 0 && upload >= 0, 'a scan step and an upload step');
+  assert.ok(scan < upload, 'the scan runs before the upload');
+  assert.equal(steps[scan]!.if, '${{ !cancelled() }}');
+  assert.equal(steps[scan]!.run, 'node evals/smoke.ts scan evals/out');
+  assert.ok(steps[scan]!.text.includes('AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}'), 'the scan knows the Bedrock key');
+  assert.equal(steps[upload]!.if, "${{ !cancelled() && steps.scan.outcome == 'success' }}");
+  assert.ok(steps[upload]!.text.includes('path: evals/out/'), 'the upload takes the folder the scan read');
+  assert.equal(upload, steps.length - 1, 'nothing runs after the upload');
+});
+
+test('the smoke eval workflow fails the job when the run or the check fails, and checks out the full history', () => {
+  const steps = workflowSteps();
+  for (const step of steps) assert.doesNotMatch(step.text, /continue-on-error/, step.name || step.uses || step.run);
+  const run = steps.findIndex((s) => s.name === 'Run the smoke subset');
+  const compare = steps.findIndex((s) => s.name === 'Compare with the baseline');
+  assert.ok(run >= 0 && compare > run, 'the check runs after the run');
+  assert.equal(steps[run]!.if, '');
+  assert.ok(steps[run]!.run.includes('--label "$LABEL"'), steps[run]!.run);
+  assert.equal(steps[compare]!.if, '');
+  assert.equal(steps[compare]!.run, 'node evals/smoke.ts check "evals/out/$LABEL.json"');
+  assert.ok(steps[0]!.uses.startsWith('actions/checkout@'), steps[0]!.uses);
+  assert.ok(steps[0]!.text.includes('fetch-depth: 0'), 'the full history');
+});
+
 const evalCase: EvalCase = {
   id: 'x', repository: 'o/r', pullRequest: 5, title: 'T', body: 'Closes #3 and fixes #4', base: 'b'.repeat(40), head: 'h'.repeat(40),
   expect: [], mustNotFlag: [], source: '',

From 123b14f34ab09152ae7992cd8c15692dbade42a6 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 16:57:10 -0600
Subject: [PATCH 04/10] Plan step 4: Write the eval output before failing a
 capped run, in a tested function

---
 evals/run.ts            | 14 +++++++++++---
 test/smoke-eval.test.ts | 23 ++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/evals/run.ts b/evals/run.ts
index 95c4b3d..827293a 100644
--- a/evals/run.ts
+++ b/evals/run.ts
@@ -262,6 +262,16 @@ export function tallyCase(
 // How the output names a finding that raised a concern.
 const raisedLabel = (finding: Finding & { reviewers?: string[] }) => `${finding.file}:${finding.line} (${(finding.reviewers ?? []).join(', ')})`;
 
+// Writes the run's output to <dir>/<label>.json, then fails a run that passed --max-cost, so the smoke eval still checks
+// and uploads the cases that ran.
+export function writeRunOutput(dir: string, label: string, output: { summary: unknown; cases: unknown[] }, capError: string | undefined): string {
+  mkdirSync(dir, { recursive: true });
+  const file = path.join(dir, `${label}.json`);
+  writeFileSync(file, JSON.stringify(output, null, 2));
+  if (capError) throw new Error(capError);
+  return file;
+}
+
 async function main(): Promise<void> {
   const maxCost = maxCostFrom(process.argv);
   const precisionOn = parsePrecision(arg('precision'));
@@ -418,9 +428,7 @@ async function main(): Promise<void> {
   if (precision) console.log(`Precision ${percent(precision.precision)}: ${precision.matched} matched + ${precision.real} real of ${precision.posted} blockers and should-fixes; ${precision.unsure} unsure, ${precision.notReal} not real, ${precision.mustNot} must-not`);
   console.log(`Findings per case: mean ${summary.findingsPerCase.mean.toFixed(1)}, max ${summary.findingsPerCase.max}; review cost per case: mean $${summary.reviewCostPerCase.mean.toFixed(2)}, max $${summary.reviewCostPerCase.max.toFixed(2)}`);
   if (shadowNames.size) console.log(`Shadow reviewers (${[...shadowNames].join(', ')}), not posted: caught ${caughtShadow}/${expected}; must-not violations ${shadowViolations}; ${shadowFindingCount} findings`);
-  mkdirSync(path.join(root, 'evals', 'out'), { recursive: true });
-  writeFileSync(path.join(root, 'evals', 'out', `${label}.json`), JSON.stringify({ summary, cases: report }, null, 2));
-  if (cap.error) throw new Error(cap.error);
+  writeRunOutput(path.join(root, 'evals', 'out'), label, { summary, cases: report }, cap.error);
 }
 
 if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
diff --git a/test/smoke-eval.test.ts b/test/smoke-eval.test.ts
index 9100c72..a2e3a82 100644
--- a/test/smoke-eval.test.ts
+++ b/test/smoke-eval.test.ts
@@ -8,7 +8,7 @@ import { pathToFileURL } from 'node:url';
 import type { EvalCase } from '../evals/cases.ts';
 import type { EvalOutput } from '../evals/compare.ts';
 import type { CasePrecision, RealityJudge } from '../evals/precision.ts';
-import { capOutcome, caseIssues, casePrecision, casePullRequest, maxCostFrom, parseMaxCost, parsePrecision, runCapped, selectCases, tallyCase, type CaseTally } from '../evals/run.ts';
+import { capOutcome, caseIssues, casePrecision, casePullRequest, maxCostFrom, parseMaxCost, parsePrecision, runCapped, selectCases, tallyCase, writeRunOutput, type CaseTally } from '../evals/run.ts';
 import { baselinePathFrom, checkFiles, commandData, lastBaselineCommit, proposedName, scanDir, smokeCheck, trimmed } from '../evals/smoke.ts';
 import type { CallRecord } from '../src/agent.ts';
 import type { Finding } from '../src/review.ts';
@@ -113,6 +113,27 @@ test('capOutcome passes a run within the cap, and records nothing without one',
   assert.deepEqual(capOutcome(undefined, { costUsd: 80, ran: 2, exceeded: false }, 2, 'smoke'), {});
 });
 
+test('writeRunOutput writes the output before it fails a run that passed the cap', () => {
+  const dir = mkdtempSync(path.join(tmpdir(), 'run-output-'));
+  try {
+    const out = path.join(dir, 'out');
+    const output = { summary: { caught: 1 }, cases: [{ id: 'large' }] };
+    assert.throws(() => writeRunOutput(out, 'smoke', output, 'Stopped over --max-cost'), /Stopped over --max-cost/);
+    assert.deepEqual(JSON.parse(readFileSync(path.join(out, 'smoke.json'), 'utf8')), output);
+    assert.equal(writeRunOutput(out, 'full', output, undefined), path.join(out, 'full.json'));
+    assert.ok(existsSync(path.join(out, 'full.json')));
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// main needs Bedrock, so its source is checked: a capped run fails only after writeRunOutput has written the output.
+test("evals/run.ts's main fails a capped run only through writeRunOutput", () => {
+  const source = readFileSync(path.join(root, 'evals', 'run.ts'), 'utf8');
+  assert.ok(source.includes("writeRunOutput(path.join(root, 'evals', 'out'), label, { summary, cases: report }, cap.error);"));
+  assert.doesNotMatch(source, /throw new Error\(cap\.error\)/);
+});
+
 test('the smoke subset is 4 existing cases, with the must-not case', () => {
   const smoke = selectCases(allCases, undefined, 'smoke');
   assert.equal(smoke.length, 4);

From 377eb22080924f033db98332fdfc79a2fe02d4e2 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 16:58:12 -0600
Subject: [PATCH 05/10] Plan step 5: Count a private member's comment for the
 loop limit

---
 src/main.ts        |  4 ++--
 src/pause.ts       | 20 ++++++++++++-----
 test/pause.test.ts | 56 +++++++++++++++++++++++++++++++---------------
 3 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/src/main.ts b/src/main.ts
index 225ba54..588f070 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -181,10 +181,11 @@ async function main(): Promise<void> {
     await teach({ outDir, repository, number, githubToken, event }, command);
     return;
   }
+  const isWriter = writerCheck(repository, githubToken, fetch, log);
   // After three reviews with no word from the maintainer, an automatic run reviews nothing and makes no model call. Its
   // status lets the finish step complete an acknowledged check run, so a paused run never leaves one in progress.
   if (!event.comment) {
-    const pause = await checkPause({ repository, number, token: githubToken, issueComments: await issueComments(number), log });
+    const pause = await checkPause({ repository, number, token: githubToken, issueComments: await issueComments(number), isWriter, log });
     if (pause !== 'review') writeStatus(pausedStatus());
     if (pause === 'paused') {
       writeLog('Paused, and the notice is posted already; comment /review to resume.');
@@ -204,7 +205,6 @@ async function main(): Promise<void> {
   const credentials = takeCredentials();
   const started = Date.now();
   const pullRequestEvent = event.pull_request ?? event.issue;
-  const isWriter = writerCheck(repository, githubToken, fetch, log);
   const authorCanWrite = await isWriter(pullRequestEvent?.user?.login, pullRequestEvent?.author_association);
   const pullRequest = readPullRequest(event, repository, authorCanWrite);
   run.headSha = pullRequest.headSha || undefined;
diff --git a/src/pause.ts b/src/pause.ts
index cd2245e..0f5385b 100644
--- a/src/pause.ts
+++ b/src/pause.ts
@@ -2,12 +2,16 @@
 // after that, an automatic review is skipped before any model call, and the bot posts once that it has paused. The
 // autopilot posts as the maintainer's account, with comments that start with "Autopilot:", so those don't count as the
 // maintainer's. A /review comment from a writer is the maintainer's, so it always reviews and resumes.
+//
+// Write access is the engine's writer check (writers.ts): the comment's association, or failing that its author's
+// permission on the repository, so a private organization member's comment counts, and one whose permission can't be
+// read doesn't.
 
 import type { RunStatus } from './checks.ts';
-import { writerAssociations } from './command.ts';
 import { readComments, type ListedComment } from './comments.ts';
 import { writeLog } from './log.ts';
 import { botLogin, summaryMarker, supersededMarker } from './results.ts';
+import type { WriterCheck } from './writers.ts';
 
 export const maxUnansweredReviews = 3;
 export const pausedMarker = '<!-- ncode-review:paused -->';
@@ -32,9 +36,10 @@ export function pausedStatus(): RunStatus {
 }
 
 // A comment from someone with write access in their own words: not a bot's, and not one the autopilot posted.
-export function isMaintainerComment(comment: ListedComment): boolean {
+export async function isMaintainerComment(comment: ListedComment, isWriter: WriterCheck): Promise<boolean> {
   if (comment.user?.type === 'Bot' || comment.user?.login === botLogin) return false;
-  return writerAssociations.has(comment.author_association ?? '') && !(comment.body ?? '').trimStart().startsWith(autopilotPrefix);
+  if ((comment.body ?? '').trimStart().startsWith(autopilotPrefix)) return false;
+  return isWriter(comment.user?.login, comment.author_association);
 }
 
 export interface PauseState {
@@ -47,8 +52,10 @@ export interface PauseState {
 const time = (comment: ListedComment) => Date.parse(comment.created_at ?? '');
 
 // Counts from a pull request's issue comments and review comments together, in any order.
-export function unansweredReviews(comments: ListedComment[]): PauseState {
-  const since = Math.max(...comments.filter(isMaintainerComment).map(time).filter(Number.isFinite));
+export async function unansweredReviews(comments: ListedComment[], isWriter: WriterCheck): Promise<PauseState> {
+  const maintainers: ListedComment[] = [];
+  for (const comment of comments) if (await isMaintainerComment(comment, isWriter)) maintainers.push(comment);
+  const since = Math.max(...maintainers.map(time).filter(Number.isFinite));
   const later = comments.filter((c) => c.user?.login === botLogin && time(c) > since);
   const body = (c: ListedComment) => c.body ?? '';
   return {
@@ -73,6 +80,7 @@ export async function checkPause(options: {
   number: number;
   token: string | undefined;
   issueComments: ListedComment[] | undefined;
+  isWriter: WriterCheck;
   fetchPage?: typeof fetch;
   log: (line: string) => void;
 }): Promise<PauseDecision> {
@@ -82,7 +90,7 @@ export async function checkPause(options: {
     log("::warning::Couldn't read the pull request's comments, so the loop limit isn't checked.");
     return 'review';
   }
-  const state = unansweredReviews([...issueComments, ...reviewComments]);
+  const state = await unansweredReviews([...issueComments, ...reviewComments], options.isWriter);
   const decision = pauseDecision(state);
   if (decision !== 'review') log(`${state.reviews} reviews since the last comment from someone with write access; paused until a /review.`);
   return decision;
diff --git a/test/pause.test.ts b/test/pause.test.ts
index 295d5a1..72ced23 100644
--- a/test/pause.test.ts
+++ b/test/pause.test.ts
@@ -3,9 +3,11 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'no
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { after, test } from 'node:test';
+import { writerAssociations } from '../src/command.ts';
 import type { ListedComment } from '../src/comments.ts';
 import { checkPause, isMaintainerComment, maxUnansweredReviews, pauseDecision, pausedMarker, pausedNotice, pausedStatus, postPausedNotice, unansweredReviews } from '../src/pause.ts';
 import { headMarker, summaryMarker, supersededComment } from '../src/results.ts';
+import { writerCheck, type WriterCheck } from '../src/writers.ts';
 import { pullRequestCheckout, reviewStep, runStep } from './steps.ts';
 
 const made: string[] = [];
@@ -24,6 +26,8 @@ const maintainer = { login: 'polewskm', type: 'User' };
 const at = (minute: number) => `2026-10-31T08:${String(minute).padStart(2, '0')}:00Z`;
 const reviewed = 'a'.repeat(40);
 let ids = 0;
+// Writers by their association alone, with no permission lookup.
+const byAssociation: WriterCheck = async (_login, association) => writerAssociations.has(association ?? '');
 
 function summary(minute: number, collapsed = false, head = reviewed): ListedComment {
   const body = collapsed ? supersededComment(head, 'https://github.com/o/r/pull/1#issuecomment-1') : `${summaryMarker}\n${headMarker(head)}\n### 🤖 Review complete`;
@@ -46,12 +50,12 @@ const threeReviews = (head = reviewed): ListedComment[] => [
   summary(6, false, head),
 ];
 
-test('unansweredReviews counts the summaries, collapsed or not, since the last comment from someone with write access', () => {
-  assert.deepEqual(unansweredReviews(threeReviews()), { reviews: 3, noticePosted: false });
-  assert.deepEqual(unansweredReviews([summary(0), summary(1)]), { reviews: 2, noticePosted: false });
+test('unansweredReviews counts the summaries, collapsed or not, since the last comment from someone with write access', async () => {
+  assert.deepEqual(await unansweredReviews(threeReviews(), byAssociation), { reviews: 3, noticePosted: false });
+  assert.deepEqual(await unansweredReviews([summary(0), summary(1)], byAssociation), { reviews: 2, noticePosted: false });
 });
 
-test('comments from the autopilot, a bot or someone without write access are not the maintainer\'s', () => {
+test('comments from the autopilot, a bot or someone without write access are not the maintainer\'s', async () => {
   const comments = [
     ...threeReviews(),
     said(7, '  Autopilot: replied in the thread.'),
@@ -59,24 +63,40 @@ test('comments from the autopilot, a bot or someone without write access are not
     said(9, 'Bump the version', 'MEMBER', { login: 'dependabot[bot]', type: 'Bot' }),
     said(10, 'I am the bot', 'MEMBER', bot),
   ];
-  assert.equal(unansweredReviews(comments).reviews, 3);
-  assert.ok(isMaintainerComment(said(0, 'Fine as is.', 'COLLABORATOR')));
-  assert.ok(!isMaintainerComment(said(0, 'Autopilot: fixed in abc123.', 'OWNER')));
+  assert.equal((await unansweredReviews(comments, byAssociation)).reviews, 3);
+  assert.ok(await isMaintainerComment(said(0, 'Fine as is.', 'COLLABORATOR'), byAssociation));
+  assert.ok(!(await isMaintainerComment(said(0, 'Autopilot: fixed in abc123.', 'OWNER'), byAssociation)));
 });
 
-test('a maintainer comment in a review thread, or a /review, starts the count again', () => {
-  assert.equal(unansweredReviews([...threeReviews(), said(7, 'This one can stay.')]).reviews, 0);
-  assert.equal(unansweredReviews([...threeReviews(), said(7, '/review', 'OWNER'), summary(8)]).reviews, 1);
+test('a maintainer comment in a review thread, or a /review, starts the count again', async () => {
+  assert.equal((await unansweredReviews([...threeReviews(), said(7, 'This one can stay.')], byAssociation)).reviews, 0);
+  assert.equal((await unansweredReviews([...threeReviews(), said(7, '/review', 'OWNER'), summary(8)], byAssociation)).reviews, 1);
 });
 
-test('the fourth review without a maintainer comment pauses, once', () => {
+test('the fourth review without a maintainer comment pauses, once', async () => {
   assert.equal(maxUnansweredReviews, 3);
   assert.equal(pauseDecision({ reviews: 2, noticePosted: false }), 'review');
-  assert.equal(pauseDecision(unansweredReviews(threeReviews())), 'pause');
+  assert.equal(pauseDecision(await unansweredReviews(threeReviews(), byAssociation)), 'pause');
   const notice = { id: ++ids, user: bot, author_association: 'NONE', created_at: at(7), body: pausedNotice() };
-  assert.equal(pauseDecision(unansweredReviews([...threeReviews(), notice])), 'paused');
+  assert.equal(pauseDecision(await unansweredReviews([...threeReviews(), notice], byAssociation)), 'paused');
   // A notice from before the maintainer last spoke is from an earlier pause.
-  assert.equal(pauseDecision(unansweredReviews([summary(0), notice, said(8, 'Carry on.'), summary(9), summary(10), summary(11)])), 'pause');
+  assert.equal(pauseDecision(await unansweredReviews([summary(0), notice, said(8, 'Carry on.'), summary(9), summary(10), summary(11)], byAssociation)), 'pause');
+});
+
+test("a private organization member's comment counts as the maintainer's when the permission lookup says write, and not when it fails", async () => {
+  const comments = [...threeReviews(), said(7, 'Looks fine.', 'CONTRIBUTOR', { login: 'private-member', type: 'User' })];
+  const urls: string[] = [];
+  const writes = (async (url: string | URL | Request) => {
+    urls.push(String(url));
+    return Response.json({ permission: 'write' });
+  }) as typeof fetch;
+  assert.equal((await unansweredReviews(comments, writerCheck('o/r', 't', writes))).reviews, 0);
+  // The bot's summaries and the autopilot's comments are never looked up.
+  assert.deepEqual(urls, ['https://api.github.com/repos/o/r/collaborators/private-member/permission']);
+  const lines: string[] = [];
+  const outage = (async () => new Response('', { status: 502 })) as unknown as typeof fetch;
+  assert.equal((await unansweredReviews(comments, writerCheck('o/r', 't', outage, (line) => lines.push(line)))).reviews, 3);
+  assert.equal(lines.length, 1);
 });
 
 test('the paused notice says how to resume and carries its marker', () => {
@@ -92,13 +112,13 @@ test('checkPause reads the review comments too, and reviews when it cannot read
     return Response.json([said(7, 'Fine as is.')]);
   }) as typeof fetch;
   const logs: string[] = [];
-  assert.equal(await checkPause({ repository: 'o/r', number: 1, token: 't', issueComments: threeReviews(), fetchPage, log: (l) => logs.push(l) }), 'review');
+  assert.equal(await checkPause({ repository: 'o/r', number: 1, token: 't', issueComments: threeReviews(), isWriter: byAssociation, fetchPage, log: (l) => logs.push(l) }), 'review');
   assert.deepEqual(urls, ['https://api.github.com/repos/o/r/pulls/1/comments?per_page=100&page=1']);
   const empty = (async () => Response.json([])) as unknown as typeof fetch;
-  assert.equal(await checkPause({ repository: 'o/r', number: 1, token: 't', issueComments: threeReviews(), fetchPage: empty, log: (l) => logs.push(l) }), 'pause');
+  assert.equal(await checkPause({ repository: 'o/r', number: 1, token: 't', issueComments: threeReviews(), isWriter: byAssociation, fetchPage: empty, log: (l) => logs.push(l) }), 'pause');
   const refused = (async () => new Response('', { status: 500 })) as unknown as typeof fetch;
-  assert.equal(await checkPause({ repository: 'o/r', number: 1, token: 't', issueComments: threeReviews(), fetchPage: refused, log: (l) => logs.push(l) }), 'review');
-  assert.equal(await checkPause({ repository: 'o/r', number: 1, token: 't', issueComments: undefined, fetchPage: empty, log: (l) => logs.push(l) }), 'review');
+  assert.equal(await checkPause({ repository: 'o/r', number: 1, token: 't', issueComments: threeReviews(), isWriter: byAssociation, fetchPage: refused, log: (l) => logs.push(l) }), 'review');
+  assert.equal(await checkPause({ repository: 'o/r', number: 1, token: 't', issueComments: undefined, isWriter: byAssociation, fetchPage: empty, log: (l) => logs.push(l) }), 'review');
   assert.equal(logs.filter((l) => l === "::warning::Couldn't read the pull request's comments, so the loop limit isn't checked.").length, 2);
   assert.ok(logs.includes("::warning::Couldn't read /repos/o/r/pulls/1/comments (HTTP 500); the run goes on without those comments."));
 });

From 93a76c0d870252bbf47f1379b3a51d68bea4b0c6 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 16:58:41 -0600
Subject: [PATCH 06/10] Plan step 6: Read a private member's open issues for
 synthesis

---
 src/issues.ts       | 37 +++++++++++++++++++++++++++++++------
 src/main.ts         |  2 +-
 test/issues.test.ts | 42 +++++++++++++++++++++++++++++++++++++-----
 3 files changed, 69 insertions(+), 12 deletions(-)

diff --git a/src/issues.ts b/src/issues.ts
index 26812ae..b8f4497 100644
--- a/src/issues.ts
+++ b/src/issues.ts
@@ -1,17 +1,28 @@
 // The repository's open issues, for synthesis to mark a finding one of them already tracks. Only issues opened by
-// someone with write access are read, so a stranger can't file an issue whose title waves a finding away.
+// someone with write access are read, so a stranger can't file an issue whose title waves a finding away. Write access
+// is the engine's writer check (writers.ts), so a private organization member's issues count.
 
 import { writerAssociations } from './command.ts';
+import type { WriterCheck } from './writers.ts';
 
 // Pages of 100 open issues read at most, the most recently updated first.
 export const maxOpenIssuePages = 2;
 const pageSize = 100;
+// Authors looked up at most per review, beyond the ones whose association says they can write: a public repository's
+// open issues could otherwise cost an API call each.
+export const maxIssueAuthorLookups = 20;
 
 // The open issues, leaving out pull requests and the issues `exclude` names, such as the ones this pull request closes:
 // those are its own work, not somewhere else to track a finding. It keeps the pages it could read, and none when it
 // couldn't read the first.
-export async function readOpenIssues(repository: string, token: string | undefined, exclude: number[] = [], fetchPage: typeof fetch = fetch): Promise<{ number: number; title: string }[]> {
-  type Listed = { number?: number; title?: string; author_association?: string; pull_request?: unknown };
+export async function readOpenIssues(
+  repository: string,
+  token: string | undefined,
+  isWriter: WriterCheck,
+  exclude: number[] = [],
+  fetchPage: typeof fetch = fetch,
+): Promise<{ number: number; title: string }[]> {
+  type Listed = { number?: number; title?: string; author_association?: string; user?: { login?: string; type?: string }; pull_request?: unknown };
   const read: Listed[] = [];
   for (let page = 1; page <= maxOpenIssuePages; page++) {
     let issues: Listed[];
@@ -28,7 +39,21 @@ export async function readOpenIssues(repository: string, token: string | undefin
     if (issues.length < pageSize) break;
   }
   const skip = new Set(exclude);
-  return read
-    .filter((i) => typeof i.number === 'number' && !i.pull_request && writerAssociations.has(i.author_association ?? '') && !skip.has(i.number))
-    .map((i) => ({ number: i.number!, title: i.title ?? '' }));
+  const looked = new Set<string>();
+  const openedByWriter = async (issue: Listed): Promise<boolean> => {
+    if (writerAssociations.has(issue.author_association ?? '')) return true;
+    const login = issue.user?.login;
+    if (!login || issue.user?.type === 'Bot') return false;
+    if (!looked.has(login)) {
+      if (looked.size >= maxIssueAuthorLookups) return false;
+      looked.add(login);
+    }
+    return isWriter(login, issue.author_association);
+  };
+  const kept: { number: number; title: string }[] = [];
+  for (const issue of read) {
+    if (typeof issue.number !== 'number' || issue.pull_request || skip.has(issue.number)) continue;
+    if (await openedByWriter(issue)) kept.push({ number: issue.number, title: issue.title ?? '' });
+  }
+  return kept;
 }
diff --git a/src/main.ts b/src/main.ts
index 588f070..2aff0b7 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -231,7 +231,7 @@ async function main(): Promise<void> {
     log,
   });
   // Only a full review's synthesis can mark a finding as tracked in an open issue.
-  const openIssues = plan ? [] : await readOpenIssues(repository, githubToken, closedIssues(pullRequest.body));
+  const openIssues = plan ? [] : await readOpenIssues(repository, githubToken, isWriter, closedIssues(pullRequest.body));
 
   const choice = await chooseProvider(credentials);
   log(`Provider: ${choice.reason}`);
diff --git a/test/issues.test.ts b/test/issues.test.ts
index efb5fcd..4114d2f 100644
--- a/test/issues.test.ts
+++ b/test/issues.test.ts
@@ -4,13 +4,15 @@ import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { after, test } from 'node:test';
 import type { AgentRequest, AgentResult, AgentRun } from '../src/agent.ts';
+import { writerAssociations } from '../src/command.ts';
 import { postThreads } from '../src/comments.ts';
 import type { Gathered } from '../src/gather.ts';
-import { maxOpenIssuePages, readOpenIssues } from '../src/issues.ts';
+import { maxIssueAuthorLookups, maxOpenIssuePages, readOpenIssues } from '../src/issues.ts';
 import { annotations, botLogin, checkSummary, checkText, checkTitle, headMarker, summaryComment, summaryMarker, type Results } from '../src/results.ts';
 import { mergeSynthesis, review, settleTracked, type Charters, type Finding, type ReviewedFinding } from '../src/review.ts';
 import { buildResults } from '../src/review-results.ts';
 import { placeFindings, reviewBody } from '../src/threads.ts';
+import type { WriterCheck } from '../src/writers.ts';
 import { pullRequestCheckout, reviewStep } from './steps.ts';
 
 const made: string[] = [];
@@ -19,6 +21,8 @@ after(() => {
 });
 
 const issue = (number: number, association = 'MEMBER', extra: Record<string, unknown> = {}) => ({ number, title: `Issue ${number}`, author_association: association, ...extra });
+// Writers by their association alone, with no permission lookup.
+const byAssociation: WriterCheck = async (_login, association) => writerAssociations.has(association ?? '');
 
 test('readOpenIssues keeps the open issues a writer opened, and leaves out pull requests and the issues this one closes', async () => {
   const urls: string[] = [];
@@ -28,7 +32,7 @@ test('readOpenIssues keeps the open issues a writer opened, and leaves out pull
     headers.push(new Headers(init?.headers));
     return Response.json([issue(1), issue(2, 'NONE'), issue(3, 'OWNER', { pull_request: {} }), issue(4, 'COLLABORATOR'), issue(5, 'CONTRIBUTOR'), issue(6)]);
   }) as typeof fetch;
-  assert.deepEqual(await readOpenIssues('o/r', 't', [6], fetchPage), [{ number: 1, title: 'Issue 1' }, { number: 4, title: 'Issue 4' }]);
+  assert.deepEqual(await readOpenIssues('o/r', 't', byAssociation, [6], fetchPage), [{ number: 1, title: 'Issue 1' }, { number: 4, title: 'Issue 4' }]);
   assert.deepEqual(urls, ['https://api.github.com/repos/o/r/issues?state=open&sort=updated&direction=desc&per_page=100&page=1']);
   assert.equal(headers[0]!.get('authorization'), 'Bearer t');
 });
@@ -39,15 +43,43 @@ test('readOpenIssues reads at most its page cap, and keeps what it read when a l
     calls++;
     return Response.json(Array.from({ length: 100 }, (_, i) => issue(calls * 1000 + i)));
   }) as unknown as typeof fetch;
-  assert.equal((await readOpenIssues('o/r', 't', [], full)).length, 100 * maxOpenIssuePages);
+  assert.equal((await readOpenIssues('o/r', 't', byAssociation, [], full)).length, 100 * maxOpenIssuePages);
   assert.equal(calls, maxOpenIssuePages);
   let page = 0;
   const failing = (async () => (++page === 1 ? Response.json(Array.from({ length: 100 }, (_, i) => issue(i + 1))) : new Response('', { status: 502 }))) as unknown as typeof fetch;
-  assert.equal((await readOpenIssues('o/r', 't', [], failing)).length, 100);
+  assert.equal((await readOpenIssues('o/r', 't', byAssociation, [], failing)).length, 100);
   const rejecting = (async () => {
     throw new TypeError('fetch failed');
   }) as unknown as typeof fetch;
-  assert.deepEqual(await readOpenIssues('o/r', undefined, [], rejecting), []);
+  assert.deepEqual(await readOpenIssues('o/r', undefined, byAssociation, [], rejecting), []);
+});
+
+test('readOpenIssues keeps an issue a private organization member opened, and never looks a bot up', async () => {
+  const fetchPage = (async () => Response.json([
+    issue(7, 'CONTRIBUTOR', { user: { login: 'private-member', type: 'User' } }),
+    issue(8, 'NONE', { user: { login: 'dependabot[bot]', type: 'Bot' } }),
+    issue(9, 'CONTRIBUTOR', { user: { login: 'outsider', type: 'User' } }),
+  ])) as unknown as typeof fetch;
+  const lookups: string[] = [];
+  const isWriter: WriterCheck = async (login) => {
+    lookups.push(login ?? '');
+    return login === 'private-member';
+  };
+  assert.deepEqual(await readOpenIssues('o/r', 't', isWriter, [], fetchPage), [{ number: 7, title: 'Issue 7' }]);
+  assert.deepEqual(lookups, ['private-member', 'outsider']);
+});
+
+test('readOpenIssues looks up at most maxIssueAuthorLookups authors', async () => {
+  const fetchPage = (async () => Response.json(Array.from({ length: 25 }, (_, i) => issue(i + 1, 'CONTRIBUTOR', { user: { login: `author-${i + 1}`, type: 'User' } })))) as unknown as typeof fetch;
+  const lookups: string[] = [];
+  const isWriter: WriterCheck = async (login) => {
+    lookups.push(login ?? '');
+    return true;
+  };
+  const kept = await readOpenIssues('o/r', 't', isWriter, [], fetchPage);
+  assert.equal(maxIssueAuthorLookups, 20);
+  assert.deepEqual(kept.map((i) => i.number), Array.from({ length: 20 }, (_, i) => i + 1));
+  assert.equal(lookups.length, 20);
 });
 
 const finding = (title: string, severity: Finding['severity'] = 'should-fix'): Finding =>

From a028ddb1676253c1137110ec0ce1d96779b83f78 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 16:59:24 -0600
Subject: [PATCH 07/10] Plan step 7: Answer a private member's Autopilot reply

---
 src/follow-up.ts       | 17 ++++++++++-------
 src/publish.ts         |  3 ++-
 test/follow-up.test.ts | 40 +++++++++++++++++++++++++++++-----------
 3 files changed, 41 insertions(+), 19 deletions(-)

diff --git a/src/follow-up.ts b/src/follow-up.ts
index f58be5d..0208533 100644
--- a/src/follow-up.ts
+++ b/src/follow-up.ts
@@ -1,10 +1,10 @@
 // Closing the loop on the bot's own review threads, in the publish step with the app's token. A re-review that finds an
 // earlier finding fixed, where the thread is outdated or its lines changed, resolves the thread. One that finds it still
-// open after the autopilot replied posts one reply saying why and asking the maintainer, and never a second. It never
-// resolves or answers anyone else's thread. A thread someone with write access dismissed, by resolving it without a code
-// change or with a 👎, counts as already posted, so its finding isn't posted again.
+// open after the autopilot replied, from an account with write access by the engine's writer check (its association, or
+// failing that its permission on the repository), posts one reply saying why and asking the maintainer, and never a
+// second. It never resolves or answers anyone else's thread. A thread someone with write access dismissed, by resolving
+// it without a code change or with a 👎, counts as already posted, so its finding isn't posted again.
 
-import { writerAssociations } from './command.ts';
 import { errorText } from './comments.ts';
 import { writeLog } from './log.ts';
 import { assertNoSecrets } from './outputs.ts';
@@ -12,6 +12,7 @@ import { autopilotPrefix } from './pause.ts';
 import { botAuthor, type ThreadNode } from './rereview.ts';
 import type { Results } from './results.ts';
 import { parsePosted, snapDistance, type PostedThread } from './threads.ts';
+import type { WriterCheck } from './writers.ts';
 
 export const stillOpenMarker = '<!-- ncode-review:still-open -->';
 
@@ -79,11 +80,12 @@ export function stillOpenReply(headSha: string, note: string, maintainer: string
 
 // What to do with each of the bot's unresolved threads after a re-review, from the status it gave each one. A full
 // review gives none, so it does nothing: it can't tell an earlier finding it no longer raises from one it words anew.
-export function planFollowUps(
+export async function planFollowUps(
   threads: ThreadNode[],
   results: Pick<Results, 'headSha' | 'reReview'>,
+  isWriter: WriterCheck,
   dismissed: ReadonlySet<string> = new Set(),
-): FollowUp[] {
+): Promise<FollowUp[]> {
   const reReview = results.reReview;
   if (!reReview) return [];
   const statuses = new Map(reReview.earlier.map((e) => [e.url, e]));
@@ -100,8 +102,9 @@ export function planFollowUps(
     if (status.status !== 'open' || dismissed.has(thread.id)) continue;
     const last = thread.comments.nodes.at(-1);
     const login = last?.author?.login;
-    if (!last || !login || login === botAuthor || !writerAssociations.has(last.authorAssociation) || !last.body.trimStart().startsWith(autopilotPrefix)) continue;
+    if (!last || !login || login === botAuthor || !last.body.trimStart().startsWith(autopilotPrefix)) continue;
     if (alreadyAnswered(thread)) continue;
+    if (!(await isWriter(login, last.authorAssociation))) continue;
     actions.push({ kind: 'reply', thread, id: posted.id, body: stillOpenReply(results.headSha, status.note, login) });
   }
   return actions;
diff --git a/src/publish.ts b/src/publish.ts
index 3fb99c4..8c9220d 100644
--- a/src/publish.ts
+++ b/src/publish.ts
@@ -22,6 +22,7 @@ import { postPausedNotice, type Paused } from './pause.ts';
 import { readReviewThreads } from './rereview.ts';
 import { annotations, checkSummary, checkText, checkTitle, summaryComment, type Results } from './results.ts';
 import { publishTeaching } from './teach.ts';
+import { writerCheck } from './writers.ts';
 
 const interactUrl = 'https://github.com/NCodeGroup/code-review#commands';
 
@@ -108,7 +109,7 @@ async function main(): Promise<void> {
 
   await postThreads(api, results, [token], writeLog, dismissed.map((d) => d.posted));
   if (threads) {
-    const actions = planFollowUps(threads, results, new Set(dismissed.map((d) => d.thread.id)));
+    const actions = await planFollowUps(threads, results, writerCheck(results.repository, token, fetch, writeLog), new Set(dismissed.map((d) => d.thread.id)));
     await followUpThreads((query, variables) => graphql(api, query, variables), actions, [token]);
   }
 
diff --git a/test/follow-up.test.ts b/test/follow-up.test.ts
index 38b34f0..0fdc8e8 100644
--- a/test/follow-up.test.ts
+++ b/test/follow-up.test.ts
@@ -3,12 +3,14 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { after, test } from 'node:test';
+import { writerAssociations } from '../src/command.ts';
 import { postThreads } from '../src/comments.ts';
 import { dismissedThreads, followUpThreads, planFollowUps, stillOpenMarker, stillOpenReply } from '../src/follow-up.ts';
 import type { EarlierState, ThreadNode } from '../src/rereview.ts';
 import type { Results } from '../src/results.ts';
 import type { ReviewedFinding } from '../src/review.ts';
 import { threadBody } from '../src/threads.ts';
+import { writerCheck, type WriterCheck } from '../src/writers.ts';
 import { runStep } from './steps.ts';
 
 const made: string[] = [];
@@ -54,6 +56,8 @@ function thread(o: ThreadOptions = {}): ThreadNode {
 }
 
 const url = (t: ThreadNode) => t.first.nodes[0]!.url;
+// Writers by their association alone, with no permission lookup.
+const byAssociation: WriterCheck = async (_login, association) => writerAssociations.has(association ?? '');
 const headSha = 'c'.repeat(40);
 
 // What a re-review hands the publish step about each earlier thread.
@@ -61,15 +65,15 @@ function reReviewed(statuses: [ThreadNode, EarlierState, string?][], changedLine
   return { headSha, reReview: { changedLines, earlier: statuses.map(([t, status, note]) => ({ url: url(t), status, note: note ?? '' })) } };
 }
 
-test('planFollowUps resolves the bot\'s thread of a fixed finding when the thread is outdated or its lines changed', () => {
+test('planFollowUps resolves the bot\'s thread of a fixed finding when the thread is outdated or its lines changed', async () => {
   const outdated = thread({ outdated: true });
   const changed = thread({ finding: finding({ id: 'F2', file: 'src/b.ts', line: 20 }) });
   const untouched = thread({ finding: finding({ id: 'F3', file: 'src/c.ts', line: 40 }) });
-  const actions = planFollowUps([outdated, changed, untouched], reReviewed([[outdated, 'fixed'], [changed, 'fixed'], [untouched, 'fixed']], { 'src/b.ts': [22], 'src/c.ts': [10] }));
+  const actions = await planFollowUps([outdated, changed, untouched], reReviewed([[outdated, 'fixed'], [changed, 'fixed'], [untouched, 'fixed']], { 'src/b.ts': [22], 'src/c.ts': [10] }), byAssociation);
   assert.deepEqual(actions.map((a) => [a.kind, a.id, a.thread.id]), [['resolve', 'F1', outdated.id], ['resolve', 'F2', changed.id]]);
 });
 
-test('planFollowUps never resolves anyone else\'s thread, a resolved one, or one the re-review did not find fixed', () => {
+test('planFollowUps never resolves anyone else\'s thread, a resolved one, or one the re-review did not find fixed', async () => {
   const theirs = thread({ author: 'polewskm', outdated: true });
   const resolved = thread({ outdated: true, resolved: true });
   const open = thread({ outdated: true });
@@ -77,9 +81,9 @@ test('planFollowUps never resolves anyone else\'s thread, a resolved one, or one
   const unchecked = thread({ outdated: true });
   const unlisted = thread({ outdated: true });
   const all = [theirs, resolved, open, answered, unchecked, unlisted];
-  assert.deepEqual(planFollowUps(all, reReviewed([[theirs, 'fixed'], [resolved, 'fixed'], [open, 'open'], [answered, 'answered'], [unchecked, 'unchecked']])), []);
+  assert.deepEqual(await planFollowUps(all, reReviewed([[theirs, 'fixed'], [resolved, 'fixed'], [open, 'open'], [answered, 'answered'], [unchecked, 'unchecked']]), byAssociation), []);
   // A full review gives no status for the earlier threads, so it resolves nothing.
-  assert.deepEqual(planFollowUps(all, { headSha }), []);
+  assert.deepEqual(await planFollowUps(all, { headSha }, byAssociation), []);
 });
 
 test('followUpThreads resolves a thread with resolveReviewThread, and only warns when GitHub refuses a resolve or a reply', async () => {
@@ -94,7 +98,7 @@ test('followUpThreads resolves a thread with resolveReviewThread, and only warns
     return {};
   };
   const logs: string[] = [];
-  const actions = planFollowUps([first, answerable, second], reReviewed([[first, 'fixed'], [answerable, 'open'], [second, 'fixed']]));
+  const actions = await planFollowUps([first, answerable, second], reReviewed([[first, 'fixed'], [answerable, 'open'], [second, 'fixed']]), byAssociation);
   await followUpThreads(graphql, actions, ['app-token'], (l) => logs.push(l));
   assert.deepEqual(sent.map(([query, variables]) => [/resolveReviewThread\(input: \{ threadId: \$id \}\)/.test(query), variables['id']]), [[true, first.id], [false, answerable.id], [true, second.id]]);
   assert.deepEqual(logs, [
@@ -129,11 +133,11 @@ test('an Autopilot reply on a still-valid finding gets exactly one bot reply acr
   const logs: string[] = [];
   store.reply(stillValid, 'Autopilot: fixed in abc123');
   // The next review finds it still open, and says why, once.
-  const second = planFollowUps([stillValid], reReviewed([[stillValid, 'open', 'The second run still reads `state` before\nthe lock.']]));
+  const second = await planFollowUps([stillValid], reReviewed([[stillValid, 'open', 'The second run still reads `state` before\nthe lock.']]), byAssociation);
   await followUpThreads(store.graphql, second, ['app-token'], (l) => logs.push(l));
   // The autopilot answers again, and the review after that still finds it open.
   store.reply(stillValid, 'Autopilot: fixed in def456');
-  const third = planFollowUps([stillValid], reReviewed([[stillValid, 'open', 'Still reads `state` before the lock.']]));
+  const third = await planFollowUps([stillValid], reReviewed([[stillValid, 'open', 'Still reads `state` before the lock.']]), byAssociation);
   await followUpThreads(store.graphql, third, ['app-token'], (l) => logs.push(l));
   assert.equal(store.replies.length, 1);
   assert.deepEqual(third, []);
@@ -144,7 +148,7 @@ test('an Autopilot reply on a still-valid finding gets exactly one bot reply acr
   assert.deepEqual(logs, ['Replied in the thread of F1 that it stays open.']);
 });
 
-test('planFollowUps replies only to the autopilot, from someone with write access, and never on a dismissed thread or one it cannot read whole', () => {
+test('planFollowUps replies only to the autopilot, from someone with write access, and never on a dismissed thread or one it cannot read whole', async () => {
   const answeredByHand = thread({ replies: [{ login: 'polewskm', body: 'I will look at this.' }] });
   const stranger = thread({ replies: [{ login: 'drive-by', association: 'NONE', body: 'Autopilot: fixed in abc123' }] });
   const dismissed = thread({ replies: [{ login: 'polewskm', body: 'Autopilot: fixed in abc123' }] });
@@ -154,10 +158,24 @@ test('planFollowUps replies only to the autopilot, from someone with write acces
   long.comments.nodes = long.comments.nodes.slice(1);
   const answerable = thread({ replies: [{ login: 'polewskm', body: '  Autopilot: fixed in abc123' }] });
   const all = [answeredByHand, stranger, dismissed, long, answerable];
-  const actions = planFollowUps(all, reReviewed(all.map((t) => [t, 'open'])), new Set([dismissed.id]));
+  const actions = await planFollowUps(all, reReviewed(all.map((t) => [t, 'open'])), byAssociation, new Set([dismissed.id]));
   assert.deepEqual(actions.map((a) => [a.kind, a.thread.id]), [['reply', answerable.id]]);
   // A finding the re-review found fixed is never answered as still open, even when its thread can't be resolved.
-  assert.deepEqual(planFollowUps([answerable], reReviewed([[answerable, 'fixed']])), []);
+  assert.deepEqual(await planFollowUps([answerable], reReviewed([[answerable, 'fixed']]), byAssociation), []);
+});
+
+test("planFollowUps answers a private organization member's Autopilot reply when the permission lookup says write, and not when it fails", async () => {
+  const privateReply = () => thread({ replies: [{ login: 'private-member', association: 'CONTRIBUTOR', body: 'Autopilot: fixed in abc123' }] });
+  const writes = (async () => Response.json({ permission: 'write' })) as unknown as typeof fetch;
+  const answered = privateReply();
+  const actions = await planFollowUps([answered], reReviewed([[answered, 'open']]), writerCheck('o/r', 't', writes));
+  assert.deepEqual(actions.map((a) => [a.kind, a.thread.id]), [['reply', answered.id]]);
+  assert.equal(actions[0]!.kind === 'reply' && actions[0]!.body.split('\n').at(-1), '@private-member: needs you.');
+  const lines: string[] = [];
+  const outage = (async () => new Response('', { status: 502 })) as unknown as typeof fetch;
+  const unread = privateReply();
+  assert.deepEqual(await planFollowUps([unread], reReviewed([[unread, 'open']]), writerCheck('o/r', 't', outage, (line) => lines.push(line))), []);
+  assert.equal(lines.length, 1);
 });
 
 test('stillOpenReply keeps the note to one line of text and asks the maintainer on its last line', () => {

From 3555b0738c17b09380c6407081a116b45d578db6 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 17:00:22 -0600
Subject: [PATCH 08/10] Plan step 8: Word the lookup warning for every use, and
 document private members

---
 README.md      | 19 ++++++++++++++-----
 src/writers.ts |  5 +++--
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index af33ecc..399fb98 100644
--- a/README.md
+++ b/README.md
@@ -42,8 +42,11 @@ The reviewers see a pull request's description, its commit subjects and the issu
 their authors can write to the repository. The engine asks the repository's permissions when the event doesn't say so:
 GitHub reports an organization member whose membership is private as a contributor. The prompt says whether the pull
 request has a description, and when one was left out, so no reviewer reports a description it wasn't shown as missing.
-A `/review` comment still counts only when its `author_association` names a writer, as the workflow's `if:` below
-checks before the engine starts, so a private organization member can't start a review that way.
+The loop limit's count of maintainer comments, the open issues a finding can be tracked in and the `Autopilot:` replies
+the bot answers use the same check. A `/review` comment still counts only when its `author_association` names a writer,
+as the workflow's `if:` below checks before the engine starts: a permission lookup there would let more people start a
+run that holds the model credentials. So a private organization member can't start a review that way;
+[Commands](#commands) says what they can do.
 
 Besides the reviewers, **rule checks** go through the rules the context stage picked for the pull request, twelve at a time, on Sonnet: each call checks its rules one by one against every part of the diff, searching the changed files for what the rule's check names. They report as `rule-check` and run on every review, outside the reviewer roster, so selection can't skip them. In the evals they caught about one more expected finding in 55, at about 70% more cost. `CODE_REVIEW_RULE_CHECKS=off` turns them off.
 
@@ -192,6 +195,9 @@ the source above but hasn't been tried on a consumer yet.
 ## Commands
 
 Comment on a pull request, as a top-level comment, not a reply in a review thread. Only people with write access can.
+GitHub reports an organization member whose membership is private as a contributor, so their commands are ignored.
+To use them, change your membership from Private to Public on the organization's People page, or have an admin add
+you to the repository as a collaborator.
 
 | Comment | What happens |
 |---|---|
@@ -329,7 +335,9 @@ After three reviews with no comment from someone with write access in between, c
 automatic review stops before any model call and posts once "Paused: comment `/review` to resume". An acknowledged
 "Code review" check run is completed as "Review paused", so it never stays in progress. Comments the
 autopilot posts as the maintainer's account, which start with `Autopilot:`, don't count as the maintainer's. A
-`/review` from someone with write access always reviews, and counts as their comment.
+`/review` from someone with write access always reviews, and counts as their comment. A comment counts as from someone
+with write access by its author's association or, when that doesn't say so, their permission on the repository; one
+whose permission can't be read doesn't count.
 
 ### Re-reviews
 
@@ -361,8 +369,9 @@ of review converge:
   says so instead, and its verdict is never Approve;
 - it closes the loop on the bot's own threads, never anyone else's: it resolves the thread of an earlier finding it
   found fixed when the thread is outdated or its lines changed since the last review. When it finds one still open
-  and the thread's last comment is an `Autopilot:` reply, it replies once to say why it stays open, ending with a line
-  that asks the maintainer (`@<them>: needs you.`), and never a second time in that thread.
+  and the thread's last comment is an `Autopilot:` reply from someone with write access, it replies once to say why it
+  stays open, ending with a line that asks the maintainer (`@<them>: needs you.`), and never a second time in that
+  thread.
 
 A `/review` of the commit already reviewed, or a last reviewed head that can't be fetched any more (after a force-push),
 gets a full review instead. A review whose reviewers all failed doesn't count as the last review, nor does a re-review
diff --git a/src/writers.ts b/src/writers.ts
index e014c5f..6d74ef1 100644
--- a/src/writers.ts
+++ b/src/writers.ts
@@ -3,7 +3,8 @@
 //
 // author_association alone misses writers: GitHub reports an organization member whose membership is private as
 // CONTRIBUTOR, in event payloads and to the workflow's token. So an author it doesn't name a writer is looked up in the
-// repository's permissions, which the workflow's token can read (metadata: read).
+// repository's permissions, which the workflow's token can read (metadata: read). The loop limit (pause.ts), the open
+// issues synthesis reads (issues.ts) and the replies to the autopilot (follow-up.ts) use the same check.
 
 import { writerAssociations } from './command.ts';
 import { errorText } from './comments.ts';
@@ -34,7 +35,7 @@ export function writerCheck(repository: string, token: string | undefined, fetch
     } catch (error) {
       reason = errorText(error);
     }
-    log(`::warning::Couldn't read ${login}'s permission (${reason}); their text counts as a non-writer's and stays out of the prompts.`);
+    log(`::warning::Couldn't read ${login}'s permission (${reason}); their text counts as a non-writer's.`);
     return false;
   };
   return async (login, association) => {

From 21ba2f5a2244eb129ab73994cf531e13b0c71e96 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 17:00:35 -0600
Subject: [PATCH 09/10] Plan step 9: Write the changelog entries

---
 CHANGELOG.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ea28062..0ab5bf9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 - The tests check that the evals' review cost leaves out the concern judge's and precision judge's calls, which the case cost counts, so `eval:compare` never reports the judges' spend as review cost ([#107](https://github.com/NCodeGroup/code-review/issues/107)).
 - The tests check what a `/review explain` run leaves for the publish step: only `answer.json` for an unknown finding, without a model call, `trace.json` too when the call ran, the `answered` output, and never `results.json` ([#108](https://github.com/NCodeGroup/code-review/issues/108)).
 - The evals read the issues a case's pull request closes with the review step's own reader, so the two can't drift apart ([#111](https://github.com/NCodeGroup/code-review/issues/111)).
+- The smoke eval fails when `evals/baseline-smoke.json` is missing after it has been committed, naming the commit that deleted it, so deleting the baseline can't turn the gate off; until the first baseline is committed it still passes with a notice. Its workflow checks out the full history for this ([#93](https://github.com/NCodeGroup/code-review/issues/93)).
+- The tests check that the smoke eval workflow uploads its outputs only after the secret scan passes and fails the job when the run or the check fails, and that `evals/run.ts` writes its output before it fails a run over `--max-cost` ([#93](https://github.com/NCodeGroup/code-review/issues/93)).
+- A private organization member's comment counts as the maintainer's for the loop limit, their open issues can track a finding, and the bot answers their `Autopilot:` replies: these use the engine's writer check, the author's association or else their permission on the repository, and a lookup that fails counts as no write access. The README says how such a member can use `/review`, whose gate still reads only the association ([#89](https://github.com/NCodeGroup/code-review/issues/89)).
 
 ### Fixed
 

From fd9d5440edd170d9d8e0791fe80e02dd7c9695c7 Mon Sep 17 00:00:00 2001
From: Marcin Polewski <marcin.polewski@gmail.com>
Date: Sat, 31 Oct 2026 17:15:48 -0600
Subject: [PATCH 10/10] Plan step 12: Share the association writer stub between
 the tests

---
 test/follow-up.test.ts | 6 ++----
 test/issues.test.ts    | 4 +---
 test/pause.test.ts     | 6 ++----
 test/writer-stub.ts    | 5 +++++
 4 files changed, 10 insertions(+), 11 deletions(-)
 create mode 100644 test/writer-stub.ts

diff --git a/test/follow-up.test.ts b/test/follow-up.test.ts
index 0fdc8e8..5584bd2 100644
--- a/test/follow-up.test.ts
+++ b/test/follow-up.test.ts
@@ -3,15 +3,15 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { after, test } from 'node:test';
-import { writerAssociations } from '../src/command.ts';
 import { postThreads } from '../src/comments.ts';
 import { dismissedThreads, followUpThreads, planFollowUps, stillOpenMarker, stillOpenReply } from '../src/follow-up.ts';
 import type { EarlierState, ThreadNode } from '../src/rereview.ts';
 import type { Results } from '../src/results.ts';
 import type { ReviewedFinding } from '../src/review.ts';
 import { threadBody } from '../src/threads.ts';
-import { writerCheck, type WriterCheck } from '../src/writers.ts';
+import { writerCheck } from '../src/writers.ts';
 import { runStep } from './steps.ts';
+import { byAssociation } from './writer-stub.ts';
 
 const made: string[] = [];
 after(() => {
@@ -56,8 +56,6 @@ function thread(o: ThreadOptions = {}): ThreadNode {
 }
 
 const url = (t: ThreadNode) => t.first.nodes[0]!.url;
-// Writers by their association alone, with no permission lookup.
-const byAssociation: WriterCheck = async (_login, association) => writerAssociations.has(association ?? '');
 const headSha = 'c'.repeat(40);
 
 // What a re-review hands the publish step about each earlier thread.
diff --git a/test/issues.test.ts b/test/issues.test.ts
index 4114d2f..84625fd 100644
--- a/test/issues.test.ts
+++ b/test/issues.test.ts
@@ -4,7 +4,6 @@ import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { after, test } from 'node:test';
 import type { AgentRequest, AgentResult, AgentRun } from '../src/agent.ts';
-import { writerAssociations } from '../src/command.ts';
 import { postThreads } from '../src/comments.ts';
 import type { Gathered } from '../src/gather.ts';
 import { maxIssueAuthorLookups, maxOpenIssuePages, readOpenIssues } from '../src/issues.ts';
@@ -14,6 +13,7 @@ import { buildResults } from '../src/review-results.ts';
 import { placeFindings, reviewBody } from '../src/threads.ts';
 import type { WriterCheck } from '../src/writers.ts';
 import { pullRequestCheckout, reviewStep } from './steps.ts';
+import { byAssociation } from './writer-stub.ts';
 
 const made: string[] = [];
 after(() => {
@@ -21,8 +21,6 @@ after(() => {
 });
 
 const issue = (number: number, association = 'MEMBER', extra: Record<string, unknown> = {}) => ({ number, title: `Issue ${number}`, author_association: association, ...extra });
-// Writers by their association alone, with no permission lookup.
-const byAssociation: WriterCheck = async (_login, association) => writerAssociations.has(association ?? '');
 
 test('readOpenIssues keeps the open issues a writer opened, and leaves out pull requests and the issues this one closes', async () => {
   const urls: string[] = [];
diff --git a/test/pause.test.ts b/test/pause.test.ts
index 72ced23..ddac7b6 100644
--- a/test/pause.test.ts
+++ b/test/pause.test.ts
@@ -3,12 +3,12 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'no
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { after, test } from 'node:test';
-import { writerAssociations } from '../src/command.ts';
 import type { ListedComment } from '../src/comments.ts';
 import { checkPause, isMaintainerComment, maxUnansweredReviews, pauseDecision, pausedMarker, pausedNotice, pausedStatus, postPausedNotice, unansweredReviews } from '../src/pause.ts';
 import { headMarker, summaryMarker, supersededComment } from '../src/results.ts';
-import { writerCheck, type WriterCheck } from '../src/writers.ts';
+import { writerCheck } from '../src/writers.ts';
 import { pullRequestCheckout, reviewStep, runStep } from './steps.ts';
+import { byAssociation } from './writer-stub.ts';
 
 const made: string[] = [];
 after(() => {
@@ -26,8 +26,6 @@ const maintainer = { login: 'polewskm', type: 'User' };
 const at = (minute: number) => `2026-10-31T08:${String(minute).padStart(2, '0')}:00Z`;
 const reviewed = 'a'.repeat(40);
 let ids = 0;
-// Writers by their association alone, with no permission lookup.
-const byAssociation: WriterCheck = async (_login, association) => writerAssociations.has(association ?? '');
 
 function summary(minute: number, collapsed = false, head = reviewed): ListedComment {
   const body = collapsed ? supersededComment(head, 'https://github.com/o/r/pull/1#issuecomment-1') : `${summaryMarker}\n${headMarker(head)}\n### 🤖 Review complete`;
diff --git a/test/writer-stub.ts b/test/writer-stub.ts
new file mode 100644
index 0000000..32859df
--- /dev/null
+++ b/test/writer-stub.ts
@@ -0,0 +1,5 @@
+import { writerAssociations } from '../src/command.ts';
+import type { WriterCheck } from '../src/writers.ts';
+
+// Judges by the author's association alone, as the `/review` gate does, so a test needs no permission lookup.
+export const byAssociation: WriterCheck = async (_login, association) => writerAssociations.has(association ?? '');