D1: benchmark corpus + CI accuracy gate (closes #86)

.benchmark-fixtures-sha

@@ -0,0 +1 @@
+a8d77b3b4fefc54e1f2f5e88ee2c5224956891ca

.github/workflows/benchmark.yml

@@ -0,0 +1,53 @@
+name: benchmark
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: benchmark-${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout extension
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install deps
+        run: npm ci
+
+      - name: Build extension
+        run: npm run build:ext
+
+      - name: Read fixtures SHA
+        id: sha
+        run: echo "value=$(cat .benchmark-fixtures-sha)" >> "$GITHUB_OUTPUT"
+
+      - name: Clone fixtures repo at pinned SHA
+        run: |
+          git clone --depth 1 https://github.com/recost-dev/extension_benchmark.git benchmark-fixtures
+          cd benchmark-fixtures
+          git fetch --depth 1 origin ${{ steps.sha.outputs.value }}
+          git checkout ${{ steps.sha.outputs.value }}
+
+      - name: Run benchmark
+        run: npm run benchmark -- --fixtures ./benchmark-fixtures --report benchmark/report.json
+
+      - name: Upload report artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-report
+          path: benchmark/report.json
+          if-no-files-found: ignore

benchmark/README.md

@@ -0,0 +1,46 @@
+# Benchmark
+
+Hand-labeled accuracy gate for the scanner. Fails CI when precision or recall drops > 1pp.
+
+## Quick start
+
+```bash
+# Smoke (no clone, fast):
+npm run benchmark:smoke
+
+# Full (requires extension_benchmark cloned as sibling dir):
+git clone https://github.com/recost-dev/extension_benchmark.git ../extension_benchmark
+npm run benchmark
+```
+
+## Layout
+
+- `runner.ts` — orchestrates per-fixture scan + metric computation. Reads `--fixtures <dir>` (default `../extension_benchmark`).
+- `metrics.ts` — pure precision/recall math.
+- `schema.ts` — `expected.json` types + validator.
+- `report.ts` — console + markdown report formatting.
+- `baseline.json` — committed metric baseline. Gate compares current run vs this.
+- `_smoke/` — tiny in-repo fixture for runner development.
+
+## CI
+
+`.github/workflows/benchmark.yml` runs on every PR. It reads `.benchmark-fixtures-sha` (repo root), clones `extension_benchmark` at that SHA, then runs `npm run benchmark`.
+
+## Adding a fixture
+
+Fixtures live in `extension_benchmark`, not here. To add one:
+
+1. Open a PR in `recost-dev/extension_benchmark` with a new `<slug>/src/...`, `<slug>/expected.json`, `<slug>/FIXTURE.md`.
+2. Once merged, bump `.benchmark-fixtures-sha` in `extension`.
+3. Run `npm run benchmark -- --update-baseline` locally and commit `baseline.json` if the new fixture changed it.
+
+## Updating the baseline
+
+When a PR legitimately improves accuracy:
+
+```bash
+npm run benchmark -- --update-baseline
+git add benchmark/baseline.json
+```
+
+Commit the new baseline in the same PR as the code change. Explain why in the PR description.

benchmark/_smoke/FIXTURE.md

@@ -0,0 +1,7 @@
+# _smoke fixture
+
+Hand-crafted minimal fixture for runner unit tests. Not vendored from any upstream.
+
+**Scope:** 2 OpenAI calls (chat completion + embeddings in a loop) + 1 expected `batch` finding (embeddings in a `for` loop should be a single batch request).
+
+**Why this fixture exists:** The benchmark runner needs something to iterate against during development without cloning `extension_benchmark`. This fixture is deliberately small and stable.

benchmark/_smoke/expected.json

@@ -0,0 +1,32 @@
+{
+  "schemaVersion": 1,
+  "fixtureSlug": "_smoke",
+  "endpoints": [
+    {
+      "file": "src/openai-helper.ts",
+      "function": "complete",
+      "line": 6,
+      "provider": "openai",
+      "method": "chat.completions.create",
+      "must_detect": true
+    },
+    {
+      "file": "src/openai-helper.ts",
+      "function": "embedBatch",
+      "line": 16,
+      "provider": "openai",
+      "method": "embeddings.create",
+      "must_detect": true
+    }
+  ],
+  "findings": [
+    {
+      "file": "src/openai-helper.ts",
+      "function": "embedBatch",
+      "line": 16,
+      "type": "batch",
+      "is_true_positive": true,
+      "notes": "Embedding call inside a for loop over items — should be batched."
+    }
+  ]
+}

benchmark/_smoke/src/openai-helper.ts

@@ -0,0 +1,20 @@
+import OpenAI from "openai";
+
+const client = new OpenAI();
+
+export async function complete(prompt: string): Promise<string> {
+  const result = await client.chat.completions.create({
+    model: "gpt-4o-mini",
+    messages: [{ role: "user", content: prompt }],
+  });
+  return result.choices[0]?.message.content ?? "";
+}
+
+export async function embedBatch(items: string[]): Promise<number[][]> {
+  const out: number[][] = [];
+  for (const item of items) {
+    const res = await client.embeddings.create({ model: "text-embedding-3-small", input: item });
+    out.push(res.data[0]?.embedding ?? []);
+  }
+  return out;
+}

benchmark/baseline.json

@@ -0,0 +1,7 @@
+{
+  "detectionPrecision": 0.2988505747126437,
+  "detectionRecall": 0.4262295081967213,
+  "providerAttributionAccuracy": 0.7959183673469388,
+  "findingPrecision": 0.07142857142857142,
+  "findingRecall": 0.3333333333333333
+}

benchmark/metrics.ts

@@ -0,0 +1,182 @@
+import type { ExpectedJson } from "./schema";
+
+const LINE_TOLERANCE = 2;
+
+export interface DetectedEndpoint {
+  file: string;
+  line: number;
+  provider: string;
+  method: string;
+}
+
+export interface DetectedFinding {
+  file: string;
+  line: number;
+  type: string;
+}
+
+export interface MetricsReport {
+  detectionPrecision: number;
+  detectionRecall: number;
+  providerAttributionAccuracy: number;
+  findingPrecision: number;
+  findingRecall: number;
+  /** Per-fixture breakdown, useful for diagnosing where a regression landed. */
+  perFixture: PerFixtureMetrics[];
+}
+
+export interface PerFixtureMetrics {
+  fixtureSlug: string;
+  detectionPrecision: number;
+  detectionRecall: number;
+  providerAttributionAccuracy: number;
+  findingPrecision: number;
+  findingRecall: number;
+  truePositiveEndpoints: number;
+  falsePositiveEndpoints: number;
+  falseNegativeEndpoints: number;
+  truePositiveFindings: number;
+  falsePositiveFindings: number;
+  falseNegativeFindings: number;
+  /** Number of expected endpoints with a same-file-line detection whose provider was not "unknown". */
+  providerAttributionTotal: number;
+  /** Of those, how many had the detected provider matching the expected provider. */
+  providerAttributionCorrect: number;
+}
+
+/**
+ * Compute metrics for a single fixture against its expected.json + scanner output.
+ * Pure function — no I/O.
+ *
+ * Matching rules:
+ *  - An expected endpoint matches a detected endpoint when file matches AND provider matches AND
+ *    (method OR methodSignature equivalent) matches AND |detected.line - expected.line| <= 2.
+ *  - When file+line match but provider differs, the detection is BOTH a false positive (wrong provider)
+ *    AND a false negative (missed the expected entry).
+ *  - Provider attribution accuracy: of the detected endpoints with provider !== "unknown" that matched
+ *    something on file+line, what fraction got the provider right?
+ *  - Findings: file + type + |line| <= 2 are matched. Same FP/FN logic.
+ */
+export function computeMetrics(
+  expected: ExpectedJson,
+  detected: DetectedEndpoint[],
+  detectedFindings: DetectedFinding[]
+): PerFixtureMetrics {
+  const endpointMatch = matchPairs(
+    expected.endpoints,
+    detected,
+    (e, d) => e.file === d.file && e.provider === d.provider && methodsEquivalent(e.method, d.method) && Math.abs(e.line - d.line) <= LINE_TOLERANCE,
+  );
+
+  // Provider attribution: any detected entry on the right file+line, regardless of provider, counts toward the denominator.
+  let attributionCorrect = 0;
+  let attributionTotal = 0;
+  for (const e of expected.endpoints) {
+    const sameFileLine = detected.find(d => d.file === e.file && Math.abs(d.line - e.line) <= LINE_TOLERANCE);
+    if (sameFileLine && sameFileLine.provider !== "unknown") {
+      attributionTotal += 1;
+      if (sameFileLine.provider === e.provider) attributionCorrect += 1;
+    }
+  }
+
+  const findingMatch = matchPairs(
+    expected.findings,
+    detectedFindings,
+    (e, d) => e.file === d.file && e.type === d.type && Math.abs(e.line - d.line) <= LINE_TOLERANCE,
+  );
+
+  return {
+    fixtureSlug: expected.fixtureSlug,
+    detectionPrecision: safeRatio(endpointMatch.truePositives, endpointMatch.truePositives + endpointMatch.falsePositives),
+    detectionRecall: safeRatio(endpointMatch.truePositives, endpointMatch.truePositives + endpointMatch.falseNegatives),
+    providerAttributionAccuracy: attributionTotal === 0 ? 1 : attributionCorrect / attributionTotal,
+    findingPrecision: safeRatio(findingMatch.truePositives, findingMatch.truePositives + findingMatch.falsePositives),
+    findingRecall: safeRatio(findingMatch.truePositives, findingMatch.truePositives + findingMatch.falseNegatives),
+    truePositiveEndpoints: endpointMatch.truePositives,
+    falsePositiveEndpoints: endpointMatch.falsePositives,
+    falseNegativeEndpoints: endpointMatch.falseNegatives,
+    truePositiveFindings: findingMatch.truePositives,
+    falsePositiveFindings: findingMatch.falsePositives,
+    falseNegativeFindings: findingMatch.falseNegatives,
+    providerAttributionTotal: attributionTotal,
+    providerAttributionCorrect: attributionCorrect,
+  };
+}
+
+/** Aggregate per-fixture metrics into a global report. */
+export function aggregate(perFixture: PerFixtureMetrics[]): MetricsReport {
+  const sum = perFixture.reduce(
+    (acc, m) => ({
+      tpE: acc.tpE + m.truePositiveEndpoints,
+      fpE: acc.fpE + m.falsePositiveEndpoints,
+      fnE: acc.fnE + m.falseNegativeEndpoints,
+      tpF: acc.tpF + m.truePositiveFindings,
+      fpF: acc.fpF + m.falsePositiveFindings,
+      fnF: acc.fnF + m.falseNegativeFindings,
+      attCorrect: acc.attCorrect + m.providerAttributionCorrect,
+      attTotal: acc.attTotal + m.providerAttributionTotal,
+    }),
+    { tpE: 0, fpE: 0, fnE: 0, tpF: 0, fpF: 0, fnF: 0, attCorrect: 0, attTotal: 0 },
+  );
+
+  return {
+    detectionPrecision: safeRatio(sum.tpE, sum.tpE + sum.fpE),
+    detectionRecall: safeRatio(sum.tpE, sum.tpE + sum.fnE),
+    providerAttributionAccuracy: sum.attTotal === 0 ? 1 : sum.attCorrect / sum.attTotal,
+    findingPrecision: safeRatio(sum.tpF, sum.tpF + sum.fpF),
+    findingRecall: safeRatio(sum.tpF, sum.tpF + sum.fnF),
+    perFixture,
+  };
+}
+
+interface MatchResult { truePositives: number; falsePositives: number; falseNegatives: number; }
+
+function matchPairs<E, D>(expected: E[], detected: D[], match: (e: E, d: D) => boolean): MatchResult {
+  const expectedMatched = new Array<boolean>(expected.length).fill(false);
+  const detectedMatched = new Array<boolean>(detected.length).fill(false);
+
+  for (let ei = 0; ei < expected.length; ei++) {
+    for (let di = 0; di < detected.length; di++) {
+      if (detectedMatched[di]) continue;
+      if (match(expected[ei], detected[di])) {
+        expectedMatched[ei] = true;
+        detectedMatched[di] = true;
+        break;
+      }
+    }
+  }
+
+  const truePositives = expectedMatched.filter(Boolean).length;
+  const falsePositives = detectedMatched.filter(m => !m).length;
+  const falseNegatives = expectedMatched.filter(m => !m).length;
+  return { truePositives, falsePositives, falseNegatives };
+}
+
+function safeRatio(numerator: number, denominator: number): number {
+  if (denominator === 0) return 1;
+  return numerator / denominator;
+}
+
+/**
+ * Methods are considered equivalent if either:
+ *  - exact string match
+ *  - both refer to the same SDK chain regardless of separator (defensive — fixture authors might use "." or " ")
+ *  - one is a dot-suffix of the other (e.g. fixture: "chat.completions.create",
+ *    scanner output: "client.chat.completions.create"). The AST scanner emits the
+ *    full receiver chain ("client.foo.bar.baz"); fixtures typically omit the receiver.
+ */
+function methodsEquivalent(a: string, b: string): boolean {
+  if (a === b) return true;
+  const na = normalize(a);
+  const nb = normalize(b);
+  if (na === nb) return true;
+  if (na.length > 0 && nb.length > 0) {
+    if (na.endsWith("." + nb)) return true;
+    if (nb.endsWith("." + na)) return true;
+  }
+  return false;
+}
+
+function normalize(s: string): string {
+  return s.replace(/\s+/g, "").toLowerCase();
+}