Clarify missing per-run AI credit rejection evidence

.github/scripts/check-max-ai-credits-artifact-wiring.test.cjs

@@ -0,0 +1,104 @@
+"use strict";
+
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const { test } = require("node:test");
+const { parseDocument } = require("../../actions/setup/js/work_queue_yaml.cjs");
+
+const workflowBase = path.join(__dirname, "../workflows/daily-max-ai-credits-test");
+const evidenceDirectory = "/tmp/gh-aw/";
+
+function readWorkflow(extension) {
+  let text = fs.readFileSync(`${workflowBase}.${extension}`, "utf8");
+  if (extension === "md") {
+    const frontmatter = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(text);
+    assert.ok(frontmatter, "workflow Markdown must have YAML frontmatter");
+    text = frontmatter[1];
+  }
+  const document = parseDocument(text);
+  assert.deepEqual(document.errors, [], "workflow YAML must parse without errors");
+  return document.toJS();
+}
+
+function assertVerifierDownloads(workflow) {
+  const job = workflow.jobs.verify_credit_limit;
+  assert.equal(job.needs, "agent");
+  assert.equal(job.if, "always() && !cancelled() && needs.agent.result != 'skipped'");
+  assert.equal(job.permissions.actions, "read");
+  const steps = job.steps;
+  const verifierIndex = steps.findIndex(step => step.run === `node .github/scripts/check-max-ai-credits.cjs ${evidenceDirectory}`);
+  assert.ok(verifierIndex >= 0, "verifier must read the artifact extraction directory");
+  for (const name of ["agent", "info"]) {
+    const downloads = steps.filter(step => step.uses?.startsWith("actions/download-artifact@") && step.with?.name === name);
+    assert.equal(downloads.length, 1, `expected exactly one ${name} download`);
+    const download = downloads[0];
+    assert.deepEqual(download.with, { name, path: evidenceDirectory }, `${name} must use current-run, same-repository download defaults`);
+    assert.ok(steps.indexOf(download) < verifierIndex, `${name} must be downloaded before verification`);
+    assert.equal(download.if, undefined, `${name} download must not be skipped`);
+    assert.equal(download["continue-on-error"], undefined, `${name} download failure must fail the job`);
+  }
+}
+
+test("authored workflow downloads agent evidence and current-run identity before verification", () => {
+  assertVerifierDownloads(readWorkflow("md"));
+});
+
+test("compiled workflow preserves the current-run artifact downloads", () => {
+  const workflow = readWorkflow("lock.yml");
+  assertVerifierDownloads(workflow);
+  const sourceSteps = readWorkflow("md").jobs.verify_credit_limit.steps;
+  for (const name of ["agent", "info"]) {
+    const sourceDownload = sourceSteps.find(step => step.with?.name === name);
+    const compiledDownload = workflow.jobs.verify_credit_limit.steps.find(step => step.with?.name === name);
+    assert.equal(sourceDownload.uses, "actions/download-artifact@v8.0.2");
+    assert.match(compiledDownload.uses, /^actions\/download-artifact@[a-f0-9]{40}$/);
+    assert.deepEqual(compiledDownload.with, sourceDownload.with);
+  }
+});
+
+test("info producer packages aw_info.json at the extraction root consumed by the verifier", () => {
+  const workflow = readWorkflow("lock.yml");
+  const producer = workflow.jobs.activation;
+  const uploads = producer.steps.filter(step => step.uses?.startsWith("actions/upload-artifact@") && step.with?.name === "info");
+  assert.equal(uploads.length, 1);
+  const upload = uploads[0];
+  assert.equal(upload.if, "success() || failure()");
+  assert.equal(upload.with.path, "/tmp/gh-aw/aw_info.json");
+  assert.equal(workflow.jobs.agent.needs, "activation", "identity producer must precede the agent");
+
+  const download = workflow.jobs.verify_credit_limit.steps.find(step => step.with?.name === "info");
+  assert.ok(download, "verifier must download the separate info artifact");
+  // A single-file upload is archived relative to its parent directory.
+  const archiveEntry = path.posix.relative(path.posix.dirname(upload.with.path), upload.with.path);
+  assert.equal(archiveEntry, "aw_info.json");
+  assert.equal(path.posix.join(download.with.path, archiveEntry), "/tmp/gh-aw/aw_info.json");
+
+  const agentUpload = workflow.jobs.agent.steps.find(step => step.uses?.startsWith("actions/upload-artifact@") && step.with?.name === "agent");
+  assert.ok(agentUpload);
+  const agentPaths = agentUpload.with.path.trim().split(/\s+/);
+  assert.ok(!agentPaths.includes(upload.with.path), "run identity is a separate artifact, not part of agent evidence");
+});
+
+test("artifact contract rejects missing, foreign-run, misplaced and late identity downloads", () => {
+  for (const mutate of [
+    steps =>
+      steps.splice(
+        steps.findIndex(step => step.with?.name === "info"),
+        1
+      ),
+    steps => (steps.find(step => step.with?.name === "info").with["run-id"] = "another-run"),
+    steps => (steps.find(step => step.with?.name === "info").with.path = "/tmp/gh-aw/info/"),
+    steps =>
+      steps.push(
+        steps.splice(
+          steps.findIndex(step => step.with?.name === "info"),
+          1
+        )[0]
+      ),
+  ]) {
+    const workflow = readWorkflow("md");
+    mutate(workflow.jobs.verify_credit_limit.steps);
+    assert.throws(() => assertVerifierDownloads(workflow), assert.AssertionError);
+  }
+});

.github/scripts/check-max-ai-credits.cjs

@@ -13,8 +13,43 @@ const LOG_PATHS = [
   "sandbox/firewall/logs/audit.jsonl",
   "sandbox/firewall/audit/log.jsonl",
 ];
+const TRACKER_PATHS = ["sandbox/firewall/logs/api-proxy-logs/token-tracker-audit.jsonl", "sandbox/firewall/audit/api-proxy-logs/token-tracker-audit.jsonl"];
+const USAGE_PATH = "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl";
 
-function verifyCreditLimit(directory) {
+function readJSON(directory, name) {
+  const filename = path.join(directory, name);
+  if (!fs.existsSync(filename)) throw new Error(`Missing ${name}; enforcement remains unverified.`);
+  try {
+    return JSON.parse(fs.readFileSync(filename, "utf8"));
+  } catch {
+    throw new Error(`Invalid JSON in ${name}; enforcement remains unverified.`);
+  }
+}
+
+function readJSONL(directory, name) {
+  const filename = path.join(directory, name);
+  if (!fs.existsSync(filename)) return null;
+  return fs
+    .readFileSync(filename, "utf8")
+    .split("\n")
+    .flatMap((line, index) => {
+      if (!line.trim()) return [];
+      try {
+        return [JSON.parse(line)];
+      } catch {
+        throw new Error(`Invalid firewall JSONL in ${name} at line ${index + 1}.`);
+      }
+    });
+}
+
+function verifyCreditLimit(directory, expected = { runId: process.env.GITHUB_RUN_ID, runAttempt: process.env.GITHUB_RUN_ATTEMPT, repository: process.env.GITHUB_REPOSITORY }) {
+  if (!/^[1-9]\d*$/.test(expected.runId || "") || !/^[1-9]\d*$/.test(expected.runAttempt || "") || !expected.repository) {
+    throw new Error("Expected repository, run ID and run attempt are required; enforcement remains unverified.");
+  }
+  const info = readJSON(directory, "aw_info.json");
+  if (String(info?.run_id) !== String(expected.runId) || String(info?.run_attempt) !== String(expected.runAttempt) || info?.repository !== expected.repository) {
+    throw new Error("Agent artifact repository/run/attempt does not match this verification job; enforcement remains unverified.");
+  }
   const configText = fs.readFileSync(path.join(directory, "awf-config.json"), "utf8");
   let config;
   try {
@@ -27,25 +62,75 @@ function verifyCreditLimit(directory) {
   }
 
   let rejectionFound = false;
+  let observedFiles = 0;
+  let observedRecords = 0;
+  let mismatchedRejections = 0;
   for (const relativePath of LOG_PATHS) {
-    const filename = path.join(directory, relativePath);
-    if (!fs.existsSync(filename)) continue;
-    const lines = fs.readFileSync(filename, "utf8").split("\n");
-    for (let index = 0; index < lines.length; index++) {
-      if (!lines[index].trim()) continue;
-      let entry;
-      try {
-        entry = JSON.parse(lines[index]);
-      } catch {
-        throw new Error(`Invalid firewall JSONL in ${relativePath} at line ${index + 1}.`);
+    const entries = readJSONL(directory, relativePath);
+    if (entries === null) continue;
+    observedFiles++;
+    observedRecords += entries.length;
+    for (const entry of entries) {
+      if (entry?.event !== "ai_credits_limit_exceeded") continue;
+      if (
+        (entry.run_id !== undefined && String(entry.run_id) !== String(expected.runId)) ||
+        (entry.run_attempt !== undefined && String(entry.run_attempt) !== String(expected.runAttempt)) ||
+        (entry.repository !== undefined && entry.repository !== expected.repository)
+      ) {
+        throw new Error("Firewall rejection repository/run/attempt does not match this verification job; enforcement remains unverified.");
       }
-      if (entry?.event === "ai_credits_limit_exceeded" && entry.max_ai_credits === 1 && typeof entry.total_ai_credits === "number" && Number.isFinite(entry.total_ai_credits) && entry.total_ai_credits >= 1 && entry.hard_cap === false) {
+      if (entry.max_ai_credits === 1 && typeof entry.total_ai_credits === "number" && Number.isFinite(entry.total_ai_credits) && entry.total_ai_credits >= 1 && entry.hard_cap === false) {
         rejectionFound = true;
+      } else {
+        mismatchedRejections++;
       }
     }
   }
+  if (rejectionFound && mismatchedRejections) {
+    throw new Error("Firewall credit rejection records contain invalid cap, usage or hard_cap fields; enforcement remains unverified.");
+  }
   if (!rejectionFound) {
-    throw new Error("No AWF per-run credit rejection at cap 1 was recorded. The workload may have ended below the cap or runtime evidence may be missing; enforcement remains unverified.");
+    const diagnostics = [`Inspected ${observedRecords} records in ${observedFiles} candidate firewall files.`];
+    if (observedFiles === 0) diagnostics.push("Runtime evidence may be missing: no candidate firewall event/audit file was collected.");
+    else diagnostics.push("Collected firewall evidence contains no qualifying structured rejection; check AWF rejection-event emission and artifact collection, not just workload duration.");
+    if (mismatchedRejections) diagnostics.push(`${mismatchedRejections} credit rejection records had invalid cap, usage or hard_cap fields.`);
+    const readDiagnostic = relativePath => {
+      try {
+        return readJSONL(directory, relativePath);
+      } catch {
+        diagnostics.push(`Invalid diagnostic JSONL in ${relativePath}; diagnostic data is not rejection evidence.`);
+        return null;
+      }
+    };
+    for (const relativePath of TRACKER_PATHS) {
+      const tracker = readDiagnostic(relativePath);
+      if (tracker !== null) diagnostics.push(`Collected ${tracker.length} token-tracker diagnostic records; tracker data is not rejection evidence.`);
+    }
+    const usage = readDiagnostic(USAGE_PATH);
+    const totals = (usage || []).filter(entry => entry?.event === "token_usage" && typeof entry.ai_credits_total === "number" && Number.isFinite(entry.ai_credits_total) && entry.ai_credits_total >= 0);
+    if (totals.length) {
+      const total = totals[totals.length - 1].ai_credits_total;
+      diagnostics.push(`Last AWF token_usage total: ${total} (${total >= 1 ? "at/above" : "below"} cap 1); usage is not rejection evidence.`);
+    }
+    const stdioPath = path.join(directory, "agent-stdio.log");
+    if (info.engine_id === "codex" && fs.existsSync(stdioPath)) {
+      for (const line of fs.readFileSync(stdioPath, "utf8").split("\n")) {
+        if (!line.startsWith("{")) continue;
+        let entry;
+        try {
+          entry = JSON.parse(line);
+        } catch {
+          continue; // Mixed engine output is diagnostic-only, never passing evidence.
+        }
+        if (entry?.type !== "error" || typeof entry.message !== "string") continue;
+        const match = /^unexpected status 403 Forbidden: Maximum AI credits exceeded \((\d+(?:\.\d+)?) \/ (\d+(?:\.\d+)?)\)\./.exec(entry.message);
+        if (match && Number.isFinite(Number(match[1])) && Number.isFinite(Number(match[2]))) {
+          diagnostics.push(`Codex terminal API error reports credit rejection (${Number(match[1])} / ${Number(match[2])}); agent text cannot substitute for a structured AWF rejection event.`);
+          break;
+        }
+      }
+    }
+    throw new Error(`No AWF per-run credit rejection at cap 1 was recorded. ${diagnostics.join(" ")} Per-run enforcement remains unverified.`);
   }
   return "Passed: AWF recorded a per-run AI credit rejection at cap 1.";
 }
@@ -62,4 +147,4 @@ if (require.main === module) {
   }
 }
 
-module.exports = { LOG_PATHS, verifyCreditLimit };
+module.exports = { LOG_PATHS, TRACKER_PATHS, USAGE_PATH, verifyCreditLimit };