From ece51b8a21ac0bf90d2b37cc7611b2870bfeda0e Mon Sep 17 00:00:00 2001 From: pelikhan Date: Wed, 14 Oct 2026 08:16:58 -0700 Subject: [PATCH 1/2] Clarify missing per-run credit rejection evidence Bind credit verification to the current repository, run and attempt. Distinguish absent structured AWF rejection events from collected usage and diagnostic-only Codex errors while preserving fail-closed proof requirements. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/check-max-ai-credits.cjs | 97 +++++++++++++-- .github/scripts/check-max-ai-credits.test.cjs | 113 +++++++++++++++++- .../max-ai-credits-rejection-38556022048.json | 22 ++++ .../daily-max-ai-credits-test.lock.yml | 2 +- .../workflows/daily-max-ai-credits-test.md | 6 +- 5 files changed, 222 insertions(+), 18 deletions(-) create mode 100644 .github/scripts/fixtures/max-ai-credits-rejection-38556022048.json diff --git a/.github/scripts/check-max-ai-credits.cjs b/.github/scripts/check-max-ai-credits.cjs index ea450644612..57efe4477d3 100644 --- a/.github/scripts/check-max-ai-credits.cjs +++ b/.github/scripts/check-max-ai-credits.cjs @@ -10,11 +10,47 @@ const LOG_PATHS = [ "sandbox/firewall/audit/api-proxy-logs/events.jsonl", "sandbox/firewall-audit-logs/api-proxy-logs/event-logs.jsonl", "sandbox/firewall-audit-logs/api-proxy-logs/events.jsonl", + "sandbox/firewall/logs/api-proxy-logs/token-tracker-audit.jsonl", + "sandbox/firewall/audit/api-proxy-logs/token-tracker-audit.jsonl", "sandbox/firewall/logs/audit.jsonl", "sandbox/firewall/audit/log.jsonl", ]; +const USAGE_PATH = "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl"; -function verifyCreditLimit(directory) { +function readJSON(directory, name) { + const filename = path.join(directory, name); + if (!fs.existsSync(filename)) throw new Error(`Missing ${name}; enforcement remains unverified.`); + try { + return JSON.parse(fs.readFileSync(filename, "utf8")); + } catch { + throw new Error(`Invalid JSON in ${name}; enforcement remains unverified.`); + } +} + +function readJSONL(directory, name) { + const filename = path.join(directory, name); + if (!fs.existsSync(filename)) return null; + return fs + .readFileSync(filename, "utf8") + .split("\n") + .flatMap((line, index) => { + if (!line.trim()) return []; + try { + return [JSON.parse(line)]; + } catch { + throw new Error(`Invalid firewall JSONL in ${name} at line ${index + 1}.`); + } + }); +} + +function verifyCreditLimit(directory, expected = { runId: process.env.GITHUB_RUN_ID, runAttempt: process.env.GITHUB_RUN_ATTEMPT, repository: process.env.GITHUB_REPOSITORY }) { + if (!/^[1-9]\d*$/.test(expected.runId || "") || !/^[1-9]\d*$/.test(expected.runAttempt || "") || !expected.repository) { + throw new Error("Expected repository, run ID and run attempt are required; enforcement remains unverified."); + } + const info = readJSON(directory, "aw_info.json"); + if (String(info?.run_id) !== String(expected.runId) || String(info?.run_attempt) !== String(expected.runAttempt) || info?.repository !== expected.repository) { + throw new Error("Agent artifact repository/run/attempt does not match this verification job; enforcement remains unverified."); + } const configText = fs.readFileSync(path.join(directory, "awf-config.json"), "utf8"); let config; try { @@ -27,25 +63,60 @@ function verifyCreditLimit(directory) { } let rejectionFound = false; + let observedFiles = 0; + let observedRecords = 0; + let mismatchedRejections = 0; for (const relativePath of LOG_PATHS) { - const filename = path.join(directory, relativePath); - if (!fs.existsSync(filename)) continue; - const lines = fs.readFileSync(filename, "utf8").split("\n"); - for (let index = 0; index < lines.length; index++) { - if (!lines[index].trim()) continue; - let entry; - try { - entry = JSON.parse(lines[index]); - } catch { - throw new Error(`Invalid firewall JSONL in ${relativePath} at line ${index + 1}.`); + const entries = readJSONL(directory, relativePath); + if (entries === null) continue; + observedFiles++; + observedRecords += entries.length; + for (const entry of entries) { + if (entry?.event !== "ai_credits_limit_exceeded") continue; + if ( + (entry.run_id !== undefined && String(entry.run_id) !== String(expected.runId)) || + (entry.run_attempt !== undefined && String(entry.run_attempt) !== String(expected.runAttempt)) || + (entry.repository !== undefined && entry.repository !== expected.repository) + ) { + throw new Error("Firewall rejection repository/run/attempt does not match this verification job; enforcement remains unverified."); } if (entry?.event === "ai_credits_limit_exceeded" && entry.max_ai_credits === 1 && typeof entry.total_ai_credits === "number" && Number.isFinite(entry.total_ai_credits) && entry.total_ai_credits >= 1 && entry.hard_cap === false) { rejectionFound = true; + } else { + mismatchedRejections++; } } } + const usage = readJSONL(directory, USAGE_PATH); if (!rejectionFound) { - throw new Error("No AWF per-run credit rejection at cap 1 was recorded. The workload may have ended below the cap or runtime evidence may be missing; enforcement remains unverified."); + const diagnostics = [`Inspected ${observedRecords} records in ${observedFiles} candidate firewall files.`]; + if (observedFiles === 0) diagnostics.push("Runtime evidence may be missing: no candidate firewall event/audit/tracker file was collected."); + else diagnostics.push("Collected firewall evidence contains no qualifying structured rejection; check AWF rejection-event emission and artifact collection, not just workload duration."); + if (mismatchedRejections) diagnostics.push(`${mismatchedRejections} credit rejection records had invalid cap, usage or hard_cap fields.`); + const totals = (usage || []).filter(entry => entry?.event === "token_usage" && typeof entry.ai_credits_total === "number" && Number.isFinite(entry.ai_credits_total) && entry.ai_credits_total >= 0); + if (totals.length) { + const total = totals[totals.length - 1].ai_credits_total; + diagnostics.push(`Last AWF token_usage total: ${total} (${total >= 1 ? "at/above" : "below"} cap 1); usage is not rejection evidence.`); + } + const stdioPath = path.join(directory, "agent-stdio.log"); + if (info.engine_id === "codex" && fs.existsSync(stdioPath)) { + for (const line of fs.readFileSync(stdioPath, "utf8").split("\n")) { + if (!line.startsWith("{")) continue; + let entry; + try { + entry = JSON.parse(line); + } catch { + continue; // Mixed engine output is diagnostic-only, never passing evidence. + } + if (entry?.type !== "error" || typeof entry.message !== "string") continue; + const match = /^unexpected status 403 Forbidden: Maximum AI credits exceeded \((\d+(?:\.\d+)?) \/ (\d+(?:\.\d+)?)\)\./.exec(entry.message); + if (match && Number.isFinite(Number(match[1])) && Number.isFinite(Number(match[2]))) { + diagnostics.push(`Codex terminal API error reports credit rejection (${Number(match[1])} / ${Number(match[2])}); agent text cannot substitute for a structured AWF rejection event.`); + break; + } + } + } + throw new Error(`No AWF per-run credit rejection at cap 1 was recorded. ${diagnostics.join(" ")} Per-run enforcement remains unverified.`); } return "Passed: AWF recorded a per-run AI credit rejection at cap 1."; } @@ -62,4 +133,4 @@ if (require.main === module) { } } -module.exports = { LOG_PATHS, verifyCreditLimit }; +module.exports = { LOG_PATHS, USAGE_PATH, verifyCreditLimit }; diff --git a/.github/scripts/check-max-ai-credits.test.cjs b/.github/scripts/check-max-ai-credits.test.cjs index 0c4b6f1a2c1..b240e75feac 100644 --- a/.github/scripts/check-max-ai-credits.test.cjs +++ b/.github/scripts/check-max-ai-credits.test.cjs @@ -6,8 +6,12 @@ const os = require("node:os"); const path = require("node:path"); const { spawnSync } = require("node:child_process"); const { test } = require("node:test"); -const { LOG_PATHS, verifyCreditLimit } = require("./check-max-ai-credits.cjs"); +const { LOG_PATHS, USAGE_PATH, verifyCreditLimit: verify } = require("./check-max-ai-credits.cjs"); +const expected = { runId: "38556022048", runAttempt: "1", repository: "github/gh-aw" }; +const verifyCreditLimit = directory => verify(directory, expected); +const captured = require("./fixtures/max-ai-credits-rejection-38556022048.json"); +// Contract fixture, not a captured producer event. const rejection = { event: "ai_credits_limit_exceeded", total_ai_credits: 1.02, max_ai_credits: 1, hard_cap: false }; test("workflow bounds one-session tool continuation and gates its conclusion on runtime evidence", () => { @@ -39,6 +43,7 @@ function fixture(t) { fs.writeFileSync(filename, content); }; write("awf-config.json", JSON.stringify({ apiProxy: { maxAiCredits: 1 } })); + write("aw_info.json", JSON.stringify(captured.info)); return { directory, write }; } @@ -91,7 +96,7 @@ test("accepts a rejection at exactly the cap", t => { test("missing logs fail verification", t => { const { directory } = fixture(t); - assert.throws(() => verifyCreditLimit(directory), /runtime evidence may be missing/); + assert.throws(() => verifyCreditLimit(directory), /Runtime evidence may be missing/); }); test("a different compiled cap fails even with a matching event", t => { @@ -122,7 +127,11 @@ test("malformed configuration fails without echoing its contents", t => { test("CLI exits nonzero for unexpected completion and writes a summary only on pass", t => { const { directory, write } = fixture(t); const summary = path.join(directory, "summary.md"); - const run = () => spawnSync(process.execPath, [path.join(__dirname, "check-max-ai-credits.cjs"), directory], { encoding: "utf8", env: { ...process.env, GITHUB_STEP_SUMMARY: summary } }); + const run = () => + spawnSync(process.execPath, [path.join(__dirname, "check-max-ai-credits.cjs"), directory], { + encoding: "utf8", + env: { ...process.env, GITHUB_STEP_SUMMARY: summary, GITHUB_RUN_ID: expected.runId, GITHUB_RUN_ATTEMPT: expected.runAttempt, GITHUB_REPOSITORY: expected.repository }, + }); const failed = run(); assert.equal(failed.status, 1); assert.match(failed.stderr, /enforcement remains unverified/); @@ -132,3 +141,101 @@ test("CLI exits nonzero for unexpected completion and writes a summary only on p assert.equal(passed.status, 0); assert.match(fs.readFileSync(summary, "utf8"), /Passed: AWF recorded/); }); + +test("captured genuine Codex rejection remains unverified without a structured AWF rejection", t => { + const { directory, write } = fixture(t); + write("awf-config.json", JSON.stringify(captured.config)); + write( + LOG_PATHS.find(name => name.includes("token-tracker-audit")), + captured.tracker.map(entry => JSON.stringify(entry)).join("\n") + ); + write(USAGE_PATH, JSON.stringify(captured.usage)); + write("agent-stdio.log", JSON.stringify(captured.terminalError)); + assert.throws( + () => verifyCreditLimit(directory), + error => + /Inspected 2 records in 1 candidate firewall files/.test(error.message) && + /check AWF rejection-event emission and artifact collection/.test(error.message) && + /Last AWF token_usage total: 1.021966 \(at\/above cap 1\)/.test(error.message) && + /Codex terminal API error reports credit rejection \(1.021966 \/ 1\)/.test(error.message) && + /enforcement remains unverified/.test(error.message) + ); +}); + +for (const info of [{ ...captured.info, run_id: 38242557753 }, { ...captured.info, run_attempt: "2" }, { ...captured.info, repository: "another/repository" }, {}, null]) { + test(`rejects wrong or missing artifact identity ${JSON.stringify(info)}`, t => { + const { directory, write } = fixture(t); + write("aw_info.json", JSON.stringify(info)); + write(LOG_PATHS[0], JSON.stringify(rejection)); + assert.throws(() => verifyCreditLimit(directory), /artifact repository\/run\/attempt does not match/); + }); +} + +for (const identity of [{ run_id: 38242557753 }, { run_attempt: 2 }, { repository: "another/repository" }]) { + test(`rejects explicit wrong-run rejection even after qualifying evidence ${JSON.stringify(identity)}`, t => { + const { directory, write } = fixture(t); + write(LOG_PATHS[0], [rejection, { ...rejection, ...identity }].map(entry => JSON.stringify(entry)).join("\n")); + assert.throws(() => verifyCreditLimit(directory), /Firewall rejection repository\/run\/attempt does not match/); + }); +} + +test("verification requires an independently supplied expected run identity", t => { + const { directory, write } = fixture(t); + write(LOG_PATHS[0], JSON.stringify(rejection)); + assert.throws(() => verify(directory, {}), /Expected repository, run ID and run attempt are required/); + fs.rmSync(path.join(directory, "aw_info.json")); + assert.throws(() => verifyCreditLimit(directory), /Missing aw_info.json/); + write("aw_info.json", "PRIVATE_PAYLOAD"); + assert.throws( + () => verifyCreditLimit(directory), + error => /Invalid JSON in aw_info.json/.test(error.message) && !error.message.includes("PRIVATE_PAYLOAD") + ); +}); + +test("malformed tracker or usage evidence fails even after a valid rejection", t => { + const { directory, write } = fixture(t); + write(LOG_PATHS[0], JSON.stringify(rejection)); + const trackerPath = LOG_PATHS.find(name => name.includes("token-tracker-audit")); + write(trackerPath, "PRIVATE_PAYLOAD"); + assert.throws(() => verifyCreditLimit(directory), /Invalid firewall JSONL.*token-tracker-audit/); + write(trackerPath, ""); + write(USAGE_PATH, "PRIVATE_PAYLOAD"); + assert.throws(() => verifyCreditLimit(directory), /Invalid firewall JSONL.*token-usage/); +}); + +for (const entry of [ + { type: "error", message: "unexpected status 403 Forbidden: Access denied." }, + { type: "item.completed", item: { type: "command_execution", aggregated_output: JSON.stringify(captured.terminalError) } }, + { type: "error", message: `Quoted previous-run log: ${captured.terminalError.message}` }, + { type: "report_incomplete", message: captured.terminalError.message }, + { type: "error", message: "unexpected status 403 Forbidden: Maximum AI credits exceeded (1.02 / 2)." }, +]) { + test(`never passes on non-authoritative agent evidence ${entry.type}: ${entry.message || "tool output"}`, t => { + const { directory, write } = fixture(t); + write("agent-stdio.log", JSON.stringify(entry)); + assert.throws(() => verifyCreditLimit(directory), /enforcement remains unverified/); + if (entry !== undefined && entry.message !== "unexpected status 403 Forbidden: Maximum AI credits exceeded (1.02 / 2).") { + assert.throws( + () => verifyCreditLimit(directory), + error => !error.message.includes("Codex terminal API error reports") + ); + } + }); +} + +test("diagnostics omit untrusted payloads and do not mistake below-cap usage for missing usage", t => { + const { directory, write } = fixture(t); + write(LOG_PATHS[0], JSON.stringify({ event: "request_complete", message: "PRIVATE_PAYLOAD" })); + write(USAGE_PATH, JSON.stringify({ event: "token_usage", ai_credits_total: 0.39811 })); + write("agent-stdio.log", JSON.stringify({ ...captured.terminalError, message: `${captured.terminalError.message} PRIVATE_PAYLOAD` })); + assert.throws( + () => verifyCreditLimit(directory), + error => /below cap 1/.test(error.message) && !error.message.includes("PRIVATE_PAYLOAD") + ); +}); + +test("invalid structured rejection fields are diagnosed without becoming passing evidence", t => { + const { directory, write } = fixture(t); + write(LOG_PATHS[0], JSON.stringify({ ...rejection, max_ai_credits: 2 })); + assert.throws(() => verifyCreditLimit(directory), /1 credit rejection records had invalid cap, usage or hard_cap fields/); +}); diff --git a/.github/scripts/fixtures/max-ai-credits-rejection-38556022048.json b/.github/scripts/fixtures/max-ai-credits-rejection-38556022048.json new file mode 100644 index 00000000000..2b89b998d53 --- /dev/null +++ b/.github/scripts/fixtures/max-ai-credits-rejection-38556022048.json @@ -0,0 +1,22 @@ +{ + "provenance": "Normalized diagnostic fields from github/gh-aw Actions run 38556022048, attempt 1, SHA 462ed682857c4ab64d40f2c85e3738fa27834839. URLs, request IDs, prompts and payloads omitted. No structured AWF rejection event was collected.", + "info": { + "engine_id": "codex", + "run_id": 38556022048, + "run_attempt": "1", + "repository": "github/gh-aw" + }, + "config": { + "apiProxy": { "maxAiCredits": 1 } + }, + "usage": { + "event": "token_usage", + "status": 200, + "ai_credits_total": 1.021966 + }, + "tracker": [{ "event": "TRACK_START" }, { "event": "TRACK_END" }], + "terminalError": { + "type": "error", + "message": "unexpected status 403 Forbidden: Maximum AI credits exceeded (1.021966 / 1)." + } +} diff --git a/.github/workflows/daily-max-ai-credits-test.lock.yml b/.github/workflows/daily-max-ai-credits-test.lock.yml index 487a3f5afb1..719b1f29557 100644 --- a/.github/workflows/daily-max-ai-credits-test.lock.yml +++ b/.github/workflows/daily-max-ai-credits-test.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"bc616eaa42dd55efaa49cbdf34c330b6fef327b3e2ca445bb28f86854a7c4cc7","body_hash":"a8fc52677ad3998753c7f8a47c369fef0f837e45fe9c3fdb4f5830ba52315249","strict":true,"agent_id":"codex","agent_model":"openai/gpt-5.6-luna","engine_versions":{"codex":"0.162.1"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"bc616eaa42dd55efaa49cbdf34c330b6fef327b3e2ca445bb28f86854a7c4cc7","body_hash":"835137a3ce8441073e5d47d45a94a877e5d9989af6510ccb715de6646f87753a","strict":true,"agent_id":"codex","agent_model":"openai/gpt-5.6-luna","engine_versions":{"codex":"0.162.1"}} # gh-aw-manifest: {"version":1,"secrets":["CODEX_API_KEY","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.57","digest":"sha256:a9861b2fad1d29c586bbbf98840a61056e30e2720d8d9ec2c0def9049566bd8a","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.57@sha256:a9861b2fad1d29c586bbbf98840a61056e30e2720d8d9ec2c0def9049566bd8a"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.57","digest":"sha256:f2925ff32dc972a5d8b90a7785b6ca68052efd2575fe896556ac457e22e86dcc","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.57@sha256:f2925ff32dc972a5d8b90a7785b6ca68052efd2575fe896556ac457e22e86dcc"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.57","digest":"sha256:be808c16b535ebabd070eebe086fe6bffd5bb86b2e5fe8e5cdad30068eb9c0ac","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.57@sha256:be808c16b535ebabd070eebe086fe6bffd5bb86b2e5fe8e5cdad30068eb9c0ac"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.31","digest":"sha256:dbec0f2ddef6adf83b7764019323096e3e5cc0f3886255f6320555b7fe2a7caf","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.31@sha256:dbec0f2ddef6adf83b7764019323096e3e5cc0f3886255f6320555b7fe2a7caf"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/daily-max-ai-credits-test.md b/.github/workflows/daily-max-ai-credits-test.md index 31d438adec5..63af598b238 100644 --- a/.github/workflows/daily-max-ai-credits-test.md +++ b/.github/workflows/daily-max-ai-credits-test.md @@ -70,7 +70,11 @@ jobs: > passes only when the agent artifact contains the firewall's > `ai_credits_limit_exceeded` event for the configured cap of 1 AI credit. > Assistant text, a successful exit, `noop`, and credit totals alone are not -> evidence that the firewall rejected a request. +> evidence that the firewall rejected a request. The verifier binds the artifact +> to this repository, run ID and run attempt. If structured rejection evidence +> is missing, it reports the collected firewall record count, last AWF usage +> total and any top-level Codex terminal credit rejection separately. These +> diagnostics never turn an unverified outcome into a pass. #### What This Tests From 0d79259ebe99944e631787865795c65a05d4f716 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Wed, 14 Oct 2026 09:04:35 -0700 Subject: [PATCH 2/2] Fix credit verifier artifact wiring and optional diagnostics Download the separate current-run identity artifact and keep tracker and usage sidecars off the positive proof path. Cover source/lock packaging, malformed optional diagnostics and strict authoritative rejection evidence. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...ck-max-ai-credits-artifact-wiring.test.cjs | 104 ++++++++++++++++++ .github/scripts/check-max-ai-credits.cjs | 26 ++++- .github/scripts/check-max-ai-credits.test.cjs | 83 +++++++++++--- .../daily-max-ai-credits-test.lock.yml | 7 +- .../workflows/daily-max-ai-credits-test.md | 8 +- 5 files changed, 207 insertions(+), 21 deletions(-) create mode 100644 .github/scripts/check-max-ai-credits-artifact-wiring.test.cjs diff --git a/.github/scripts/check-max-ai-credits-artifact-wiring.test.cjs b/.github/scripts/check-max-ai-credits-artifact-wiring.test.cjs new file mode 100644 index 00000000000..dd84a85d571 --- /dev/null +++ b/.github/scripts/check-max-ai-credits-artifact-wiring.test.cjs @@ -0,0 +1,104 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +const { test } = require("node:test"); +const { parseDocument } = require("../../actions/setup/js/work_queue_yaml.cjs"); + +const workflowBase = path.join(__dirname, "../workflows/daily-max-ai-credits-test"); +const evidenceDirectory = "/tmp/gh-aw/"; + +function readWorkflow(extension) { + let text = fs.readFileSync(`${workflowBase}.${extension}`, "utf8"); + if (extension === "md") { + const frontmatter = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(text); + assert.ok(frontmatter, "workflow Markdown must have YAML frontmatter"); + text = frontmatter[1]; + } + const document = parseDocument(text); + assert.deepEqual(document.errors, [], "workflow YAML must parse without errors"); + return document.toJS(); +} + +function assertVerifierDownloads(workflow) { + const job = workflow.jobs.verify_credit_limit; + assert.equal(job.needs, "agent"); + assert.equal(job.if, "always() && !cancelled() && needs.agent.result != 'skipped'"); + assert.equal(job.permissions.actions, "read"); + const steps = job.steps; + const verifierIndex = steps.findIndex(step => step.run === `node .github/scripts/check-max-ai-credits.cjs ${evidenceDirectory}`); + assert.ok(verifierIndex >= 0, "verifier must read the artifact extraction directory"); + for (const name of ["agent", "info"]) { + const downloads = steps.filter(step => step.uses?.startsWith("actions/download-artifact@") && step.with?.name === name); + assert.equal(downloads.length, 1, `expected exactly one ${name} download`); + const download = downloads[0]; + assert.deepEqual(download.with, { name, path: evidenceDirectory }, `${name} must use current-run, same-repository download defaults`); + assert.ok(steps.indexOf(download) < verifierIndex, `${name} must be downloaded before verification`); + assert.equal(download.if, undefined, `${name} download must not be skipped`); + assert.equal(download["continue-on-error"], undefined, `${name} download failure must fail the job`); + } +} + +test("authored workflow downloads agent evidence and current-run identity before verification", () => { + assertVerifierDownloads(readWorkflow("md")); +}); + +test("compiled workflow preserves the current-run artifact downloads", () => { + const workflow = readWorkflow("lock.yml"); + assertVerifierDownloads(workflow); + const sourceSteps = readWorkflow("md").jobs.verify_credit_limit.steps; + for (const name of ["agent", "info"]) { + const sourceDownload = sourceSteps.find(step => step.with?.name === name); + const compiledDownload = workflow.jobs.verify_credit_limit.steps.find(step => step.with?.name === name); + assert.equal(sourceDownload.uses, "actions/download-artifact@v8.0.2"); + assert.match(compiledDownload.uses, /^actions\/download-artifact@[a-f0-9]{40}$/); + assert.deepEqual(compiledDownload.with, sourceDownload.with); + } +}); + +test("info producer packages aw_info.json at the extraction root consumed by the verifier", () => { + const workflow = readWorkflow("lock.yml"); + const producer = workflow.jobs.activation; + const uploads = producer.steps.filter(step => step.uses?.startsWith("actions/upload-artifact@") && step.with?.name === "info"); + assert.equal(uploads.length, 1); + const upload = uploads[0]; + assert.equal(upload.if, "success() || failure()"); + assert.equal(upload.with.path, "/tmp/gh-aw/aw_info.json"); + assert.equal(workflow.jobs.agent.needs, "activation", "identity producer must precede the agent"); + + const download = workflow.jobs.verify_credit_limit.steps.find(step => step.with?.name === "info"); + assert.ok(download, "verifier must download the separate info artifact"); + // A single-file upload is archived relative to its parent directory. + const archiveEntry = path.posix.relative(path.posix.dirname(upload.with.path), upload.with.path); + assert.equal(archiveEntry, "aw_info.json"); + assert.equal(path.posix.join(download.with.path, archiveEntry), "/tmp/gh-aw/aw_info.json"); + + const agentUpload = workflow.jobs.agent.steps.find(step => step.uses?.startsWith("actions/upload-artifact@") && step.with?.name === "agent"); + assert.ok(agentUpload); + const agentPaths = agentUpload.with.path.trim().split(/\s+/); + assert.ok(!agentPaths.includes(upload.with.path), "run identity is a separate artifact, not part of agent evidence"); +}); + +test("artifact contract rejects missing, foreign-run, misplaced and late identity downloads", () => { + for (const mutate of [ + steps => + steps.splice( + steps.findIndex(step => step.with?.name === "info"), + 1 + ), + steps => (steps.find(step => step.with?.name === "info").with["run-id"] = "another-run"), + steps => (steps.find(step => step.with?.name === "info").with.path = "/tmp/gh-aw/info/"), + steps => + steps.push( + steps.splice( + steps.findIndex(step => step.with?.name === "info"), + 1 + )[0] + ), + ]) { + const workflow = readWorkflow("md"); + mutate(workflow.jobs.verify_credit_limit.steps); + assert.throws(() => assertVerifierDownloads(workflow), assert.AssertionError); + } +}); diff --git a/.github/scripts/check-max-ai-credits.cjs b/.github/scripts/check-max-ai-credits.cjs index 57efe4477d3..71a73e9d9ef 100644 --- a/.github/scripts/check-max-ai-credits.cjs +++ b/.github/scripts/check-max-ai-credits.cjs @@ -10,11 +10,10 @@ const LOG_PATHS = [ "sandbox/firewall/audit/api-proxy-logs/events.jsonl", "sandbox/firewall-audit-logs/api-proxy-logs/event-logs.jsonl", "sandbox/firewall-audit-logs/api-proxy-logs/events.jsonl", - "sandbox/firewall/logs/api-proxy-logs/token-tracker-audit.jsonl", - "sandbox/firewall/audit/api-proxy-logs/token-tracker-audit.jsonl", "sandbox/firewall/logs/audit.jsonl", "sandbox/firewall/audit/log.jsonl", ]; +const TRACKER_PATHS = ["sandbox/firewall/logs/api-proxy-logs/token-tracker-audit.jsonl", "sandbox/firewall/audit/api-proxy-logs/token-tracker-audit.jsonl"]; const USAGE_PATH = "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl"; function readJSON(directory, name) { @@ -80,19 +79,34 @@ function verifyCreditLimit(directory, expected = { runId: process.env.GITHUB_RUN ) { throw new Error("Firewall rejection repository/run/attempt does not match this verification job; enforcement remains unverified."); } - if (entry?.event === "ai_credits_limit_exceeded" && entry.max_ai_credits === 1 && typeof entry.total_ai_credits === "number" && Number.isFinite(entry.total_ai_credits) && entry.total_ai_credits >= 1 && entry.hard_cap === false) { + if (entry.max_ai_credits === 1 && typeof entry.total_ai_credits === "number" && Number.isFinite(entry.total_ai_credits) && entry.total_ai_credits >= 1 && entry.hard_cap === false) { rejectionFound = true; } else { mismatchedRejections++; } } } - const usage = readJSONL(directory, USAGE_PATH); + if (rejectionFound && mismatchedRejections) { + throw new Error("Firewall credit rejection records contain invalid cap, usage or hard_cap fields; enforcement remains unverified."); + } if (!rejectionFound) { const diagnostics = [`Inspected ${observedRecords} records in ${observedFiles} candidate firewall files.`]; - if (observedFiles === 0) diagnostics.push("Runtime evidence may be missing: no candidate firewall event/audit/tracker file was collected."); + if (observedFiles === 0) diagnostics.push("Runtime evidence may be missing: no candidate firewall event/audit file was collected."); else diagnostics.push("Collected firewall evidence contains no qualifying structured rejection; check AWF rejection-event emission and artifact collection, not just workload duration."); if (mismatchedRejections) diagnostics.push(`${mismatchedRejections} credit rejection records had invalid cap, usage or hard_cap fields.`); + const readDiagnostic = relativePath => { + try { + return readJSONL(directory, relativePath); + } catch { + diagnostics.push(`Invalid diagnostic JSONL in ${relativePath}; diagnostic data is not rejection evidence.`); + return null; + } + }; + for (const relativePath of TRACKER_PATHS) { + const tracker = readDiagnostic(relativePath); + if (tracker !== null) diagnostics.push(`Collected ${tracker.length} token-tracker diagnostic records; tracker data is not rejection evidence.`); + } + const usage = readDiagnostic(USAGE_PATH); const totals = (usage || []).filter(entry => entry?.event === "token_usage" && typeof entry.ai_credits_total === "number" && Number.isFinite(entry.ai_credits_total) && entry.ai_credits_total >= 0); if (totals.length) { const total = totals[totals.length - 1].ai_credits_total; @@ -133,4 +147,4 @@ if (require.main === module) { } } -module.exports = { LOG_PATHS, USAGE_PATH, verifyCreditLimit }; +module.exports = { LOG_PATHS, TRACKER_PATHS, USAGE_PATH, verifyCreditLimit }; diff --git a/.github/scripts/check-max-ai-credits.test.cjs b/.github/scripts/check-max-ai-credits.test.cjs index b240e75feac..e0647528a22 100644 --- a/.github/scripts/check-max-ai-credits.test.cjs +++ b/.github/scripts/check-max-ai-credits.test.cjs @@ -6,7 +6,7 @@ const os = require("node:os"); const path = require("node:path"); const { spawnSync } = require("node:child_process"); const { test } = require("node:test"); -const { LOG_PATHS, USAGE_PATH, verifyCreditLimit: verify } = require("./check-max-ai-credits.cjs"); +const { LOG_PATHS, TRACKER_PATHS, USAGE_PATH, verifyCreditLimit: verify } = require("./check-max-ai-credits.cjs"); const expected = { runId: "38556022048", runAttempt: "1", repository: "github/gh-aw" }; const verifyCreditLimit = directory => verify(directory, expected); const captured = require("./fixtures/max-ai-credits-rejection-38556022048.json"); @@ -132,9 +132,12 @@ test("CLI exits nonzero for unexpected completion and writes a summary only on p encoding: "utf8", env: { ...process.env, GITHUB_STEP_SUMMARY: summary, GITHUB_RUN_ID: expected.runId, GITHUB_RUN_ATTEMPT: expected.runAttempt, GITHUB_REPOSITORY: expected.repository }, }); + for (const diagnosticPath of [...TRACKER_PATHS, USAGE_PATH]) write(diagnosticPath, "PRIVATE_PAYLOAD"); const failed = run(); assert.equal(failed.status, 1); assert.match(failed.stderr, /enforcement remains unverified/); + assert.doesNotMatch(failed.stderr, /PRIVATE_PAYLOAD/); + assert.ok(failed.stderr.length < 2000); assert.equal(fs.existsSync(summary), false); write(LOG_PATHS[0], JSON.stringify(rejection)); const passed = run(); @@ -145,17 +148,14 @@ test("CLI exits nonzero for unexpected completion and writes a summary only on p test("captured genuine Codex rejection remains unverified without a structured AWF rejection", t => { const { directory, write } = fixture(t); write("awf-config.json", JSON.stringify(captured.config)); - write( - LOG_PATHS.find(name => name.includes("token-tracker-audit")), - captured.tracker.map(entry => JSON.stringify(entry)).join("\n") - ); + write(TRACKER_PATHS[0], captured.tracker.map(entry => JSON.stringify(entry)).join("\n")); write(USAGE_PATH, JSON.stringify(captured.usage)); write("agent-stdio.log", JSON.stringify(captured.terminalError)); assert.throws( () => verifyCreditLimit(directory), error => - /Inspected 2 records in 1 candidate firewall files/.test(error.message) && - /check AWF rejection-event emission and artifact collection/.test(error.message) && + /Inspected 0 records in 0 candidate firewall files/.test(error.message) && + /Collected 2 token-tracker diagnostic records/.test(error.message) && /Last AWF token_usage total: 1.021966 \(at\/above cap 1\)/.test(error.message) && /Codex terminal API error reports credit rejection \(1.021966 \/ 1\)/.test(error.message) && /enforcement remains unverified/.test(error.message) @@ -192,15 +192,72 @@ test("verification requires an independently supplied expected run identity", t ); }); -test("malformed tracker or usage evidence fails even after a valid rejection", t => { +test("event-local run labels do not replace required generated artifact provenance", t => { + const { directory, write } = fixture(t); + write(LOG_PATHS[0], JSON.stringify({ ...rejection, run_id: expected.runId, run_attempt: expected.runAttempt, repository: expected.repository })); + fs.rmSync(path.join(directory, "aw_info.json")); + assert.throws(() => verifyCreditLimit(directory), /Missing aw_info.json/); +}); + +test("malformed optional tracker and usage diagnostics cannot override a valid rejection", t => { const { directory, write } = fixture(t); write(LOG_PATHS[0], JSON.stringify(rejection)); - const trackerPath = LOG_PATHS.find(name => name.includes("token-tracker-audit")); - write(trackerPath, "PRIVATE_PAYLOAD"); - assert.throws(() => verifyCreditLimit(directory), /Invalid firewall JSONL.*token-tracker-audit/); - write(trackerPath, ""); + for (const trackerPath of TRACKER_PATHS) write(trackerPath, "PRIVATE_PAYLOAD"); + assert.match(verifyCreditLimit(directory), /^Passed:/); write(USAGE_PATH, "PRIVATE_PAYLOAD"); - assert.throws(() => verifyCreditLimit(directory), /Invalid firewall JSONL.*token-usage/); + assert.match(verifyCreditLimit(directory), /^Passed:/); +}); + +test("missing proof stays unverified with bounded, private-text-free malformed diagnostics", t => { + const { directory, write } = fixture(t); + for (const diagnosticPath of [...TRACKER_PATHS, USAGE_PATH]) write(diagnosticPath, "PRIVATE_PAYLOAD".repeat(10000)); + assert.throws( + () => verifyCreditLimit(directory), + error => + /No AWF per-run credit rejection/.test(error.message) && + /Invalid diagnostic JSONL/.test(error.message) && + /enforcement remains unverified/.test(error.message) && + !error.message.includes("PRIVATE_PAYLOAD") && + error.message.length < 2000 + ); +}); + +for (const trackerPath of TRACKER_PATHS) { + test(`diagnostic-only tracker records cannot prove rejection in ${trackerPath}`, t => { + const { directory, write } = fixture(t); + write(trackerPath, [...captured.tracker, rejection].map(entry => JSON.stringify(entry)).join("\n")); + assert.throws(() => verifyCreditLimit(directory), /No AWF per-run credit rejection/); + }); +} + +for (const invalid of [ + { ...rejection, max_ai_credits: 2 }, + { ...rejection, total_ai_credits: "1.02" }, + { ...rejection, hard_cap: true }, +]) { + test(`invalid authoritative rejection still fails after valid proof ${JSON.stringify(invalid)}`, t => { + const { directory, write } = fixture(t); + write(LOG_PATHS[0], JSON.stringify(rejection)); + write(LOG_PATHS[1], JSON.stringify(invalid)); + assert.throws(() => verifyCreditLimit(directory), /invalid cap, usage or hard_cap fields/); + }); +} + +test("malformed later authoritative proof file still fails after a valid rejection", t => { + const { directory, write } = fixture(t); + write(LOG_PATHS[0], JSON.stringify(rejection)); + write(LOG_PATHS[1], "PRIVATE_PAYLOAD"); + assert.throws( + () => verifyCreditLimit(directory), + error => /Invalid firewall JSONL/.test(error.message) && !error.message.includes("PRIVATE_PAYLOAD") + ); +}); + +test("a later authoritative proof file with a conflicting identity still fails", t => { + const { directory, write } = fixture(t); + write(LOG_PATHS[0], JSON.stringify(rejection)); + write(LOG_PATHS[1], JSON.stringify({ ...rejection, run_id: "38242557753" })); + assert.throws(() => verifyCreditLimit(directory), /Firewall rejection repository\/run\/attempt does not match/); }); for (const entry of [ diff --git a/.github/workflows/daily-max-ai-credits-test.lock.yml b/.github/workflows/daily-max-ai-credits-test.lock.yml index 719b1f29557..9194dbf2a19 100644 --- a/.github/workflows/daily-max-ai-credits-test.lock.yml +++ b/.github/workflows/daily-max-ai-credits-test.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"bc616eaa42dd55efaa49cbdf34c330b6fef327b3e2ca445bb28f86854a7c4cc7","body_hash":"835137a3ce8441073e5d47d45a94a877e5d9989af6510ccb715de6646f87753a","strict":true,"agent_id":"codex","agent_model":"openai/gpt-5.6-luna","engine_versions":{"codex":"0.162.1"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"2198527d2aa13d1bcf70b17b4ca2602f1ce4d14b84bfe3d3e549992fcc2c0327","body_hash":"1c544d9bb7e7edcc8fc7908a9fcfe9a8570543a9c0e24af310866baeaa91e0a1","strict":true,"agent_id":"codex","agent_model":"openai/gpt-5.6-luna","engine_versions":{"codex":"0.162.1"}} # gh-aw-manifest: {"version":1,"secrets":["CODEX_API_KEY","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.57","digest":"sha256:a9861b2fad1d29c586bbbf98840a61056e30e2720d8d9ec2c0def9049566bd8a","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.57@sha256:a9861b2fad1d29c586bbbf98840a61056e30e2720d8d9ec2c0def9049566bd8a"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.57","digest":"sha256:f2925ff32dc972a5d8b90a7785b6ca68052efd2575fe896556ac457e22e86dcc","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.57@sha256:f2925ff32dc972a5d8b90a7785b6ca68052efd2575fe896556ac457e22e86dcc"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.57","digest":"sha256:be808c16b535ebabd070eebe086fe6bffd5bb86b2e5fe8e5cdad30068eb9c0ac","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.57@sha256:be808c16b535ebabd070eebe086fe6bffd5bb86b2e5fe8e5cdad30068eb9c0ac"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.31","digest":"sha256:dbec0f2ddef6adf83b7764019323096e3e5cc0f3886255f6320555b7fe2a7caf","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.31@sha256:dbec0f2ddef6adf83b7764019323096e3e5cc0f3886255f6320555b7fe2a7caf"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1923,5 +1923,10 @@ jobs: with: name: agent path: /tmp/gh-aw/ + - name: Download run identity + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: info + path: /tmp/gh-aw/ - name: Verify per-run credit rejection run: node .github/scripts/check-max-ai-credits.cjs /tmp/gh-aw/ diff --git a/.github/workflows/daily-max-ai-credits-test.md b/.github/workflows/daily-max-ai-credits-test.md index 63af598b238..80bd5ce2e54 100644 --- a/.github/workflows/daily-max-ai-credits-test.md +++ b/.github/workflows/daily-max-ai-credits-test.md @@ -58,6 +58,11 @@ jobs: with: name: agent path: /tmp/gh-aw/ + - name: Download run identity + uses: actions/download-artifact@v8.0.2 + with: + name: info + path: /tmp/gh-aw/ - name: Verify per-run credit rejection run: node .github/scripts/check-max-ai-credits.cjs /tmp/gh-aw/ --- @@ -71,7 +76,8 @@ jobs: > `ai_credits_limit_exceeded` event for the configured cap of 1 AI credit. > Assistant text, a successful exit, `noop`, and credit totals alone are not > evidence that the firewall rejected a request. The verifier binds the artifact -> to this repository, run ID and run attempt. If structured rejection evidence +> to this repository, run ID and run attempt using `aw_info.json` from the +> same run's separate `info` artifact. If structured rejection evidence > is missing, it reports the collected firewall record count, last AWF usage > total and any top-level Codex terminal credit rejection separately. These > diagnostics never turn an unverified outcome into a pass.