From f6f7d3d22d239549bef8b851e9c42002800b63f1 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Fri, 28 Aug 2026 12:59:33 +0300 Subject: [PATCH] fix(warehouse-e2e): stop the secret rule refusing every credential question The first real run of the warehouse e2e tier failed the ask contract on all three legs: 2 questions on stripe-node, 1 on multi-source-next, 3 on monorepo-env. Every one was a refusal, not a missing rule. The data-warehouse-source-setup skill tells the agent to collect a warehouse credential as ordinary, non-sensitive text, because `external-data-sources-create` rejects a `{ secretRef }`. A `secret` askAnswers rule answers only a *sensitive* text question. So the `stripe` and `password` rules refused every credential the skill asked for, the run answered with the `e2e` sentinel, and no source could ever be created. The LLM trace for the run shows the agent quoting that skill line as its reason for leaving `sensitive` off. Drop `secret` from both rules. The values are fake, unroutable placeholders, and the workbench's "no secret leakage" check is what keeps them out of the report, the result payload, the MCP journal and the captured frames. Also close the routing gaps the same trace exposed: - `account[ _-]?id` ahead of `stripe`, so "Stripe Account ID (optional)" takes an account id instead of the API key. - `connection[ _-]?string` for the optional Postgres and Supabase field. - `author` on the user rule, for the Hugging Face namespace field. - a trailing credential net, so a token question the vendor rules miss gets a placeholder instead of the sentinel. Tests pin the precedence the order relies on, and replay the real question ids the run asked: api_key, account_id, table_prefix, stripe_secret_key, and both the bare and `pg-` prefixed Postgres fields. The net needs `E2E_API_TOKEN` from the workbench runner. An unset var leaves the rule inert, so this degrades to today's behaviour on an older runner. --- .../e2e-flow-snapshot.test.ts.snap | 16 ++- e2e-harness/__tests__/e2e-profile-ask.test.ts | 134 ++++++++++++++---- .../programs/warehouse-source/test/e2e.json | 29 +++- 3 files changed, 145 insertions(+), 34 deletions(-) diff --git a/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap b/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap index 450740722..aff3100a2 100644 --- a/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap +++ b/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap @@ -178,13 +178,20 @@ exports[`e2e flow snapshot — warehouse-source > walks intro → auth → run "profile": { "ask": "first", "askAnswers": [ + { + "match": "account[ _-]?id", + "value": "acct_e2efixture", + }, { "match": "prefix", "value": "\${E2E_SOURCE_PREFIX}", }, + { + "match": "connection[ _-]?string", + "value": "postgresql://\${E2E_PG_USER}:\${E2E_PG_PASSWORD}@\${E2E_PG_HOST}:\${E2E_PG_PORT}/\${E2E_PG_DATABASE}", + }, { "match": "stripe", - "secret": true, "value": "\${E2E_STRIPE_API_KEY}", }, { @@ -201,17 +208,20 @@ exports[`e2e flow snapshot — warehouse-source > walks intro → auth → run }, { "match": "password|passwd|pwd", - "secret": true, "value": "\${E2E_PG_PASSWORD}", }, { - "match": "user|username|role", + "match": "user|username|role|author", "value": "\${E2E_PG_USER}", }, { "match": "schema", "value": "public", }, + { + "match": "api[ _-]?key|api[ _-]?token|access[ _-]?token|auth[ _-]?token|^token$|secret|credential", + "value": "\${E2E_API_TOKEN}", + }, ], "healthCheck": "dismiss", "mcp": "skip", diff --git a/e2e-harness/__tests__/e2e-profile-ask.test.ts b/e2e-harness/__tests__/e2e-profile-ask.test.ts index f722cede1..5bc8fdf97 100644 --- a/e2e-harness/__tests__/e2e-profile-ask.test.ts +++ b/e2e-harness/__tests__/e2e-profile-ask.test.ts @@ -524,6 +524,18 @@ describe('E2E_DRIVABLE_SCREENS', () => { }); }); +/** The env the workbench runner injects for a warehouse e2e leg. */ +const WAREHOUSE_ENV = { + E2E_SOURCE_PREFIX: 'e2e_7_', + E2E_STRIPE_API_KEY: 'sk_test_7', + E2E_PG_HOST: 'db.internal', + E2E_PG_PORT: '5432', + E2E_PG_DATABASE: 'appdb', + E2E_PG_USER: 'app', + E2E_PG_PASSWORD: 'hunter2', + E2E_API_TOKEN: 'hf_e2e_token', +}; + describe('warehouse-source profile', () => { const warehouse = profileFor(Program.WarehouseSource); @@ -538,64 +550,136 @@ describe('warehouse-source profile', () => { }); it('routes each credential question to its own env var', () => { - const env = { - E2E_SOURCE_PREFIX: 'e2e_7_', - E2E_STRIPE_API_KEY: 'sk_test_7', - E2E_PG_HOST: 'db.internal', - E2E_PG_PORT: '5432', - E2E_PG_DATABASE: 'appdb', - E2E_PG_USER: 'app', - E2E_PG_PASSWORD: 'hunter2', - }; - const resolved = resolveE2eProfile(warehouse, { env }); + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); const batch = answerQuestions( [ text('prefix', 'Table prefix'), - sensitiveText('stripe_api_key', 'Stripe API key'), + text('stripe_secret_key', 'Stripe API key'), text('host', 'Postgres host'), text('port', 'Port'), text('database', 'Database name'), text('user', 'Username'), - sensitiveText('password', 'Password'), + text('password', 'Password'), + text('schema', 'Schema'), ], resolved, ); expect(batch.answers).toEqual({ prefix: 'e2e_7_', - stripe_api_key: 'sk_test_7', + stripe_secret_key: 'sk_test_7', host: 'db.internal', port: '5432', database: 'appdb', user: 'app', password: 'hunter2', + schema: 'public', }); expect(batch.sentinelIds).toEqual([]); expect(batch.refusedIds).toEqual([]); }); - it('marks its credential-bearing rules secret', () => { - const secretMatches = (warehouse.askAnswers ?? []) - .filter((r) => r.secret) - .map((r) => r.match); - expect(secretMatches).toEqual(['stripe', 'password|passwd|pwd']); + /** + * The skill tells the agent to collect a warehouse credential as ordinary + * text, because `external-data-sources-create` rejects a `{ secretRef }`. A + * `secret` rule answers only a *sensitive* text question, so marking these + * rules secret refused every credential the skill asked for — 2 refusals on + * the Stripe leg, 3 on the Postgres+Stripe leg, and no create anywhere. + */ + it('marks no rule secret, so a plain credential question is answered', () => { + expect((warehouse.askAnswers ?? []).filter((r) => r.secret)).toEqual([]); + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); + const batch = answerQuestions([text('password', 'Password')], resolved); + expect(batch.answers.password).toBe('hunter2'); + expect(batch.refusedIds).toEqual([]); }); - it('withholds the API key from a question that is not sensitive', () => { - const resolved = resolveE2eProfile(warehouse, { - env: { E2E_STRIPE_API_KEY: 'sk_test_7' }, + /** + * The exact batch the 2026-08-28 CI run asked on the Stripe leg. Two of these + * three questions fell back to the sentinel then; none may now. + */ + it("answers the Stripe leg's real question ids", () => { + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); + const batch = answerQuestions( + [ + text('api_key', 'Paste your restricted Stripe API key here:'), + text('account_id', 'Stripe Account ID (optional — leave blank)'), + text('table_prefix', 'Table name prefix (optional)'), + ], + resolved, + ); + expect(batch.answers).toEqual({ + // The credential net claims a vendor-less `api_key` on the id pass, + // before the `stripe` rule can claim it on the prompt pass. Both values + // are fake placeholders, so the routing costs nothing and the question + // never reaches the sentinel. + api_key: 'hf_e2e_token', + account_id: 'acct_e2efixture', + table_prefix: 'e2e_7_', }); + expect(batch.sentinelIds).toEqual([]); + expect(batch.refusedIds).toEqual([]); + }); + + /** The exact batch the same run asked on the Postgres legs. */ + it("answers the Postgres leg's real question ids, prefixed or not", () => { + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); const batch = answerQuestions( - [text('stripe_api_key', 'Stripe API key')], + [ + text('pg-host', 'PostgreSQL host'), + text('pg-port', 'PostgreSQL port'), + text('pg-database', 'PostgreSQL database name'), + text('pg-user', 'PostgreSQL user'), + text('pg-password', 'PostgreSQL password'), + text('pg-schema', 'PostgreSQL schema'), + ], resolved, ); - expect(batch.answers.stripe_api_key).toBe(E2E_ANSWER_SENTINEL); - expect(batch.refusedIds).toEqual(['stripe_api_key']); + expect(batch.answers).toEqual({ + 'pg-host': 'db.internal', + 'pg-port': '5432', + 'pg-database': 'appdb', + 'pg-user': 'app', + 'pg-password': 'hunter2', + 'pg-schema': 'public', + }); + expect(batch.sentinelIds).toEqual([]); + }); + + /** + * Precedence the rule order is built on: `account[ _-]?id` sits ahead of + * `stripe`, so an optional account-id question cannot take the API key. + */ + it('gives the Stripe account id an account id, never the API key', () => { + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); + const batch = answerQuestions( + [text('stripe_account_id', 'Stripe account id (optional)')], + resolved, + ); + expect(batch.answers.stripe_account_id).toBe('acct_e2efixture'); + }); + + it('covers the optional fields that used to match no rule at all', () => { + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); + const batch = answerQuestions( + [ + text('connection_string', 'Connection string (optional)'), + text('api_token', 'Hugging Face access token'), + text('author', 'Username or organization'), + ], + resolved, + ); + expect(batch.answers).toEqual({ + connection_string: 'postgresql://app:hunter2@db.internal:5432/appdb', + api_token: 'hf_e2e_token', + author: 'app', + }); + expect(batch.sentinelIds).toEqual([]); }); it('sentinels every credential when the env is empty', () => { const resolved = resolveE2eProfile(warehouse, { env: {} }); const batch = answerQuestions( - [text('host', 'Postgres host'), sensitiveText('password', 'Password')], + [text('host', 'Postgres host'), text('password', 'Password')], resolved, ); expect(batch.sentinelIds).toEqual(['host', 'password']); diff --git a/src/lib/programs/warehouse-source/test/e2e.json b/src/lib/programs/warehouse-source/test/e2e.json index 3ad20592a..a51d36a3b 100644 --- a/src/lib/programs/warehouse-source/test/e2e.json +++ b/src/lib/programs/warehouse-source/test/e2e.json @@ -1,6 +1,6 @@ { "program": "warehouse-source", - "summary": "Happy path: confirm the intro over the detected sources, keep any task notice, answer the agent's credential questions from E2E_* env vars, delete installed skills. Rules carrying a real credential are marked \"secret\": they answer only a sensitive text question, so a skill cannot name a plain question \"password\" and get the value back unvaulted.", + "summary": "Happy path: confirm the intro over the detected sources, keep any task notice, answer the agent's credential questions from E2E_* env vars, delete installed skills. No rule is marked \"secret\". The data-warehouse-source-setup skill tells the agent to collect a warehouse credential as ordinary (non-sensitive) text, because external-data-sources-create rejects a { secretRef }. A secret rule answers only a sensitive text question, so a secret rule refused every credential question the skill asked, and no source could ever be created. Each value here is a fake, unroutable placeholder, and the workbench's \"no secret leakage\" check is what holds those placeholders out of the report, the result payload, the MCP journal and the captured frames.", "profile": { "setup": "first", "healthCheck": "dismiss", @@ -10,14 +10,21 @@ "ask": "first", "notice": "keep", "askAnswers": [ + { + "match": "account[ _-]?id", + "value": "acct_e2efixture" + }, { "match": "prefix", "value": "${E2E_SOURCE_PREFIX}" }, + { + "match": "connection[ _-]?string", + "value": "postgresql://${E2E_PG_USER}:${E2E_PG_PASSWORD}@${E2E_PG_HOST}:${E2E_PG_PORT}/${E2E_PG_DATABASE}" + }, { "match": "stripe", - "value": "${E2E_STRIPE_API_KEY}", - "secret": true + "value": "${E2E_STRIPE_API_KEY}" }, { "match": "host|hostname|server|endpoint", @@ -33,19 +40,29 @@ }, { "match": "password|passwd|pwd", - "value": "${E2E_PG_PASSWORD}", - "secret": true + "value": "${E2E_PG_PASSWORD}" }, { - "match": "user|username|role", + "match": "user|username|role|author", "value": "${E2E_PG_USER}" }, { "match": "schema", "value": "public" + }, + { + "match": "api[ _-]?key|api[ _-]?token|access[ _-]?token|auth[ _-]?token|^token$|secret|credential", + "value": "${E2E_API_TOKEN}" } ] }, + "askAnswerNotes": [ + "Rules are first-match-wins. Every rule is tested against the question id before any rule is tested against the prompt, so a rule that claims an id shadows a later rule that would have claimed the prompt.", + "The agent writes its own question ids. The 2026-08-28 CI run asked: api_key, account_id, table_prefix and stripe_secret_key for Stripe; host, port, database, user, password, schema and pg-host, pg-port, pg-database, pg-user, pg-password, pg-schema for Postgres.", + "account[ _-]?id sits ahead of stripe so \"Stripe Account ID (optional)\" takes an account id instead of the API key.", + "The last rule is a net, not a router. It claims a vendor-less id such as api_key on the id pass, ahead of the stripe rule that would claim the same question on the prompt pass. That is deliberate: a placeholder token and a placeholder Stripe key are equally fake, and a question that reaches the sentinel breaks the create it feeds.", + "E2E_API_TOKEN comes from the workbench runner. An unset var makes the rule inert, and the question falls through to the sentinel." + ], "variations": [ { "name": "default",