diff --git a/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap b/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap index 450740722..aff3100a2 100644 --- a/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap +++ b/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap @@ -178,13 +178,20 @@ exports[`e2e flow snapshot — warehouse-source > walks intro → auth → run "profile": { "ask": "first", "askAnswers": [ + { + "match": "account[ _-]?id", + "value": "acct_e2efixture", + }, { "match": "prefix", "value": "\${E2E_SOURCE_PREFIX}", }, + { + "match": "connection[ _-]?string", + "value": "postgresql://\${E2E_PG_USER}:\${E2E_PG_PASSWORD}@\${E2E_PG_HOST}:\${E2E_PG_PORT}/\${E2E_PG_DATABASE}", + }, { "match": "stripe", - "secret": true, "value": "\${E2E_STRIPE_API_KEY}", }, { @@ -201,17 +208,20 @@ exports[`e2e flow snapshot — warehouse-source > walks intro → auth → run }, { "match": "password|passwd|pwd", - "secret": true, "value": "\${E2E_PG_PASSWORD}", }, { - "match": "user|username|role", + "match": "user|username|role|author", "value": "\${E2E_PG_USER}", }, { "match": "schema", "value": "public", }, + { + "match": "api[ _-]?key|api[ _-]?token|access[ _-]?token|auth[ _-]?token|^token$|secret|credential", + "value": "\${E2E_API_TOKEN}", + }, ], "healthCheck": "dismiss", "mcp": "skip", diff --git a/e2e-harness/__tests__/e2e-profile-ask.test.ts b/e2e-harness/__tests__/e2e-profile-ask.test.ts index f722cede1..5bc8fdf97 100644 --- a/e2e-harness/__tests__/e2e-profile-ask.test.ts +++ b/e2e-harness/__tests__/e2e-profile-ask.test.ts @@ -524,6 +524,18 @@ describe('E2E_DRIVABLE_SCREENS', () => { }); }); +/** The env the workbench runner injects for a warehouse e2e leg. */ +const WAREHOUSE_ENV = { + E2E_SOURCE_PREFIX: 'e2e_7_', + E2E_STRIPE_API_KEY: 'sk_test_7', + E2E_PG_HOST: 'db.internal', + E2E_PG_PORT: '5432', + E2E_PG_DATABASE: 'appdb', + E2E_PG_USER: 'app', + E2E_PG_PASSWORD: 'hunter2', + E2E_API_TOKEN: 'hf_e2e_token', +}; + describe('warehouse-source profile', () => { const warehouse = profileFor(Program.WarehouseSource); @@ -538,64 +550,136 @@ describe('warehouse-source profile', () => { }); it('routes each credential question to its own env var', () => { - const env = { - E2E_SOURCE_PREFIX: 'e2e_7_', - E2E_STRIPE_API_KEY: 'sk_test_7', - E2E_PG_HOST: 'db.internal', - E2E_PG_PORT: '5432', - E2E_PG_DATABASE: 'appdb', - E2E_PG_USER: 'app', - E2E_PG_PASSWORD: 'hunter2', - }; - const resolved = resolveE2eProfile(warehouse, { env }); + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); const batch = answerQuestions( [ text('prefix', 'Table prefix'), - sensitiveText('stripe_api_key', 'Stripe API key'), + text('stripe_secret_key', 'Stripe API key'), text('host', 'Postgres host'), text('port', 'Port'), text('database', 'Database name'), text('user', 'Username'), - sensitiveText('password', 'Password'), + text('password', 'Password'), + text('schema', 'Schema'), ], resolved, ); expect(batch.answers).toEqual({ prefix: 'e2e_7_', - stripe_api_key: 'sk_test_7', + stripe_secret_key: 'sk_test_7', host: 'db.internal', port: '5432', database: 'appdb', user: 'app', password: 'hunter2', + schema: 'public', }); expect(batch.sentinelIds).toEqual([]); expect(batch.refusedIds).toEqual([]); }); - it('marks its credential-bearing rules secret', () => { - const secretMatches = (warehouse.askAnswers ?? []) - .filter((r) => r.secret) - .map((r) => r.match); - expect(secretMatches).toEqual(['stripe', 'password|passwd|pwd']); + /** + * The skill tells the agent to collect a warehouse credential as ordinary + * text, because `external-data-sources-create` rejects a `{ secretRef }`. A + * `secret` rule answers only a *sensitive* text question, so marking these + * rules secret refused every credential the skill asked for — 2 refusals on + * the Stripe leg, 3 on the Postgres+Stripe leg, and no create anywhere. + */ + it('marks no rule secret, so a plain credential question is answered', () => { + expect((warehouse.askAnswers ?? []).filter((r) => r.secret)).toEqual([]); + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); + const batch = answerQuestions([text('password', 'Password')], resolved); + expect(batch.answers.password).toBe('hunter2'); + expect(batch.refusedIds).toEqual([]); }); - it('withholds the API key from a question that is not sensitive', () => { - const resolved = resolveE2eProfile(warehouse, { - env: { E2E_STRIPE_API_KEY: 'sk_test_7' }, + /** + * The exact batch the 2026-08-28 CI run asked on the Stripe leg. Two of these + * three questions fell back to the sentinel then; none may now. + */ + it("answers the Stripe leg's real question ids", () => { + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); + const batch = answerQuestions( + [ + text('api_key', 'Paste your restricted Stripe API key here:'), + text('account_id', 'Stripe Account ID (optional — leave blank)'), + text('table_prefix', 'Table name prefix (optional)'), + ], + resolved, + ); + expect(batch.answers).toEqual({ + // The credential net claims a vendor-less `api_key` on the id pass, + // before the `stripe` rule can claim it on the prompt pass. Both values + // are fake placeholders, so the routing costs nothing and the question + // never reaches the sentinel. + api_key: 'hf_e2e_token', + account_id: 'acct_e2efixture', + table_prefix: 'e2e_7_', }); + expect(batch.sentinelIds).toEqual([]); + expect(batch.refusedIds).toEqual([]); + }); + + /** The exact batch the same run asked on the Postgres legs. */ + it("answers the Postgres leg's real question ids, prefixed or not", () => { + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); const batch = answerQuestions( - [text('stripe_api_key', 'Stripe API key')], + [ + text('pg-host', 'PostgreSQL host'), + text('pg-port', 'PostgreSQL port'), + text('pg-database', 'PostgreSQL database name'), + text('pg-user', 'PostgreSQL user'), + text('pg-password', 'PostgreSQL password'), + text('pg-schema', 'PostgreSQL schema'), + ], resolved, ); - expect(batch.answers.stripe_api_key).toBe(E2E_ANSWER_SENTINEL); - expect(batch.refusedIds).toEqual(['stripe_api_key']); + expect(batch.answers).toEqual({ + 'pg-host': 'db.internal', + 'pg-port': '5432', + 'pg-database': 'appdb', + 'pg-user': 'app', + 'pg-password': 'hunter2', + 'pg-schema': 'public', + }); + expect(batch.sentinelIds).toEqual([]); + }); + + /** + * Precedence the rule order is built on: `account[ _-]?id` sits ahead of + * `stripe`, so an optional account-id question cannot take the API key. + */ + it('gives the Stripe account id an account id, never the API key', () => { + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); + const batch = answerQuestions( + [text('stripe_account_id', 'Stripe account id (optional)')], + resolved, + ); + expect(batch.answers.stripe_account_id).toBe('acct_e2efixture'); + }); + + it('covers the optional fields that used to match no rule at all', () => { + const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV }); + const batch = answerQuestions( + [ + text('connection_string', 'Connection string (optional)'), + text('api_token', 'Hugging Face access token'), + text('author', 'Username or organization'), + ], + resolved, + ); + expect(batch.answers).toEqual({ + connection_string: 'postgresql://app:hunter2@db.internal:5432/appdb', + api_token: 'hf_e2e_token', + author: 'app', + }); + expect(batch.sentinelIds).toEqual([]); }); it('sentinels every credential when the env is empty', () => { const resolved = resolveE2eProfile(warehouse, { env: {} }); const batch = answerQuestions( - [text('host', 'Postgres host'), sensitiveText('password', 'Password')], + [text('host', 'Postgres host'), text('password', 'Password')], resolved, ); expect(batch.sentinelIds).toEqual(['host', 'password']); diff --git a/src/lib/programs/warehouse-source/test/e2e.json b/src/lib/programs/warehouse-source/test/e2e.json index 3ad20592a..a51d36a3b 100644 --- a/src/lib/programs/warehouse-source/test/e2e.json +++ b/src/lib/programs/warehouse-source/test/e2e.json @@ -1,6 +1,6 @@ { "program": "warehouse-source", - "summary": "Happy path: confirm the intro over the detected sources, keep any task notice, answer the agent's credential questions from E2E_* env vars, delete installed skills. Rules carrying a real credential are marked \"secret\": they answer only a sensitive text question, so a skill cannot name a plain question \"password\" and get the value back unvaulted.", + "summary": "Happy path: confirm the intro over the detected sources, keep any task notice, answer the agent's credential questions from E2E_* env vars, delete installed skills. No rule is marked \"secret\". The data-warehouse-source-setup skill tells the agent to collect a warehouse credential as ordinary (non-sensitive) text, because external-data-sources-create rejects a { secretRef }. A secret rule answers only a sensitive text question, so a secret rule refused every credential question the skill asked, and no source could ever be created. Each value here is a fake, unroutable placeholder, and the workbench's \"no secret leakage\" check is what holds those placeholders out of the report, the result payload, the MCP journal and the captured frames.", "profile": { "setup": "first", "healthCheck": "dismiss", @@ -10,14 +10,21 @@ "ask": "first", "notice": "keep", "askAnswers": [ + { + "match": "account[ _-]?id", + "value": "acct_e2efixture" + }, { "match": "prefix", "value": "${E2E_SOURCE_PREFIX}" }, + { + "match": "connection[ _-]?string", + "value": "postgresql://${E2E_PG_USER}:${E2E_PG_PASSWORD}@${E2E_PG_HOST}:${E2E_PG_PORT}/${E2E_PG_DATABASE}" + }, { "match": "stripe", - "value": "${E2E_STRIPE_API_KEY}", - "secret": true + "value": "${E2E_STRIPE_API_KEY}" }, { "match": "host|hostname|server|endpoint", @@ -33,19 +40,29 @@ }, { "match": "password|passwd|pwd", - "value": "${E2E_PG_PASSWORD}", - "secret": true + "value": "${E2E_PG_PASSWORD}" }, { - "match": "user|username|role", + "match": "user|username|role|author", "value": "${E2E_PG_USER}" }, { "match": "schema", "value": "public" + }, + { + "match": "api[ _-]?key|api[ _-]?token|access[ _-]?token|auth[ _-]?token|^token$|secret|credential", + "value": "${E2E_API_TOKEN}" } ] }, + "askAnswerNotes": [ + "Rules are first-match-wins. Every rule is tested against the question id before any rule is tested against the prompt, so a rule that claims an id shadows a later rule that would have claimed the prompt.", + "The agent writes its own question ids. The 2026-08-28 CI run asked: api_key, account_id, table_prefix and stripe_secret_key for Stripe; host, port, database, user, password, schema and pg-host, pg-port, pg-database, pg-user, pg-password, pg-schema for Postgres.", + "account[ _-]?id sits ahead of stripe so \"Stripe Account ID (optional)\" takes an account id instead of the API key.", + "The last rule is a net, not a router. It claims a vendor-less id such as api_key on the id pass, ahead of the stripe rule that would claim the same question on the prompt pass. That is deliberate: a placeholder token and a placeholder Stripe key are equally fake, and a question that reaches the sentinel breaks the create it feeds.", + "E2E_API_TOKEN comes from the workbench runner. An unset var makes the rule inert, and the question falls through to the sentinel." + ], "variations": [ { "name": "default",