Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -178,13 +178,20 @@ exports[`e2e flow snapshot — warehouse-source > walks intro → auth → run
"profile": {
"ask": "first",
"askAnswers": [
{
"match": "account[ _-]?id",
"value": "acct_e2efixture",
},
{
"match": "prefix",
"value": "\${E2E_SOURCE_PREFIX}",
},
{
"match": "connection[ _-]?string",
"value": "postgresql://\${E2E_PG_USER}:\${E2E_PG_PASSWORD}@\${E2E_PG_HOST}:\${E2E_PG_PORT}/\${E2E_PG_DATABASE}",
},
{
"match": "stripe",
"secret": true,
"value": "\${E2E_STRIPE_API_KEY}",
},
{
Expand All @@ -201,17 +208,20 @@ exports[`e2e flow snapshot — warehouse-source > walks intro → auth → run
},
{
"match": "password|passwd|pwd",
"secret": true,
"value": "\${E2E_PG_PASSWORD}",
},
{
"match": "user|username|role",
"match": "user|username|role|author",
"value": "\${E2E_PG_USER}",
},
{
"match": "schema",
"value": "public",
},
{
"match": "api[ _-]?key|api[ _-]?token|access[ _-]?token|auth[ _-]?token|^token$|secret|credential",
"value": "\${E2E_API_TOKEN}",
},
],
"healthCheck": "dismiss",
"mcp": "skip",
Expand Down
134 changes: 109 additions & 25 deletions e2e-harness/__tests__/e2e-profile-ask.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,18 @@ describe('E2E_DRIVABLE_SCREENS', () => {
});
});

/** The env the workbench runner injects for a warehouse e2e leg. */
const WAREHOUSE_ENV = {
E2E_SOURCE_PREFIX: 'e2e_7_',
E2E_STRIPE_API_KEY: 'sk_test_7',
E2E_PG_HOST: 'db.internal',
E2E_PG_PORT: '5432',
E2E_PG_DATABASE: 'appdb',
E2E_PG_USER: 'app',
E2E_PG_PASSWORD: 'hunter2',
E2E_API_TOKEN: 'hf_e2e_token',
};

describe('warehouse-source profile', () => {
const warehouse = profileFor(Program.WarehouseSource);

Expand All @@ -538,64 +550,136 @@ describe('warehouse-source profile', () => {
});

it('routes each credential question to its own env var', () => {
const env = {
E2E_SOURCE_PREFIX: 'e2e_7_',
E2E_STRIPE_API_KEY: 'sk_test_7',
E2E_PG_HOST: 'db.internal',
E2E_PG_PORT: '5432',
E2E_PG_DATABASE: 'appdb',
E2E_PG_USER: 'app',
E2E_PG_PASSWORD: 'hunter2',
};
const resolved = resolveE2eProfile(warehouse, { env });
const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV });
const batch = answerQuestions(
[
text('prefix', 'Table prefix'),
sensitiveText('stripe_api_key', 'Stripe API key'),
text('stripe_secret_key', 'Stripe API key'),
text('host', 'Postgres host'),
text('port', 'Port'),
text('database', 'Database name'),
text('user', 'Username'),
sensitiveText('password', 'Password'),
text('password', 'Password'),
text('schema', 'Schema'),
],
resolved,
);
expect(batch.answers).toEqual({
prefix: 'e2e_7_',
stripe_api_key: 'sk_test_7',
stripe_secret_key: 'sk_test_7',
host: 'db.internal',
port: '5432',
database: 'appdb',
user: 'app',
password: 'hunter2',
schema: 'public',
});
expect(batch.sentinelIds).toEqual([]);
expect(batch.refusedIds).toEqual([]);
});

it('marks its credential-bearing rules secret', () => {
const secretMatches = (warehouse.askAnswers ?? [])
.filter((r) => r.secret)
.map((r) => r.match);
expect(secretMatches).toEqual(['stripe', 'password|passwd|pwd']);
/**
* The skill tells the agent to collect a warehouse credential as ordinary
* text, because `external-data-sources-create` rejects a `{ secretRef }`. A
* `secret` rule answers only a *sensitive* text question, so marking these
* rules secret refused every credential the skill asked for — 2 refusals on
* the Stripe leg, 3 on the Postgres+Stripe leg, and no create anywhere.
*/
it('marks no rule secret, so a plain credential question is answered', () => {
expect((warehouse.askAnswers ?? []).filter((r) => r.secret)).toEqual([]);
const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV });
const batch = answerQuestions([text('password', 'Password')], resolved);
expect(batch.answers.password).toBe('hunter2');
expect(batch.refusedIds).toEqual([]);
});

it('withholds the API key from a question that is not sensitive', () => {
const resolved = resolveE2eProfile(warehouse, {
env: { E2E_STRIPE_API_KEY: 'sk_test_7' },
/**
* The exact batch the 2026-08-28 CI run asked on the Stripe leg. Two of these
* three questions fell back to the sentinel then; none may now.
*/
it("answers the Stripe leg's real question ids", () => {
const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV });
const batch = answerQuestions(
[
text('api_key', 'Paste your restricted Stripe API key here:'),
text('account_id', 'Stripe Account ID (optional — leave blank)'),
text('table_prefix', 'Table name prefix (optional)'),
],
resolved,
);
expect(batch.answers).toEqual({
// The credential net claims a vendor-less `api_key` on the id pass,
// before the `stripe` rule can claim it on the prompt pass. Both values
// are fake placeholders, so the routing costs nothing and the question
// never reaches the sentinel.
api_key: 'hf_e2e_token',
account_id: 'acct_e2efixture',
table_prefix: 'e2e_7_',
});
expect(batch.sentinelIds).toEqual([]);
expect(batch.refusedIds).toEqual([]);
});

/** The exact batch the same run asked on the Postgres legs. */
it("answers the Postgres leg's real question ids, prefixed or not", () => {
const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV });
const batch = answerQuestions(
[text('stripe_api_key', 'Stripe API key')],
[
text('pg-host', 'PostgreSQL host'),
text('pg-port', 'PostgreSQL port'),
text('pg-database', 'PostgreSQL database name'),
text('pg-user', 'PostgreSQL user'),
text('pg-password', 'PostgreSQL password'),
text('pg-schema', 'PostgreSQL schema'),
],
resolved,
);
expect(batch.answers.stripe_api_key).toBe(E2E_ANSWER_SENTINEL);
expect(batch.refusedIds).toEqual(['stripe_api_key']);
expect(batch.answers).toEqual({
'pg-host': 'db.internal',
'pg-port': '5432',
'pg-database': 'appdb',
'pg-user': 'app',
'pg-password': 'hunter2',
'pg-schema': 'public',
});
expect(batch.sentinelIds).toEqual([]);
});

/**
* Precedence the rule order is built on: `account[ _-]?id` sits ahead of
* `stripe`, so an optional account-id question cannot take the API key.
*/
it('gives the Stripe account id an account id, never the API key', () => {
const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV });
const batch = answerQuestions(
[text('stripe_account_id', 'Stripe account id (optional)')],
resolved,
);
expect(batch.answers.stripe_account_id).toBe('acct_e2efixture');
});

it('covers the optional fields that used to match no rule at all', () => {
const resolved = resolveE2eProfile(warehouse, { env: WAREHOUSE_ENV });
const batch = answerQuestions(
[
text('connection_string', 'Connection string (optional)'),
text('api_token', 'Hugging Face access token'),
text('author', 'Username or organization'),
],
resolved,
);
expect(batch.answers).toEqual({
connection_string: 'postgresql://app:hunter2@db.internal:5432/appdb',
api_token: 'hf_e2e_token',
author: 'app',
});
expect(batch.sentinelIds).toEqual([]);
});

it('sentinels every credential when the env is empty', () => {
const resolved = resolveE2eProfile(warehouse, { env: {} });
const batch = answerQuestions(
[text('host', 'Postgres host'), sensitiveText('password', 'Password')],
[text('host', 'Postgres host'), text('password', 'Password')],
resolved,
);
expect(batch.sentinelIds).toEqual(['host', 'password']);
Expand Down
29 changes: 23 additions & 6 deletions src/lib/programs/warehouse-source/test/e2e.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"program": "warehouse-source",
"summary": "Happy path: confirm the intro over the detected sources, keep any task notice, answer the agent's credential questions from E2E_* env vars, delete installed skills. Rules carrying a real credential are marked \"secret\": they answer only a sensitive text question, so a skill cannot name a plain question \"password\" and get the value back unvaulted.",
"summary": "Happy path: confirm the intro over the detected sources, keep any task notice, answer the agent's credential questions from E2E_* env vars, delete installed skills. No rule is marked \"secret\". The data-warehouse-source-setup skill tells the agent to collect a warehouse credential as ordinary (non-sensitive) text, because external-data-sources-create rejects a { secretRef }. A secret rule answers only a sensitive text question, so a secret rule refused every credential question the skill asked, and no source could ever be created. Each value here is a fake, unroutable placeholder, and the workbench's \"no secret leakage\" check is what holds those placeholders out of the report, the result payload, the MCP journal and the captured frames.",
"profile": {
"setup": "first",
"healthCheck": "dismiss",
Expand All @@ -10,14 +10,21 @@
"ask": "first",
"notice": "keep",
"askAnswers": [
{
"match": "account[ _-]?id",
"value": "acct_e2efixture"
},
{
"match": "prefix",
"value": "${E2E_SOURCE_PREFIX}"
},
{
"match": "connection[ _-]?string",
"value": "postgresql://${E2E_PG_USER}:${E2E_PG_PASSWORD}@${E2E_PG_HOST}:${E2E_PG_PORT}/${E2E_PG_DATABASE}"
},
{
"match": "stripe",
"value": "${E2E_STRIPE_API_KEY}",
"secret": true
"value": "${E2E_STRIPE_API_KEY}"
},
{
"match": "host|hostname|server|endpoint",
Expand All @@ -33,19 +40,29 @@
},
{
"match": "password|passwd|pwd",
"value": "${E2E_PG_PASSWORD}",
"secret": true
"value": "${E2E_PG_PASSWORD}"
},
{
"match": "user|username|role",
"match": "user|username|role|author",
"value": "${E2E_PG_USER}"
},
{
"match": "schema",
"value": "public"
},
{
"match": "api[ _-]?key|api[ _-]?token|access[ _-]?token|auth[ _-]?token|^token$|secret|credential",
"value": "${E2E_API_TOKEN}"
}
]
},
"askAnswerNotes": [
"Rules are first-match-wins. Every rule is tested against the question id before any rule is tested against the prompt, so a rule that claims an id shadows a later rule that would have claimed the prompt.",
"The agent writes its own question ids. The 2026-08-28 CI run asked: api_key, account_id, table_prefix and stripe_secret_key for Stripe; host, port, database, user, password, schema and pg-host, pg-port, pg-database, pg-user, pg-password, pg-schema for Postgres.",
"account[ _-]?id sits ahead of stripe so \"Stripe Account ID (optional)\" takes an account id instead of the API key.",
"The last rule is a net, not a router. It claims a vendor-less id such as api_key on the id pass, ahead of the stripe rule that would claim the same question on the prompt pass. That is deliberate: a placeholder token and a placeholder Stripe key are equally fake, and a question that reaches the sentinel breaks the create it feeds.",
"E2E_API_TOKEN comes from the workbench runner. An unset var makes the rule inert, and the question falls through to the sentinel."
],
"variations": [
{
"name": "default",
Expand Down
Loading