Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 71 additions & 4 deletions src/lib/agent/runner/__tests__/switchboard.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,52 @@ describe('switchboard resolveHarness — CLI precedence', () => {
});
});

describe('switchboard resolveHarness — pi flag is gated to posthog-integration', () => {
it('honours the pi flag for posthog-integration', () => {
const pick = resolveHarness({
program: 'posthog-integration',
flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
});
expect(pick).toEqual({ harness: Harness.pi, model: GPT5_4_MODEL });
});

it('ignores the pi flag for self-driving — stays on the anthropic default', () => {
const pick = resolveHarness({
program: 'self-driving',
flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
});
expect(pick).toEqual({
harness: Harness.anthropic,
model: DEFAULT_AGENT_MODEL,
});
});

it('ignores the pi flag for every non-posthog-integration program', () => {
for (const program of PROGRAM_IDS) {
if (program === 'posthog-integration') continue;
expect(
resolveHarness({
program,
flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
}).harness,
).toBe(Harness.anthropic);
}
});

it('leaves the sequence unclamped for a gated program (pi never selected)', () => {
const ctx: SwitchboardCtx = {
program: 'self-driving',
flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
};
resolveBinding(ctx);
expect(ctx.trace).toEqual({
harness: 'binding',
model: 'binding',
sequence: 'binding',
});
});
});

describe('switchboard decision trace', () => {
it('stamps binding sources when nothing overrides', () => {
const ctx: SwitchboardCtx = { program: 'posthog-integration', flags: {} };
Expand Down Expand Up @@ -237,17 +283,38 @@ describe('switchboard modelCapabilities', () => {
});

describe('switchboard wizard-pi-effort flag', () => {
const PI_ON = { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' };

it('overrides effort for reasoning models; ignores invalid; skips non-reasoning', () => {
expect(
modelCapabilities(GPT5_4_MODEL, { [WIZARD_PI_EFFORT_FLAG_KEY]: 'high' })
.thinkingLevel,
modelCapabilities(GPT5_4_MODEL, {
...PI_ON,
[WIZARD_PI_EFFORT_FLAG_KEY]: 'high',
}).thinkingLevel,
).toBe('high');
expect(
modelCapabilities(GPT5_4_MODEL, { [WIZARD_PI_EFFORT_FLAG_KEY]: 'banana' })
.thinkingLevel,
modelCapabilities(GPT5_4_MODEL, {
...PI_ON,
[WIZARD_PI_EFFORT_FLAG_KEY]: 'banana',
}).thinkingLevel,
).toBe('low');
expect(
modelCapabilities('openai/gpt-4o', {
...PI_ON,
[WIZARD_PI_EFFORT_FLAG_KEY]: 'high',
}).thinkingLevel,
).toBeUndefined();
});

it('is inert unless the pi-harness flag is on — effort cannot ride a non-pi pick', () => {
// Effort set but the pi flag off: the override is ignored, so the model's
// own table effort stands (gpt-5.4 → low) and anthropic keeps no effort.
expect(
modelCapabilities(GPT5_4_MODEL, { [WIZARD_PI_EFFORT_FLAG_KEY]: 'high' })
.thinkingLevel,
).toBe('low');
expect(
modelCapabilities(DEFAULT_AGENT_MODEL, {
[WIZARD_PI_EFFORT_FLAG_KEY]: 'high',
}).thinkingLevel,
).toBeUndefined();
Expand Down
8 changes: 7 additions & 1 deletion src/lib/agent/runner/switchboard/harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
WIZARD_USE_PI_HARNESS_FLAG_KEY,
} from '@lib/constants';
import { logToFile } from '@utils/debug';
import type { ProgramId } from '@lib/programs/program-registry';
import { anthropicBackend } from '../harness/anthropic';
import { piBackend } from '../harness/pi';
import type { AgentHarness } from '../harness/types';
Expand Down Expand Up @@ -48,13 +49,18 @@ const PI_MODEL_FLAG_VARIANTS: Record<string, string> = {
'sonnet-5': SONNET_5_MODEL,
};

/** Programs the wizard-use-pi-harness flag may switch to pi; off this set the flag is a no-op and the binding default stands. */
const PI_FLAG_PROGRAMS = new Set<ProgramId>(['posthog-integration']);

/**
* `wizard-use-pi-harness` on → pi, paired with the `wizard-pi-model` variant
* (unknown/missing variant → gpt-5.4). Otherwise binding default.
* (unknown/missing variant → gpt-5.4). Off `PI_FLAG_PROGRAMS`, the flag is
* ignored and the binding default stands.
*/
const flagRunnerOverride: Middleware<HarnessPick> = (ctx, next) => {
const pick = next();
if (ctx.flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true') return pick;
if (!PI_FLAG_PROGRAMS.has(ctx.program)) return pick;
if (ctx.trace) Object.assign(ctx.trace, { harness: 'flag', model: 'flag' });
const variant = ctx.flags[WIZARD_PI_MODEL_FLAG_KEY] ?? '';
return {
Expand Down
5 changes: 3 additions & 2 deletions src/lib/agent/runner/switchboard/models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
GPT5_4_MODEL,
GPT5_MINI_MODEL,
WIZARD_PI_EFFORT_FLAG_KEY,
WIZARD_USE_PI_HARNESS_FLAG_KEY,
} from '@lib/constants';

/** Reasoning effort. pi maps it to `reasoning_effort` for openai-completions. */
Expand Down Expand Up @@ -77,8 +78,8 @@ export function modelCapabilities(
flags: Record<string, string> = {},
): ModelCapabilities {
const caps = MODEL_CAPABILITIES[modelId] ?? defaultCaps(modelId);
// `wizard-pi-effort` overrides the table for reasoning models only; an
// unknown variant is ignored.
// The wizard-pi-effort override applies only when the pi-harness flag selected the run; otherwise the model's own table effort stands.
if (flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true') return caps;
const effort = flags[WIZARD_PI_EFFORT_FLAG_KEY] as ThinkingLevel;
if (caps.reasoning && EFFORT_FLAG_VARIANTS.includes(effort)) {
return { ...caps, thinkingLevel: effort };
Expand Down
Loading