Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 115 additions & 2 deletions src/agent/runner/harness/pi/__tests__/structured-run.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import { piBackend } from '..';
import { runPiTask } from '../task';
import { AgentSignals } from '@agent/signals';
import { analytics } from '@utils/analytics';
import { createSecurityExtension } from '../security';
import { setupPostHogMcp } from '../mcp';
import {
Expand Down Expand Up @@ -30,6 +33,8 @@ const state = vi.hoisted(() => ({
noTools: false,
// Commentary blocks sent before the answer in the same turn.
preamble: [] as string[],
toolName: 'find',
toolError: undefined as string | undefined,
}));
vi.mock('@utils/analytics', () => ({
analytics: { wizardCapture: vi.fn(), capture: vi.fn() },
Expand Down Expand Up @@ -75,11 +80,17 @@ vi.mock('../mcp', () => ({
),
}));
vi.mock('../tools', () => ({
createWizardPiTools: () => [{ name: 'set_env_values' }],
createWizardPiTools: () => [
{ name: 'set_env_values' },
{ name: 'load_skill_menu' },
],
}));
vi.mock('../tasks', () => ({
createWizardPiTaskTools: () => ({ tools: [], store: state.tasks }),
}));
vi.mock('../orchestrator-tools', () => ({
createPiOrchestratorTools: () => [],
}));
vi.mock('../subagent', () => ({
createDispatchAgentTool: () => ({ name: 'dispatch_agent' }),
}));
Expand Down Expand Up @@ -135,9 +146,18 @@ vi.mock('@earendil-works/pi-coding-agent', () => ({
if (!state.noTools)
state.listener?.({
type: 'tool_execution_start',
toolName: 'find',
toolName: state.toolName,
args: {},
});
if (state.toolError)
state.listener?.({
type: 'tool_execution_end',
toolName: state.toolName,
isError: true,
result: {
content: [{ type: 'text', text: state.toolError }],
},
});
state.listener?.({
type: 'message_end',
message: {
Expand Down Expand Up @@ -248,6 +268,9 @@ beforeEach(() => {
state.hang = false;
state.noTools = false;
state.preamble = [];
state.toolName = 'find';
state.toolError = undefined;
vi.mocked(analytics.wizardCapture).mockClear();
});

it.each([
Expand Down Expand Up @@ -341,6 +364,96 @@ it('ends a scan that outlives its budget as a timeout', async () => {
});
});

describe.each(['linear', 'task'] as const)('%s setup signals', (mode) => {
const run = (inputs: BackendRunInputs) =>
mode === 'linear'
? piBackend.run(inputs)
: runPiTask({
...inputs,
orchestrator: {
currentTaskId: 'integration',
validTypes: ['integration'],
store: { get: () => ({ status: 'running' }) } as never,
},
spinnerMessage: 'Setting up SDK',
successMessage: 'Installed the SDK',
requestRemark: true,
allowedTools: ['Read', 'load_skill_menu'],
analyticsProperties: {},
});

it.each([
[AgentSignals.ERROR_MCP_MISSING, AgentErrorType.MCP_MISSING],
[AgentSignals.ERROR_RESOURCE_MISSING, AgentErrorType.RESOURCE_MISSING],
])(
'fails unavailable setup instructions marked %s',
async (marker, classification) => {
state.toolName = 'load_skill_menu';
state.toolError = 'Setup instructions could not be downloaded.';
state.text = `${marker} Could not load setup instructions and halt.`;
const inputs = runInputs(vi.fn(), null);
inputs.config.run.successMessage = 'Installed the SDK';
inputs.prompt = `Load the framework setup skill. If instructions are unavailable, emit ${marker} and halt.`;

await expect(run(inputs)).resolves.toEqual({
kind: 'failure',
classification,
});
expect(state.prompts).toHaveLength(1);
expect(analytics.wizardCapture).toHaveBeenCalledWith(
'agent aborted',
expect.objectContaining({ failure_mode: classification }),
);
expect(inputs.spinner.stop).not.toHaveBeenCalledWith('Installed the SDK');
expect(analytics.wizardCapture).not.toHaveBeenCalledWith(
'agent completed',
expect.anything(),
);
},
);

it('does not nudge unfinished work after a fatal setup signal', async () => {
state.tasks.set('1', { status: 'in_progress' });
state.text = `${AgentSignals.ERROR_MCP_MISSING} Setup unavailable.`;

await expect(run(runInputs(vi.fn(), null))).resolves.toEqual({
kind: 'failure',
classification: AgentErrorType.MCP_MISSING,
});
expect(state.prompts).toHaveLength(1);
});

it('keeps security termination ahead of a fatal setup signal', async () => {
state.text = `${AgentSignals.ERROR_RESOURCE_MISSING} Setup unavailable.`;
vi.mocked(createSecurityExtension).mockReturnValueOnce({
factory: () => undefined,
state: { criticalViolation: true, blockedCount: 1 },
} as unknown as ReturnType<typeof createSecurityExtension>);

await expect(run(runInputs(vi.fn(), null))).resolves.toEqual({
kind: 'failure',
classification: AgentErrorType.YARA_VIOLATION,
});
});

it('does not treat tool-result text as an agent-declared setup failure', async () => {
state.toolError = `${AgentSignals.ERROR_RESOURCE_MISSING} Retry loading.`;
state.text = 'Recovered and completed the setup.';

await expect(run(runInputs(vi.fn(), null))).resolves.toEqual({
kind: 'success',
});
});

it('keeps skill-install fallback nonfatal', async () => {
state.text = `${AgentSignals.SKILL_INSTALL_FAILED} integration-test — unavailable; completed without the skill.`;

await expect(run(runInputs(vi.fn(), null))).resolves.toEqual({
kind: 'success',
});
});
});

it('keeps the remark and leaves the middleware alone on an integration run', async () => {
state.text = 'Installed the SDK.';
const onMessage = vi.fn();
Expand Down
10 changes: 10 additions & 0 deletions src/agent/runner/harness/pi/completion.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,14 @@
import { AgentErrorType } from '../../../signals';
import type { AgentOutputSignals } from '../../../output-signals';

/** Fatal setup signals use the same precedence as the legacy harness. */
export function setupSignalFailure(
signals: AgentOutputSignals,
): AgentErrorType.MCP_MISSING | AgentErrorType.RESOURCE_MISSING | undefined {
if (signals.has('MCP_MISSING')) return AgentErrorType.MCP_MISSING;
if (signals.has('RESOURCE_MISSING')) return AgentErrorType.RESOURCE_MISSING;
return undefined;
}

/** Which completion guard should fail a pi run, or undefined for a clean finish. */
export function completionFailure(args: {
Expand Down
16 changes: 15 additions & 1 deletion src/agent/runner/harness/pi/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,11 @@ import type { ProgressEmitter } from '../../../progress';
import { createEmitLog } from '../../shared/progress-collector';
import type { TaskStore } from './tasks';
import type { SecurityState } from './security';
import { completionFailure, runErrorType } from './completion';
import {
completionFailure,
runErrorType,
setupSignalFailure,
} from './completion';
import { bindPiCancellation } from './cancellation';
import { structuredOutputExtension } from './structured-output';
import { classifyRunFailure, ErrorCodes } from '@shared/errors';
Expand Down Expand Up @@ -727,6 +731,7 @@ export const piBackend: AgentHarness = {
!security.state.criticalViolation &&
!inputs.signal?.aborted &&
!terminal &&
!setupSignalFailure(signals) &&
hasOpenTasks(wizardTaskTools.store)
) {
continueNudges += 1;
Expand All @@ -742,6 +747,7 @@ export const piBackend: AgentHarness = {
!structured &&
!security.state.criticalViolation &&
!terminal &&
!setupSignalFailure(signals) &&
!inputs.signal?.aborted
) {
try {
Expand Down Expand Up @@ -801,6 +807,14 @@ export const piBackend: AgentHarness = {
// matching the anthropic path's AgentErrorType.YARA_VIOLATION.
if (security.state.criticalViolation) return yaraViolationResult();

const setupFailure = setupSignalFailure(signals);
if (setupFailure) {
spinner.stop('Agent could not access setup instructions');
logToFile(`[pi] setup failed: ${setupFailure}`);
captureAborted(setupFailure);
return { kind: 'failure', classification: setupFailure };
}

// pi ends a run on any tool-call-less turn, so guard against a hollow
// success reaching the outro (nothing done, or stopped mid-plan).
const openTasks = !structured && hasOpenTasks(wizardTaskTools.store);
Expand Down
12 changes: 11 additions & 1 deletion src/agent/runner/harness/pi/task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import {
GATEWAY_PROVIDER,
withGatewayRemint,
} from './gateway';
import { runErrorType } from './completion';
import { runErrorType, setupSignalFailure } from './completion';
import { bindPiCancellation } from './cancellation';
import { classifyRunFailure, ErrorCodes } from '@shared/errors';
import { assembleCommandments } from '../../switchboard/commandments';
Expand Down Expand Up @@ -522,6 +522,7 @@ export async function runPiTask(inputs: TaskRunInputs): Promise<AgentResult> {
!security.state.criticalViolation &&
!inputs.signal?.aborted &&
!terminal &&
!setupSignalFailure(signals) &&
!isSettled(orchestrator)
) {
nudges += 1;
Expand All @@ -538,6 +539,7 @@ export async function runPiTask(inputs: TaskRunInputs): Promise<AgentResult> {
requestRemark &&
!security.state.criticalViolation &&
!terminal &&
!setupSignalFailure(signals) &&
!inputs.signal?.aborted
) {
try {
Expand Down Expand Up @@ -598,6 +600,14 @@ export async function runPiTask(inputs: TaskRunInputs): Promise<AgentResult> {
return { kind: 'failure', classification: AgentErrorType.YARA_VIOLATION };
}

const setupFailure = setupSignalFailure(signals);
if (setupFailure) {
spinner.stop('Agent could not access setup instructions');
logToFile(`[pi-task] setup failed: ${setupFailure}`);
captureAborted(setupFailure);
return { kind: 'failure', classification: setupFailure };
}

const remark = signals.remark();
if (remark) {
analytics.capture(WIZARD_REMARK_EVENT_NAME, { remark });
Expand Down
Loading