Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/error-catalog.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ screen, debug log) keep the full detail.
| `PHW_AGENT_INVALID_STRUCTURED_OUTPUT` | agent | schema-bound agent run ended without output matching its schema | yes |
| `PHW_AGENT_INCOMPLETE_TASKS` | agent | agent stopped with planned tasks open | case-by-case |
| `PHW_AGENT_ORCHESTRATOR_SKILL_VARIANT_MISSING` | agent | orchestrator preflight could not download a task skill variant | yes |
| `PHW_AGENT_ORCHESTRATOR_TASKS_FAILED` | agent | orchestrator queue drained with failed/blocked required tasks | case-by-case |
| `PHW_AGENT_ORCHESTRATOR_TASKS_FAILED` | agent | orchestrator queue drained with failed required tasks | case-by-case |
| `PHW_AGENT_ORCHESTRATOR_TASKS_BLOCKED` | agent | orchestrator queue drained with blocked tasks that never ran, none failed | case-by-case |
| `PHW_AGENT_ORCHESTRATOR_SINK_INVARIANT` | agent | orchestrator plan violates sink coverage invariant | no |
| `PHW_SETTINGS_UNFIXABLE_CONFLICT` | settings | Claude settings conflict that cannot be auto-neutralized (managed/unwritable) | no |
| `PHW_INTERNAL_UNHANDLED` | internal | catch-all: an unexpected error escaped the pipeline | yes |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@ vi.mock('@utils/analytics', () => ({
import { QueueStore } from '@agent/runner/sequence/orchestrator/queue';
import {
describeDrainFailure,
drainFailureCode,
drainVerdict,
} from '@agent/runner/sequence/orchestrator/orchestrator-runner';
import { ErrorCodes } from '@shared/errors';

describe('drainVerdict', () => {
let dir: string;
Expand Down Expand Up @@ -99,3 +101,17 @@ describe('describeDrainFailure', () => {
).toBe('the report step failed');
});
});

describe('drainFailureCode', () => {
it('names a drain with a failed required step as failed', () => {
expect(drainFailureCode({ requiredFailedTypes: ['install'] })).toBe(
ErrorCodes.AgentOrchestratorTasksFailed,
);
});

it('names a drain where nothing failed but steps never ran as blocked', () => {
expect(drainFailureCode({ requiredFailedTypes: [] })).toBe(
ErrorCodes.AgentOrchestratorTasksBlocked,
);
});
});
25 changes: 21 additions & 4 deletions src/agent/runner/sequence/orchestrator/orchestrator-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,12 @@ import { logToFile } from '@utils/debug';
import { ringTerminalBell } from '@utils/terminal-bell';
import { AGENT_ERROR_CODE } from '../../../error-map';
import { AgentErrorType } from '../../../agent-interface';
import { classifyRunFailure, ErrorCodes, WizardError } from '@shared/errors';
import {
classifyRunFailure,
type ErrorCode,
ErrorCodes,
WizardError,
} from '@shared/errors';
import type { AgentResult } from '../../harness/types';
import type { AgentInteraction } from '../../../progress';
import type {
Expand Down Expand Up @@ -471,6 +476,15 @@ export function drainVerdict(tasks: readonly QueuedTask[]): {
};
}

/** A drain where a required step failed, or where nothing failed but steps were blocked and never ran. */
export function drainFailureCode(verdict: {
requiredFailedTypes: readonly string[];
}): ErrorCode {
return verdict.requiredFailedTypes.length > 0
? ErrorCodes.AgentOrchestratorTasksFailed
: ErrorCodes.AgentOrchestratorTasksBlocked;
}

/**
* The one-line "what went wrong" the abort message leads with.
*
Expand Down Expand Up @@ -1444,19 +1458,22 @@ async function executeOrchestrator(
', ',
)}.\n\nPlease try again, approving all permissions on the PostHog authorization screen. If it still fails, report it to: ${WIZARD_CONTACT_EMAIL}`
: `The wizard was unable to set up PostHog: ${whatFailed}.\n\nPlease report this to: ${WIZARD_CONTACT_EMAIL}`;
const code = drainFailureCode(verdict);
return failed({
code: ErrorCodes.AgentOrchestratorTasksFailed,
code,
message,
error: new WizardError(
'orchestrator drain ended with failed tasks',
code === ErrorCodes.AgentOrchestratorTasksFailed
? 'orchestrator drain ended with failed tasks'
: 'orchestrator drain ended with blocked tasks',
{
tasks_failed: summary.failed,
tasks_blocked: blocked,
failed_types: failedTypes,
missing_oauth_scopes: missingScopes.join(' '),
queue_state: JSON.stringify(store.list()),
},
ErrorCodes.AgentOrchestratorTasksFailed,
code,
),
});
}
Expand Down
8 changes: 7 additions & 1 deletion src/shared/errors/catalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,13 @@ export const ERROR_CATALOG: Record<ErrorCode, ErrorCatalogEntry> = {
[ErrorCodes.AgentOrchestratorTasksFailed]: {
group: 'agent',
retry: 'case-by-case',
description: 'The orchestrator queue drained with failed or blocked tasks.',
description: 'The orchestrator queue drained with failed required tasks.',
},
[ErrorCodes.AgentOrchestratorTasksBlocked]: {
group: 'agent',
retry: 'case-by-case',
description:
'The orchestrator queue drained with no failed task but blocked ones that never ran.',
},
[ErrorCodes.AgentOrchestratorHollowRun]: {
group: 'agent',
Expand Down
1 change: 1 addition & 0 deletions src/shared/errors/codes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ export const ErrorCodes = {
AgentOrchestratorSkillVariantMissing:
'PHW_AGENT_ORCHESTRATOR_SKILL_VARIANT_MISSING',
AgentOrchestratorTasksFailed: 'PHW_AGENT_ORCHESTRATOR_TASKS_FAILED',
AgentOrchestratorTasksBlocked: 'PHW_AGENT_ORCHESTRATOR_TASKS_BLOCKED',
AgentOrchestratorHollowRun: 'PHW_AGENT_ORCHESTRATOR_HOLLOW_RUN',
AgentOrchestratorSinkInvariant: 'PHW_AGENT_ORCHESTRATOR_SINK_INVARIANT',
/** The backend answered the mint with a deliberate refusal. */
Expand Down
48 changes: 48 additions & 0 deletions src/shared/utils/__tests__/analytics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { v4 as uuidv4 } from 'uuid';
import { ANALYTICS_TEAM_TAG, WIZARD_FLAG_KEYS } from '@shared/constants';
import { VERSION } from '@shared/version';
import type { ApiUser } from '@shared/api';
import { ErrorCodes, WizardError } from '@shared/errors';

vi.mock('posthog-node');
vi.mock('uuid');
Expand Down Expand Up @@ -96,6 +97,53 @@ describe('Analytics', () => {
);
});

it('fingerprints a coded error by its code, whatever its stack', () => {
const error = new WizardError(
'orchestrator drain ended with failed tasks',
undefined,
ErrorCodes.AgentOrchestratorTasksFailed,
);

analytics.captureException(error);

expect(mockPostHogInstance.captureException).toHaveBeenCalledWith(
error,
'test-uuid',
expect.objectContaining({
$exception_fingerprint: 'wizard_PHW_AGENT_ORCHESTRATOR_TASKS_FAILED',
}),
);
});

it('fingerprints a plain error by the error_code it is captured with', () => {
analytics.captureException(new Error('x'), {
error_code: ErrorCodes.AgentYaraViolation,
});

expect(mockPostHogInstance.captureException).toHaveBeenCalledWith(
expect.any(Error),
'test-uuid',
expect.objectContaining({
$exception_fingerprint: 'wizard_PHW_AGENT_YARA_VIOLATION',
}),
);
});

it("keeps a caller's own fingerprint", () => {
analytics.captureException(new Error('x'), {
error_code: ErrorCodes.AgentYaraViolation,
$exception_fingerprint: 'wizard_oauth_invalid_grant',
});

expect(mockPostHogInstance.captureException).toHaveBeenCalledWith(
expect.any(Error),
'test-uuid',
expect.objectContaining({
$exception_fingerprint: 'wizard_oauth_invalid_grant',
}),
);
});

it('should capture exception with tags included in properties', () => {
const error = new Error('Test error');
const properties = { integration: 'nextjs' };
Expand Down
6 changes: 6 additions & 0 deletions src/shared/utils/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import type { ApiUser } from '@shared/api';
import { v4 as uuidv4 } from 'uuid';
import { IS_PRODUCTION_BUILD, RUN_SURFACE, TASK_ID, TASK_RUN_ID } from '@env';
import { VERSION } from '@shared/version';
import { WizardError } from '@shared/errors';
import { logToFile } from './debug';
import { applyCiFlagOverrides } from './ci-flag-overrides';

Expand Down Expand Up @@ -275,8 +276,13 @@ export class Analytics {
}

captureException(error: Error, properties: Record<string, unknown> = {}) {
const code =
(error instanceof WizardError && error.code) ||
(typeof properties.error_code === 'string' ? properties.error_code : '');
this.client.captureException(error, this.distinctId ?? this.anonymousId, {
team: ANALYTICS_TEAM_TAG,
// One issue per error code: the stack's install path and chunk hash otherwise split it per route and release.
...(code ? { $exception_fingerprint: `wizard_${code}` } : {}),
...this.tags,
...properties,
});
Expand Down
Loading