diff --git a/docs/error-catalog.md b/docs/error-catalog.md index 29a368f8c..ceb1495fe 100644 --- a/docs/error-catalog.md +++ b/docs/error-catalog.md @@ -106,7 +106,8 @@ screen, debug log) keep the full detail. | `PHW_AGENT_INVALID_STRUCTURED_OUTPUT` | agent | schema-bound agent run ended without output matching its schema | yes | | `PHW_AGENT_INCOMPLETE_TASKS` | agent | agent stopped with planned tasks open | case-by-case | | `PHW_AGENT_ORCHESTRATOR_SKILL_VARIANT_MISSING` | agent | orchestrator preflight could not download a task skill variant | yes | -| `PHW_AGENT_ORCHESTRATOR_TASKS_FAILED` | agent | orchestrator queue drained with failed/blocked required tasks | case-by-case | +| `PHW_AGENT_ORCHESTRATOR_TASKS_FAILED` | agent | orchestrator queue drained with failed required tasks | case-by-case | +| `PHW_AGENT_ORCHESTRATOR_TASKS_BLOCKED` | agent | orchestrator queue drained with blocked tasks that never ran, none failed | case-by-case | | `PHW_AGENT_ORCHESTRATOR_SINK_INVARIANT` | agent | orchestrator plan violates sink coverage invariant | no | | `PHW_SETTINGS_UNFIXABLE_CONFLICT` | settings | Claude settings conflict that cannot be auto-neutralized (managed/unwritable) | no | | `PHW_INTERNAL_UNHANDLED` | internal | catch-all: an unexpected error escaped the pipeline | yes | diff --git a/src/agent/runner/sequence/orchestrator/__tests__/optional-tasks.test.ts b/src/agent/runner/sequence/orchestrator/__tests__/optional-tasks.test.ts index c44157dd6..ff54c3833 100644 --- a/src/agent/runner/sequence/orchestrator/__tests__/optional-tasks.test.ts +++ b/src/agent/runner/sequence/orchestrator/__tests__/optional-tasks.test.ts @@ -10,8 +10,10 @@ vi.mock('@utils/analytics', () => ({ import { QueueStore } from '@agent/runner/sequence/orchestrator/queue'; import { describeDrainFailure, + drainFailureCode, drainVerdict, } from '@agent/runner/sequence/orchestrator/orchestrator-runner'; +import { ErrorCodes } from '@shared/errors'; describe('drainVerdict', () => { let dir: string; @@ -99,3 +101,17 @@ describe('describeDrainFailure', () => { ).toBe('the report step failed'); }); }); + +describe('drainFailureCode', () => { + it('names a drain with a failed required step as failed', () => { + expect(drainFailureCode({ requiredFailedTypes: ['install'] })).toBe( + ErrorCodes.AgentOrchestratorTasksFailed, + ); + }); + + it('names a drain where nothing failed but steps never ran as blocked', () => { + expect(drainFailureCode({ requiredFailedTypes: [] })).toBe( + ErrorCodes.AgentOrchestratorTasksBlocked, + ); + }); +}); diff --git a/src/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/agent/runner/sequence/orchestrator/orchestrator-runner.ts index 67c6d8548..adfe3ee60 100644 --- a/src/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -42,7 +42,12 @@ import { logToFile } from '@utils/debug'; import { ringTerminalBell } from '@utils/terminal-bell'; import { AGENT_ERROR_CODE } from '../../../error-map'; import { AgentErrorType } from '../../../agent-interface'; -import { classifyRunFailure, ErrorCodes, WizardError } from '@shared/errors'; +import { + classifyRunFailure, + type ErrorCode, + ErrorCodes, + WizardError, +} from '@shared/errors'; import type { AgentResult } from '../../harness/types'; import type { AgentInteraction } from '../../../progress'; import type { @@ -471,6 +476,15 @@ export function drainVerdict(tasks: readonly QueuedTask[]): { }; } +/** A drain where a required step failed, or where nothing failed but steps were blocked and never ran. */ +export function drainFailureCode(verdict: { + requiredFailedTypes: readonly string[]; +}): ErrorCode { + return verdict.requiredFailedTypes.length > 0 + ? ErrorCodes.AgentOrchestratorTasksFailed + : ErrorCodes.AgentOrchestratorTasksBlocked; +} + /** * The one-line "what went wrong" the abort message leads with. * @@ -1444,11 +1458,14 @@ async function executeOrchestrator( ', ', )}.\n\nPlease try again, approving all permissions on the PostHog authorization screen. If it still fails, report it to: ${WIZARD_CONTACT_EMAIL}` : `The wizard was unable to set up PostHog: ${whatFailed}.\n\nPlease report this to: ${WIZARD_CONTACT_EMAIL}`; + const code = drainFailureCode(verdict); return failed({ - code: ErrorCodes.AgentOrchestratorTasksFailed, + code, message, error: new WizardError( - 'orchestrator drain ended with failed tasks', + code === ErrorCodes.AgentOrchestratorTasksFailed + ? 'orchestrator drain ended with failed tasks' + : 'orchestrator drain ended with blocked tasks', { tasks_failed: summary.failed, tasks_blocked: blocked, @@ -1456,7 +1473,7 @@ async function executeOrchestrator( missing_oauth_scopes: missingScopes.join(' '), queue_state: JSON.stringify(store.list()), }, - ErrorCodes.AgentOrchestratorTasksFailed, + code, ), }); } diff --git a/src/shared/errors/catalog.ts b/src/shared/errors/catalog.ts index 676884a30..843e2fd0b 100644 --- a/src/shared/errors/catalog.ts +++ b/src/shared/errors/catalog.ts @@ -230,7 +230,13 @@ export const ERROR_CATALOG: Record = { [ErrorCodes.AgentOrchestratorTasksFailed]: { group: 'agent', retry: 'case-by-case', - description: 'The orchestrator queue drained with failed or blocked tasks.', + description: 'The orchestrator queue drained with failed required tasks.', + }, + [ErrorCodes.AgentOrchestratorTasksBlocked]: { + group: 'agent', + retry: 'case-by-case', + description: + 'The orchestrator queue drained with no failed task but blocked ones that never ran.', }, [ErrorCodes.AgentOrchestratorHollowRun]: { group: 'agent', diff --git a/src/shared/errors/codes.ts b/src/shared/errors/codes.ts index 49960adfd..51727d9d8 100644 --- a/src/shared/errors/codes.ts +++ b/src/shared/errors/codes.ts @@ -45,6 +45,7 @@ export const ErrorCodes = { AgentOrchestratorSkillVariantMissing: 'PHW_AGENT_ORCHESTRATOR_SKILL_VARIANT_MISSING', AgentOrchestratorTasksFailed: 'PHW_AGENT_ORCHESTRATOR_TASKS_FAILED', + AgentOrchestratorTasksBlocked: 'PHW_AGENT_ORCHESTRATOR_TASKS_BLOCKED', AgentOrchestratorHollowRun: 'PHW_AGENT_ORCHESTRATOR_HOLLOW_RUN', AgentOrchestratorSinkInvariant: 'PHW_AGENT_ORCHESTRATOR_SINK_INVARIANT', /** The backend answered the mint with a deliberate refusal. */ diff --git a/src/shared/utils/__tests__/analytics.test.ts b/src/shared/utils/__tests__/analytics.test.ts index 073eca80c..a2f439780 100644 --- a/src/shared/utils/__tests__/analytics.test.ts +++ b/src/shared/utils/__tests__/analytics.test.ts @@ -4,6 +4,7 @@ import { v4 as uuidv4 } from 'uuid'; import { ANALYTICS_TEAM_TAG, WIZARD_FLAG_KEYS } from '@shared/constants'; import { VERSION } from '@shared/version'; import type { ApiUser } from '@shared/api'; +import { ErrorCodes, WizardError } from '@shared/errors'; vi.mock('posthog-node'); vi.mock('uuid'); @@ -96,6 +97,53 @@ describe('Analytics', () => { ); }); + it('fingerprints a coded error by its code, whatever its stack', () => { + const error = new WizardError( + 'orchestrator drain ended with failed tasks', + undefined, + ErrorCodes.AgentOrchestratorTasksFailed, + ); + + analytics.captureException(error); + + expect(mockPostHogInstance.captureException).toHaveBeenCalledWith( + error, + 'test-uuid', + expect.objectContaining({ + $exception_fingerprint: 'wizard_PHW_AGENT_ORCHESTRATOR_TASKS_FAILED', + }), + ); + }); + + it('fingerprints a plain error by the error_code it is captured with', () => { + analytics.captureException(new Error('x'), { + error_code: ErrorCodes.AgentYaraViolation, + }); + + expect(mockPostHogInstance.captureException).toHaveBeenCalledWith( + expect.any(Error), + 'test-uuid', + expect.objectContaining({ + $exception_fingerprint: 'wizard_PHW_AGENT_YARA_VIOLATION', + }), + ); + }); + + it("keeps a caller's own fingerprint", () => { + analytics.captureException(new Error('x'), { + error_code: ErrorCodes.AgentYaraViolation, + $exception_fingerprint: 'wizard_oauth_invalid_grant', + }); + + expect(mockPostHogInstance.captureException).toHaveBeenCalledWith( + expect.any(Error), + 'test-uuid', + expect.objectContaining({ + $exception_fingerprint: 'wizard_oauth_invalid_grant', + }), + ); + }); + it('should capture exception with tags included in properties', () => { const error = new Error('Test error'); const properties = { integration: 'nextjs' }; diff --git a/src/shared/utils/analytics.ts b/src/shared/utils/analytics.ts index 714076e57..13267ee12 100644 --- a/src/shared/utils/analytics.ts +++ b/src/shared/utils/analytics.ts @@ -17,6 +17,7 @@ import type { ApiUser } from '@shared/api'; import { v4 as uuidv4 } from 'uuid'; import { IS_PRODUCTION_BUILD, RUN_SURFACE, TASK_ID, TASK_RUN_ID } from '@env'; import { VERSION } from '@shared/version'; +import { WizardError } from '@shared/errors'; import { logToFile } from './debug'; import { applyCiFlagOverrides } from './ci-flag-overrides'; @@ -275,8 +276,13 @@ export class Analytics { } captureException(error: Error, properties: Record = {}) { + const code = + (error instanceof WizardError && error.code) || + (typeof properties.error_code === 'string' ? properties.error_code : ''); this.client.captureException(error, this.distinctId ?? this.anonymousId, { team: ANALYTICS_TEAM_TAG, + // One issue per error code: the stack's install path and chunk hash otherwise split it per route and release. + ...(code ? { $exception_fingerprint: `wizard_${code}` } : {}), ...this.tags, ...properties, });