Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions server/plugins/sentry.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import { sentryCloudflareNitroPlugin } from '@sentry/nuxt/module/plugins'
import { createSentryDataCollection, isBestEffortCacheWriteError } from '../../shared/sentry'
import { createSentryDataCollection, isBestEffortCacheWriteError, isExpectedUpstreamOutageError } from '../../shared/sentry'

function isDroppedSignature(error: unknown): boolean {
return isBestEffortCacheWriteError(error) || isExpectedUpstreamOutageError(error)
}

export default defineNitroPlugin((nitroApp) => {
const { sentry } = useRuntimeConfig()
Expand All @@ -15,12 +19,13 @@ export default defineNitroPlugin((nitroApp) => {
tracesSampleRate: sentry.tracesSampleRate,
dataCollection: createSentryDataCollection(),
// Nitro's route cache catches its own KV write failures and forwards the
// caught error here as unhandled (SKILLD-17). That is the same failure
// class `writeCache` records as a wide event, so drop the duplicate and
// keep the wide event as the visibility path.
// caught error here as unhandled (SKILLD-17). The registry handlers'
// intended upstream-outage 503s land here the same way (SKILLD-11,
// SKILLD-1E). Both are failure classes already recorded as wide events,
// so drop the duplicates and keep the wide events as the visibility path.
beforeSend(event, hint) {
const values = event.exception?.values ?? []
if (isBestEffortCacheWriteError(hint.originalException) || values.some(isBestEffortCacheWriteError))
if (isDroppedSignature(hint.originalException) || values.some(isDroppedSignature))
return null
return event
},
Expand Down
40 changes: 35 additions & 5 deletions shared/sentry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,22 @@ export function sentryRelease(): string | undefined {
return process.env.SENTRY_RELEASE || process.env.GITHUB_SHA || undefined
}

/**
* The message an error carries as it reaches the Sentry `beforeSend` hook.
*
* The hook sees two shapes. `hint.originalException` is the thrown `Error`,
* so the message sits on `.message`. A serialized exception value inside
* `event.exception.values` is a plain object, and Sentry puts the message on
* `.value` (an `H3Error` serialized by Nitro keeps `.message`).
*/
function sentryExceptionMessage(error: unknown): unknown {
return error instanceof Error
? error.message
: typeof error === 'object' && error !== null
? (error as { message?: unknown }).message ?? (error as { value?: unknown }).value
: undefined
}

/**
* The message prefix Workers KV rejects every cache write with
* (`KV PUT failed: 429 Too Many Requests`, Sentry SKILLD-17). Nitro's route
Expand All @@ -48,14 +64,28 @@ export function sentryRelease(): string | undefined {
* `beforeSend`; every other error still reaches Sentry.
*/
export function isBestEffortCacheWriteError(error: unknown): boolean {
const message = error instanceof Error
? error.message
: typeof error === 'object' && error !== null
? (error as { message?: unknown }).message ?? (error as { value?: unknown }).value
: undefined
const message = sentryExceptionMessage(error)
return typeof message === 'string' && message.startsWith('KV PUT failed:')
}

/**
* The suffix shared by the intended upstream-outage 503s the registry handlers
* raise after `fetchUpstreamTree` or `fetchUpstreamText` reports the outage as
* an operational wide event ('Skill source is unavailable upstream',
* 'SKILL.md source is unavailable upstream', 'Asset source is unavailable
* upstream', Sentry SKILLD-11 and SKILLD-1E). Nitro forwards the thrown
* `createError` to Sentry as unhandled, so the capture duplicates a failure
* class the wide event already records. The server Sentry config drops that
* signature in `beforeSend`; every other error still reaches Sentry.
*
* The gone-source 410 ('Skill source is gone upstream') is a permanent
* verdict, not an outage, and does not carry the suffix, so it still lands.
*/
export function isExpectedUpstreamOutageError(error: unknown): boolean {
const message = sentryExceptionMessage(error)
return typeof message === 'string' && message.endsWith('unavailable upstream')
}

export function createSentryDataCollection() {
return {
userInfo: false,
Expand Down
47 changes: 46 additions & 1 deletion test/unit/sentry-cache-write-filter.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { isBestEffortCacheWriteError } from '../../shared/sentry'
import { isBestEffortCacheWriteError, isExpectedUpstreamOutageError } from '../../shared/sentry'

describe('isBestEffortCacheWriteError', () => {
it('matches the SKILLD-17 rate-limited KV write signature', () => {
Expand All @@ -25,6 +25,32 @@ describe('isBestEffortCacheWriteError', () => {
})
})

describe('isExpectedUpstreamOutageError', () => {
it('matches the three upstream-outage 503 signatures the handlers throw', () => {
expect(isExpectedUpstreamOutageError(new Error('Skill source is unavailable upstream'))).toBe(true)
expect(isExpectedUpstreamOutageError(new Error('SKILL.md source is unavailable upstream'))).toBe(true)
expect(isExpectedUpstreamOutageError(new Error('Asset source is unavailable upstream'))).toBe(true)
})

it('matches the Sentry-serialized exception, whose message arrives as value', () => {
expect(isExpectedUpstreamOutageError({
type: 'Error',
value: 'SKILL.md source is unavailable upstream',
mechanism: { handled: false, type: 'auto.function.nuxt.nitro' },
})).toBe(true)
})

it('ignores the gone-source 410, a permanent verdict rather than an outage', () => {
expect(isExpectedUpstreamOutageError(new Error('Skill source is gone upstream'))).toBe(false)
})

it('ignores unrelated errors and values that are not errors', () => {
expect(isExpectedUpstreamOutageError(new Error('KV PUT failed: 429 Too Many Requests'))).toBe(false)
expect(isExpectedUpstreamOutageError('Skill source is unavailable upstream')).toBe(false)
expect(isExpectedUpstreamOutageError(undefined)).toBe(false)
})
})

const { sentryCloudflareNitroPlugin } = vi.hoisted(() => ({
sentryCloudflareNitroPlugin: vi.fn(),
}))
Expand Down Expand Up @@ -66,11 +92,30 @@ describe('sentry beforeSend cache-write filter', () => {
expect(beforeSend(event, {})).toBeNull()
})

it('drops the intended upstream-outage 503 the handler already reported as a wide event', () => {
const event: SentryEventFixture = {
exception: { values: [{ type: 'Error', value: 'Skill source is unavailable upstream' }] },
}
expect(beforeSend(event, { originalException: new Error('Skill source is unavailable upstream') }))
.toBeNull()
})

it('drops the outage on the serialized exception alone, when the original error is absent', () => {
const event: SentryEventFixture = {
exception: { values: [{ type: 'Error', value: 'Asset source is unavailable upstream' }] },
}
expect(beforeSend(event, {})).toBeNull()
})

it('keeps every other event', () => {
const event: SentryEventFixture = {
exception: { values: [{ type: 'Error', value: 'D1_ERROR: D1 DB is overloaded.' }] },
}
expect(beforeSend(event, { originalException: new Error('D1_ERROR: D1 DB is overloaded.') }))
.toBe(event)
const goneEvent: SentryEventFixture = {
exception: { values: [{ type: 'Error', value: 'Skill source is gone upstream' }] },
}
expect(beforeSend(goneEvent, {})).toBe(goneEvent)
})
})
Loading