From 7eded82922493358692d098b81b6abcf9e5160d5 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Mon, 14 Sep 2026 14:47:33 +1000 Subject: [PATCH 01/11] feat(ops): share daily health and Sentry check results --- docs/shared-checkin.md | 47 +++++++++++++++++++ .../server/checks/daily-health-coverage.ts | 13 +++++ layers/identity/server/checks/daily-health.ts | 17 +++++++ .../server/tasks/daily-health-check.ts | 1 + .../server/utils/daily-health-check.spec.ts | 27 +++++++++++ .../server/utils/daily-health-check.ts | 8 +++- .../server/utils/daily-health-checkin.ts | 46 ++++++++++++++++++ nuxt.config.ts | 1 + package.json | 2 + pnpm-workspace.yaml | 2 + scripts/tools/daily-checkin-checks.mjs | 26 ++++++++++ scripts/tools/daily-checkin-data.mjs | 22 +++++++-- scripts/tools/daily-checkin-observability.mjs | 1 + test/fixtures/checkin-registry.ts | 5 ++ test/unit/daily-checkin-checks.test.ts | 39 +++++++++++++++ test/unit/daily-checkin-observability.test.ts | 1 + vitest.config.ts | 2 + 17 files changed, 256 insertions(+), 4 deletions(-) create mode 100644 docs/shared-checkin.md create mode 100644 layers/identity/server/checks/daily-health-coverage.ts create mode 100644 layers/identity/server/checks/daily-health.ts create mode 100644 layers/identity/server/utils/daily-health-checkin.ts create mode 100644 scripts/tools/daily-checkin-checks.mjs create mode 100644 test/fixtures/checkin-registry.ts create mode 100644 test/unit/daily-checkin-checks.test.ts diff --git a/docs/shared-checkin.md b/docs/shared-checkin.md new file mode 100644 index 00000000..5fda9d13 --- /dev/null +++ b/docs/shared-checkin.md @@ -0,0 +1,47 @@ +# Daily health check + +The operator email uses the public Nuxt Check-in runner. +The module discovers checks in `layers/identity/server/checks` during the build. + +The existing summary builder and health evaluator keep their current policy. +Both checks share one summary collection and one observation time. +The independent required list detects missing check files. +A run has a 60-second deadline. +If collection fails, the sender fails before claiming or sending an email. +The scheduled task records that failure through its existing reporting path. +Active reads may continue because D1 cannot cancel an active query. + +Each saved summary includes the versioned check report, deployment identity, severity, and evidence coverage. +The email shows coverage in HTML and text. +A known failure stays RED when another source is unavailable. +Unavailable evidence can never produce GREEN. +Detailed evidence stays in the existing private report storage and operator email. + +Recipients, cron schedules, delivery claims, retry behavior, and pause gates stay unchanged. +`send-digests` sends subscriber content, not system health emails. +Its outcome already enters the daily summary. +The embedding parity audit keeps its independent schedule and pruning policy. +Its recorded scheduled-run outcome already enters the daily summary. + +## Release + +This draft requires the unpublished `@harlan-zw/nuxt-checkin@0.1.0` release from harlan-nuxt PR 143. +After release, confirm the version and run `pnpm install` to update the lockfile. +The draft deliberately leaves the registry lockfile unchanged. +Local checks use the built package tarball, without committed file dependencies. + +## External daily check-in + +The external daily script runs the public [Sentry](https://sentry.io) backlog check with the saved health report. +It validates report age, required checks, site, environment, and the deployed Worker version. +Missing credentials or stale reports produce unavailable evidence. +Sentry credentials stay in the existing external locations. +Set `SENTRY_ENVIRONMENT` after confirming the production mapping. +Without that setting, the backlog check includes all project environments. + +The existing detailed Sentry probe runs only when the shared check finds unresolved issues. +It keeps recurrence and impact details for triage. +That extra request remains an optimization opportunity. +Live credentials and the environment mapping remain unverified. +The draft uses `nuxt-sentry@0.1.5` as a release placeholder for the public checks export. +Confirm its actual release version before updating the lockfile. diff --git a/layers/identity/server/checks/daily-health-coverage.ts b/layers/identity/server/checks/daily-health-coverage.ts new file mode 100644 index 00000000..97b0754f --- /dev/null +++ b/layers/identity/server/checks/daily-health-coverage.ts @@ -0,0 +1,13 @@ +import type { DailyHealthCheckEvent } from '../utils/daily-health-checkin' +import { defineCheck, pass, unavailable } from '@harlan-zw/nuxt-checkin/server' +import { collectDailyHealth } from '../utils/daily-health-checkin' + +export default defineCheck({ + id: 'skilld.daily-health-coverage', + async run(context) { + const summary = await collectDailyHealth(context) + return summary.warnings.length + ? unavailable('Daily health evidence is incomplete. See report warnings.') + : pass() + }, +}) diff --git a/layers/identity/server/checks/daily-health.ts b/layers/identity/server/checks/daily-health.ts new file mode 100644 index 00000000..7138bc4e --- /dev/null +++ b/layers/identity/server/checks/daily-health.ts @@ -0,0 +1,17 @@ +import type { DailyHealthCheckEvent } from '../utils/daily-health-checkin' +import { defineCheck, fail, pass, warn } from '@harlan-zw/nuxt-checkin/server' +import { collectDailyHealth } from '../utils/daily-health-checkin' + +export default defineCheck({ + id: 'skilld.daily-health', + async run(context) { + const summary = await collectDailyHealth(context) + const evidence = { reasons: summary.reasons } + const result = summary.status === 'RED' + ? fail(summary.reasons.join(' '), evidence) + : summary.status === 'AMBER' ? warn(summary.reasons.join(' '), evidence) : pass(evidence) + return summary.warnings.length && result._tag !== 'Pass' + ? { ...result, coverage: 'incomplete' as const } + : result + }, +}) diff --git a/layers/identity/server/tasks/daily-health-check.ts b/layers/identity/server/tasks/daily-health-check.ts index dc787d9a..3d500396 100644 --- a/layers/identity/server/tasks/daily-health-check.ts +++ b/layers/identity/server/tasks/daily-health-check.ts @@ -46,6 +46,7 @@ export default defineScheduledTask({ const attempt = await sendDailyHealthCheck(db, { now, + deployment: env.CF_VERSION_METADATA?.id, to, build: (database, options) => buildDailyHealthCheck(database, { ...options, diff --git a/layers/identity/server/utils/daily-health-check.spec.ts b/layers/identity/server/utils/daily-health-check.spec.ts index 723c601b..50654aeb 100644 --- a/layers/identity/server/utils/daily-health-check.spec.ts +++ b/layers/identity/server/utils/daily-health-check.spec.ts @@ -602,6 +602,33 @@ describe('sendDailyHealthCheck', () => { sqliteDbs.splice(0).forEach(db => db.close()) }) + it('does not send or claim when collection fails', async () => { + const { db, sqlite } = createDb() + sqliteDbs.push(sqlite) + const log = vi.spyOn(console, 'error').mockImplementation(() => {}) + const send = vi.fn() + const build = vi.fn().mockRejectedValue(new Error('Database unavailable')) + await expect(sendDailyHealthCheck(db, { to: 'ops@example.com', build, send })).rejects.toThrow('Daily health collection did not complete') + expect(build).toHaveBeenCalledTimes(1) + expect(send).not.toHaveBeenCalled() + expect(sqlite.prepare('SELECT COUNT(*) AS count FROM daily_health_checks').get()).toEqual({ count: 0 }) + expect(log).toHaveBeenCalled() + log.mockRestore() + }) + + it('preserves failures with incomplete evidence and stores the shared report', async () => { + const { db, sqlite } = createDb() + sqliteDbs.push(sqlite) + const send = vi.fn().mockResolvedValue({ _tag: 'accepted', messageId: 'msg_partial' }) + const build = vi.fn().mockResolvedValue(summary({ status: 'RED', reasons: ['Delivery failed.'], warnings: ['Inventory unavailable.'] })) + await expect(sendDailyHealthCheck(db, { now: new Date('2026-07-22T22:05:00Z'), to: 'ops@example.com', build, send })).resolves.toMatchObject({ _tag: 'Sent', status: 'RED' }) + expect(build).toHaveBeenCalledTimes(1) + const stored = sqlite.prepare('SELECT summary_json FROM daily_health_checks').get() as { summary_json: string } + expect(JSON.parse(stored.summary_json).checkin).toMatchObject({ severity: 'fail', coverage: 'incomplete', identity: { site: 'skilld.dev' } }) + expect(send.mock.calls[0]![0]).toMatchObject({ to: 'ops@example.com', subject: 'skilld health: RED (2026-07-23)' }) + expect(send.mock.calls[0]![0].text).toContain('Check coverage: incomplete') + }) + it('claims the report date before sending and deduplicates retries', async () => { const { db, sqlite } = createDb() sqliteDbs.push(sqlite) diff --git a/layers/identity/server/utils/daily-health-check.ts b/layers/identity/server/utils/daily-health-check.ts index 785002dc..f4a09a24 100644 --- a/layers/identity/server/utils/daily-health-check.ts +++ b/layers/identity/server/utils/daily-health-check.ts @@ -1,5 +1,6 @@ /// +import type { CheckReport } from '@harlan-zw/nuxt-checkin/server' import type { TokenExpiryStatus } from '#layers/registry/server/utils/github-token-expiry' import type { LatestScheduledRun, ScheduleHealth } from '#shared/schedule-policy' import type { SendEmailInput, SendEmailResult } from './email' @@ -7,6 +8,7 @@ import { TERMINAL_DISCOVERY_REJECTION_REASONS } from '#layers/registry/server/ut import { parseTokenExpiry, tokenExpiryStatus } from '#layers/registry/server/utils/github-token-expiry' import { evaluateScheduleHealth, SCHEDULE_POLICY } from '#shared/schedule-policy' import { DAILY_DISCOVERY_READ_BUDGET } from '#shared/server/x-ingest' +import { runDailyHealthChecks } from './daily-health-checkin' /** * A failed job whose exception is a decision rather than a fault. @@ -36,6 +38,7 @@ const CLAIM_STALE_SECONDS = 60 * 60 export type DailyHealthStatus = 'GREEN' | 'AMBER' | 'RED' export interface DailyHealthCheckSummary { + checkin?: CheckReport status: DailyHealthStatus reasons: string[] warnings: string[] @@ -323,6 +326,7 @@ type SummaryBuilder = (db: D1Database, options: { now: Date }) => Promise Promise interface SendDailyHealthCheckOptions { + deployment?: string now?: Date to: string build?: SummaryBuilder @@ -1344,6 +1348,7 @@ export function renderDailyHealthCheckText(summary: DailyHealthCheckSummary): st return [ `skilld daily health check: ${summary.status}`, + ...(summary.checkin ? [`Check coverage: ${summary.checkin.coverage}`] : []), `${summary.window.reportDate} (${summary.window.timeZone})`, `${summary.window.from} to ${summary.window.to}`, summary.window.workerVersion ? `Worker: ${summary.window.workerVersion}` : null, @@ -1432,6 +1437,7 @@ export function renderDailyHealthCheckHtml(summary: DailyHealthCheckSummary): st

Daily health check ${summary.status}

${escapeHtml(summary.window.reportDate)} (${escapeHtml(summary.window.timeZone)})
${escapeHtml(summary.window.from)} to ${escapeHtml(summary.window.to)}
+ ${summary.checkin ? `
Check coverage: ${escapeHtml(summary.checkin.coverage)}
` : ''} ${summary.window.workerVersion ? `
Worker ${escapeHtml(summary.window.workerVersion)}
` : ''}
Reasons
    ${htmlList(summary.reasons)}
@@ -1503,7 +1509,7 @@ export async function sendDailyHealthCheck( ): Promise { const now = options.now ?? new Date() const build = options.build ?? ((database, buildOptions) => buildDailyHealthCheck(database, buildOptions)) - const summary = await build(db, { now }) + const summary = await runDailyHealthChecks(db, build, now, options.deployment) const claimedAt = Math.floor(now.getTime() / 1000) const staleClaimBefore = claimedAt - CLAIM_STALE_SECONDS const summaryJson = JSON.stringify(summary) diff --git a/layers/identity/server/utils/daily-health-checkin.ts b/layers/identity/server/utils/daily-health-checkin.ts new file mode 100644 index 00000000..a02cbc13 --- /dev/null +++ b/layers/identity/server/utils/daily-health-checkin.ts @@ -0,0 +1,46 @@ +import type { CheckContext } from '@harlan-zw/nuxt-checkin/server' +import type { DailyHealthCheckSummary } from './daily-health-check' +import { runChecks } from '@harlan-zw/nuxt-checkin/server' +import checks from '#checkin/checks' + +export interface DailyHealthCheckEvent { + db: D1Database + build: (db: D1Database, options: { now: Date }) => Promise +} + +export function collectDailyHealth(context: CheckContext) { + const event = context.event + if (!event) + throw new Error('Daily health check context is missing.') + return context.collect(event.db, 'skilld.daily-health', async () => ({ + value: await event.build(event.db, { now: context.now }), + })) +} + +export async function runDailyHealthChecks(db: D1Database, build: DailyHealthCheckEvent['build'], now: Date, deployment = 'unknown') { + let summary: DailyHealthCheckSummary | undefined + const report = await runChecks(checks, { + event: { + db, + build: async (database: D1Database, options: { now: Date }) => { + summary = await build(database, options) + return summary + }, + } satisfies DailyHealthCheckEvent, + now, + required: ['skilld.daily-health', 'skilld.daily-health-coverage'], + identity: { site: 'skilld.dev', environment: import.meta.dev ? 'development' : 'production', deployment }, + timeoutMs: 60_000, + totalTimeoutMs: 60_000, + onError: (error, id) => console.error(`Daily health check failed: ${id}`, error), + }) + if (!summary) + throw new Error('Daily health collection did not complete. No email was sent.') + const reasons = report.results.flatMap(({ result }) => result._tag === 'Pass' ? [] : [result.reason]) + return { + ...summary, + checkin: report, + status: report.severity === 'fail' ? 'RED' as const : report.severity === 'warn' || report.coverage === 'incomplete' ? 'AMBER' as const : 'GREEN' as const, + reasons: reasons.length ? reasons : summary.reasons, + } +} diff --git a/nuxt.config.ts b/nuxt.config.ts index d46ce4b9..b9c1c826 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -46,6 +46,7 @@ export default defineNuxtConfig({ }, modules: [ + '@harlan-zw/nuxt-checkin', '@harlan-zw/nuxt-cf-jobs', '@harlan-zw/nuxt-cloudflare', '@harlan-zw/nuxt-dx', diff --git a/package.json b/package.json index a988f69e..62f3d4b2 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "@comark/nuxt": "catalog:", "@harlan-zw/comark-content": "catalog:", "@harlan-zw/nuxt-cf-jobs": "catalog:", + "@harlan-zw/nuxt-checkin": "catalog:", "@harlan-zw/nuxt-use-query": "catalog:", "@harlan-zw/nuxt-wide-events": "catalog:", "@iconify-json/lucide": "catalog:", @@ -89,6 +90,7 @@ "@cloudflare/workers-types": "catalog:", "@harlan-zw/nuxt-cloudflare": "catalog:", "@harlan-zw/nuxt-dx": "catalog:", + "@harlan-zw/nuxt-sentry": "catalog:", "@nuxt/a11y": "catalog:", "@nuxt/eslint": "catalog:", "@nuxt/kit": "catalog:", diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a082a4cc..64bfc686 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -76,8 +76,10 @@ catalog: '@comark/nuxt': 0.6.2 '@harlan-zw/comark-content': 0.1.5 '@harlan-zw/nuxt-cf-jobs': 0.2.2 + '@harlan-zw/nuxt-checkin': 0.1.0 '@harlan-zw/nuxt-cloudflare': ^0.3.1 '@harlan-zw/nuxt-dx': 0.1.2 + '@harlan-zw/nuxt-sentry': 0.1.5 # Release placeholder: requires the public checks export. '@harlan-zw/nuxt-use-query': 0.2.1 '@harlan-zw/nuxt-wide-events': ^0.1.4 '@iconify-json/lucide': ^1.2.128 diff --git a/scripts/tools/daily-checkin-checks.mjs b/scripts/tools/daily-checkin-checks.mjs new file mode 100644 index 00000000..d5da64d0 --- /dev/null +++ b/scripts/tools/daily-checkin-checks.mjs @@ -0,0 +1,26 @@ +import { checkReport, defineCheck, runChecks, unavailable } from '@harlan-zw/nuxt-checkin/server' +import { defineSentryCheck } from '@harlan-zw/nuxt-sentry/checks' + +export function runDailyOperatorChecks({ now, token, deployment, healthReport, environment, request = fetch }) { + return runChecks([ + defineSentryCheck({ id: 'sentry.skilld', org: 'harlan-zw', project: 'skilld', environment }, request), + defineCheck({ + id: 'skilld.health-email', + run: () => deployment + ? checkReport(healthReport, { + now, + identity: { site: 'skilld.dev', environment: 'production', deployment }, + required: ['skilld.daily-health', 'skilld.daily-health-coverage'], + maxAgeMs: 36 * 60 * 60 * 1000, + }) + : unavailable('Deployed Worker identity is unavailable.'), + }), + ], { + now, + identity: { site: 'skilld.dev', environment: 'production', deployment: deployment ?? 'unknown' }, + required: ['sentry.skilld', 'skilld.health-email'], + credentials: token ? { sentry: token } : {}, + timeoutMs: 30_000, + totalTimeoutMs: 30_000, + }) +} diff --git a/scripts/tools/daily-checkin-data.mjs b/scripts/tools/daily-checkin-data.mjs index d86ac453..a18c4218 100644 --- a/scripts/tools/daily-checkin-data.mjs +++ b/scripts/tools/daily-checkin-data.mjs @@ -5,6 +5,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from import { homedir } from 'node:os' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' +import { runDailyOperatorChecks } from './daily-checkin-checks.mjs' import { approximateDeployedSha, buildWorkersQuery, @@ -445,8 +446,22 @@ function sentryToken() { // both. `truncatedAtLimit` reports when it did not. const SENTRY_ISSUE_LIMIT = 25 +const resolvedSentry = sentryToken() +const checkin = await runDailyOperatorChecks({ + now, + token: resolvedSentry?.token, + deployment: deploy.latest?.versionId, + healthReport: d1.healthEmail?.[0]?.checkin, + environment: process.env.SENTRY_ENVIRONMENT, +}) + const sentry = await (async () => { - const resolved = sentryToken() + const backlog = checkin.results.find(check => check.id === 'sentry.skilld')?.result + if (backlog?._tag === 'Pass') + return { _tag: 'available', newIssues: [], recurringIssues: [], truncatedAtLimit: false } + if (backlog?._tag !== 'Warn' && backlog?._tag !== 'Fail') + return { _tag: 'missing_observability', status: null, diagnostic: backlog?.reason ?? 'Sentry evidence is unavailable.' } + const resolved = resolvedSentry if (!resolved) { return { _tag: 'missing_observability', @@ -498,12 +513,13 @@ const doc = { d1, workers, sentry, + checkin, } console.log(JSON.stringify(doc, null, 2)) -const failedProbes = Object.entries({ git, deploy, ci, http, d1, workers, sentry }).filter(([name, value]) => - value?.error || (name === 'sentry' && value?._tag !== 'available'), +const failedProbes = Object.entries({ git, deploy, ci, http, d1, workers, sentry, checkin }).filter(([name, value]) => + value?.error || (name === 'sentry' && value?._tag !== 'available') || (name === 'checkin' && value?.coverage !== 'complete'), ) if (failedProbes.length) console.error(`WARN ${failedProbes.length} probes failed: ${failedProbes.map(([name]) => name).join(', ')}. Missing data is not health.`) diff --git a/scripts/tools/daily-checkin-observability.mjs b/scripts/tools/daily-checkin-observability.mjs index 7765abcf..81224882 100644 --- a/scripts/tools/daily-checkin-observability.mjs +++ b/scripts/tools/daily-checkin-observability.mjs @@ -194,6 +194,7 @@ export function parseHealthEmailRows(rows) { reasons: stringArray(summary.reasons, 'reasons'), warnings: stringArray(summary.warnings, 'warnings'), window, + checkin: summary.checkin, } }) } diff --git a/test/fixtures/checkin-registry.ts b/test/fixtures/checkin-registry.ts new file mode 100644 index 00000000..be35c6d3 --- /dev/null +++ b/test/fixtures/checkin-registry.ts @@ -0,0 +1,5 @@ +import dailyHealth from '../../layers/identity/server/checks/daily-health' +import dailyHealthCoverage from '../../layers/identity/server/checks/daily-health-coverage' + +// Nitro owns discovery. Vitest exercises the same definitions and public runner. +export default [dailyHealth, dailyHealthCoverage] diff --git a/test/unit/daily-checkin-checks.test.ts b/test/unit/daily-checkin-checks.test.ts new file mode 100644 index 00000000..b40e56f2 --- /dev/null +++ b/test/unit/daily-checkin-checks.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { defineCheck, pass, runChecks } from '@harlan-zw/nuxt-checkin/server' +import { describe, expect, it, vi } from 'vitest' +import { runDailyOperatorChecks } from '../../scripts/tools/daily-checkin-checks.mjs' + +const now = new Date('2026-09-14T22:00:00Z') +const identity = { site: 'skilld.dev', environment: 'production', deployment: 'worker-123' } + +async function healthReport() { + return runChecks([ + defineCheck({ id: 'skilld.daily-health', run: () => pass() }), + defineCheck({ id: 'skilld.daily-health-coverage', run: () => pass() }), + ], { now, identity }) +} + +describe('external daily check-in', () => { + it('combines the saved health report with the complete Sentry backlog', async () => { + const request = vi.fn().mockResolvedValue(new Response(JSON.stringify([{ id: '123', project: { slug: 'skilld' } }]), { status: 200 })) + const report = await runDailyOperatorChecks({ now, token: 'read-token', deployment: identity.deployment, healthReport: await healthReport(), environment: 'production', request }) + expect(report).toMatchObject({ severity: 'warn', coverage: 'complete' }) + expect(report.results[0]?.result).toMatchObject({ _tag: 'Warn', evidence: { issueIds: ['123'] } }) + const url = new URL(request.mock.calls[0]![0]) + expect(url.searchParams.get('environment')).toBe('production') + expect(url.searchParams.get('start')).toBe('1970-01-01T00:00:00.000Z') + }) + + it('reports missing credentials and a stale report as incomplete', async () => { + const request = vi.fn() + const report = await runDailyOperatorChecks({ now: new Date(now.getTime() + 48 * 60 * 60 * 1000), deployment: identity.deployment, healthReport: await healthReport(), request }) + expect(report.coverage).toBe('incomplete') + expect(report.results.map(check => check.result._tag)).toEqual(['Unavailable', 'Unavailable']) + expect(request).not.toHaveBeenCalled() + }) + + it('rejects a health report from a previous deployment', async () => { + const report = await runDailyOperatorChecks({ now, deployment: 'worker-new', healthReport: await healthReport(), request: vi.fn() }) + expect(report.results[1]?.result).toMatchObject({ _tag: 'Unavailable', reason: 'Check report identity does not match.' }) + }) +}) diff --git a/test/unit/daily-checkin-observability.test.ts b/test/unit/daily-checkin-observability.test.ts index d399698f..6aa880c2 100644 --- a/test/unit/daily-checkin-observability.test.ts +++ b/test/unit/daily-checkin-observability.test.ts @@ -254,6 +254,7 @@ describe('daily check-in observability', () => { }])).toEqual([{ reportDate: '2026-07-27', healthStatus: 'RED', + checkin: undefined, deliveryStatus: 'sent', recipient: 'operator@example.com', sentAt: 123, diff --git a/vitest.config.ts b/vitest.config.ts index 686ced0b..f2fe3314 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,4 +1,5 @@ import { availableParallelism } from 'node:os' +import { fileURLToPath } from 'node:url' import { defineVitestConfig } from '@nuxt/test-utils/config' import { configDefaults } from 'vitest/config' @@ -15,6 +16,7 @@ import { configDefaults } from 'vitest/config' const MAX_WORKERS = Math.max(2, Math.min(6, availableParallelism() - 2)) export default defineVitestConfig({ + resolve: { alias: { '#checkin/checks': fileURLToPath(new URL('./test/fixtures/checkin-registry.ts', import.meta.url)) } }, test: { globals: true, environment: 'nuxt', From 58d15b4cbba64d45754b418149f15620ab34f95b Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Mon, 14 Sep 2026 14:49:20 +1000 Subject: [PATCH 02/11] refactor(ops): combine scheduled Sentry and daily check-in --- .claude/skills/daily-checkin/SKILL.md | 6 +++++- .github/routines.yml | 7 ------- docs/shared-checkin.md | 3 ++- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.claude/skills/daily-checkin/SKILL.md b/.claude/skills/daily-checkin/SKILL.md index c83b0899..5e116740 100644 --- a/.claude/skills/daily-checkin/SKILL.md +++ b/.claude/skills/daily-checkin/SKILL.md @@ -12,13 +12,17 @@ Produce one read-only morning report that answers: what changed, what broke, wha 1. Run `node scripts/tools/daily-checkin-data.mjs --save` from the repo root. Preserve every probe error as a finding. The command archives raw evidence in `docs/ops/checkins/YYYY-MM-DD.json`; same-day reruns use a timestamped sibling and do not move the next baseline. 2. Read the newest prior JSON archive and `docs/ops/triage-ledger.md`. Compare fingerprints and rates, not only totals. Treat a missing prior key as a new probe with no baseline. 3. Verify these gates first: + - Read `checkin.severity`, `checkin.coverage`, and every result. Incomplete coverage cannot support GREEN. + - `sentry.skilld` covers the retained unresolved backlog. Missing credentials or partial pagination remain findings. + - `skilld.health-email` verifies the saved report against the current Worker version and a 36-hour age limit. + - If backlog IDs lack archived triage details, fetch their details read-only and append them to the archive. - Any stable non-200 `skilld.dev` front door response is RED. A retry flap is a note. - Read every `ci.workflows` state. `failure` means the gate is broken. `pending` after a prior failure must be followed until complete. `missing` is an observability gap. Never infer overall CI health from only the deploy workflow. - Any `workers.nonOk`, new Sentry issue, failed digest, failed job, stale reserved job, stale scheduled task, or missing expected table needs an explicit verdict. - `d1.migrations.localHead !== d1.migrations.prodHead` is migration drift. Never assume deployment applied D1 migrations. - `d1.healthEmail` must show one sent operator report per Melbourne date after the feature is deployed. Missing, failed, or stuck `sending` rows are findings. Always report its archived `reasons`; the color alone is insufficient. - Use failed-job `first_failed_at` and `last_failed_at`, plus `d1.recentJobBatches` and `d1.registryMaintenance`, to distinguish an active incident from a recovered burst. Do not call a window clean because the latest batch passed, or active because an older batch failed. - - Sentry issues must include their archived permalink and culprit in the proposed action. Do not require a second unarchived API read for basic triage. + - Sentry issues must include their archived permalink and culprit in the proposed action. Archive any extra detail reads. - `d1.inventory.broken_repos` is known cumulative inventory. Review `d1.pipeline.newly_broken_repos_total`, but gate health on `d1.pipeline.newly_broken_repos_impacted`. A source removal is impacting when it still backs a skill or appears in a star, subscription, collection, or install event. - AI cost is only the recorded batch estimate. Unmeasured services are unknown, not $0. - X API is pay-per-use at $0.005 per post read, with no included allowance and a $20/month target. Accepted 2026-08-31 at a projected $18.15/month, which raised the target from $10. Gate on `d1.cost`: diff --git a/.github/routines.yml b/.github/routines.yml index a34768d7..2953ff92 100644 --- a/.github/routines.yml +++ b/.github/routines.yml @@ -8,13 +8,6 @@ # run log reads well, and each finding becomes its own issue and pull request. version: 1 routines: - - name: sentry-checkin - on: - schedule: - - cron: '20 6 * * *' - timezone: Australia/Sydney - mode: propose - enabled: true - name: daily-checkin on: schedule: diff --git a/docs/shared-checkin.md b/docs/shared-checkin.md index 5fda9d13..6b11b928 100644 --- a/docs/shared-checkin.md +++ b/docs/shared-checkin.md @@ -17,7 +17,8 @@ A known failure stays RED when another source is unavailable. Unavailable evidence can never produce GREEN. Detailed evidence stays in the existing private report storage and operator email. -Recipients, cron schedules, delivery claims, retry behavior, and pause gates stay unchanged. +Recipients, email schedules, delivery claims, retry behavior, and pause gates stay unchanged. +The agent keeps its 07:40 daily check-in and removes its separate 06:20 Sentry run. `send-digests` sends subscriber content, not system health emails. Its outcome already enters the daily summary. The embedding parity audit keeps its independent schedule and pruning policy. From 891e01356a8ed8f92e4271e835971b0a87e1af74 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Mon, 14 Sep 2026 16:51:14 +1000 Subject: [PATCH 03/11] fix(ops): validate report time after evidence collection --- scripts/tools/daily-checkin-checks.mjs | 4 ++-- test/unit/daily-checkin-checks.test.ts | 16 +++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/tools/daily-checkin-checks.mjs b/scripts/tools/daily-checkin-checks.mjs index d5da64d0..a2b3c9dd 100644 --- a/scripts/tools/daily-checkin-checks.mjs +++ b/scripts/tools/daily-checkin-checks.mjs @@ -1,14 +1,14 @@ import { checkReport, defineCheck, runChecks, unavailable } from '@harlan-zw/nuxt-checkin/server' import { defineSentryCheck } from '@harlan-zw/nuxt-sentry/checks' -export function runDailyOperatorChecks({ now, token, deployment, healthReport, environment, request = fetch }) { +export function runDailyOperatorChecks({ now, token, deployment, healthReport, environment, clock = () => new Date(), request = fetch }) { return runChecks([ defineSentryCheck({ id: 'sentry.skilld', org: 'harlan-zw', project: 'skilld', environment }, request), defineCheck({ id: 'skilld.health-email', run: () => deployment ? checkReport(healthReport, { - now, + now: clock(), identity: { site: 'skilld.dev', environment: 'production', deployment }, required: ['skilld.daily-health', 'skilld.daily-health-coverage'], maxAgeMs: 36 * 60 * 60 * 1000, diff --git a/test/unit/daily-checkin-checks.test.ts b/test/unit/daily-checkin-checks.test.ts index b40e56f2..5f337152 100644 --- a/test/unit/daily-checkin-checks.test.ts +++ b/test/unit/daily-checkin-checks.test.ts @@ -6,17 +6,17 @@ import { runDailyOperatorChecks } from '../../scripts/tools/daily-checkin-checks const now = new Date('2026-09-14T22:00:00Z') const identity = { site: 'skilld.dev', environment: 'production', deployment: 'worker-123' } -async function healthReport() { +async function healthReport(observedAt = now) { return runChecks([ defineCheck({ id: 'skilld.daily-health', run: () => pass() }), defineCheck({ id: 'skilld.daily-health-coverage', run: () => pass() }), - ], { now, identity }) + ], { now: observedAt, identity }) } describe('external daily check-in', () => { it('combines the saved health report with the complete Sentry backlog', async () => { const request = vi.fn().mockResolvedValue(new Response(JSON.stringify([{ id: '123', project: { slug: 'skilld' } }]), { status: 200 })) - const report = await runDailyOperatorChecks({ now, token: 'read-token', deployment: identity.deployment, healthReport: await healthReport(), environment: 'production', request }) + const report = await runDailyOperatorChecks({ clock: () => now, now, token: 'read-token', deployment: identity.deployment, healthReport: await healthReport(), environment: 'production', request }) expect(report).toMatchObject({ severity: 'warn', coverage: 'complete' }) expect(report.results[0]?.result).toMatchObject({ _tag: 'Warn', evidence: { issueIds: ['123'] } }) const url = new URL(request.mock.calls[0]![0]) @@ -26,14 +26,20 @@ describe('external daily check-in', () => { it('reports missing credentials and a stale report as incomplete', async () => { const request = vi.fn() - const report = await runDailyOperatorChecks({ now: new Date(now.getTime() + 48 * 60 * 60 * 1000), deployment: identity.deployment, healthReport: await healthReport(), request }) + const report = await runDailyOperatorChecks({ clock: () => new Date(now.getTime() + 48 * 60 * 60 * 1000), now: new Date(now.getTime() + 48 * 60 * 60 * 1000), deployment: identity.deployment, healthReport: await healthReport(), request }) expect(report.coverage).toBe('incomplete') expect(report.results.map(check => check.result._tag)).toEqual(['Unavailable', 'Unavailable']) expect(request).not.toHaveBeenCalled() }) + it('accepts a report collected after the external run started', async () => { + const observedAt = new Date(now.getTime() + 5000) + const report = await runDailyOperatorChecks({ now, clock: () => new Date(observedAt.getTime() + 1000), deployment: identity.deployment, healthReport: await healthReport(observedAt), request: vi.fn() }) + expect(report.results[1]?.result._tag).toBe('Pass') + }) + it('rejects a health report from a previous deployment', async () => { - const report = await runDailyOperatorChecks({ now, deployment: 'worker-new', healthReport: await healthReport(), request: vi.fn() }) + const report = await runDailyOperatorChecks({ clock: () => now, now, deployment: 'worker-new', healthReport: await healthReport(), request: vi.fn() }) expect(report.results[1]?.result).toMatchObject({ _tag: 'Unavailable', reason: 'Check report identity does not match.' }) }) }) From 4e1027f0be788940f753fc9afedb473d38041a94 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Tue, 15 Sep 2026 00:03:24 +1000 Subject: [PATCH 04/11] chore(checkin): install published integration packages --- docs/shared-checkin.md | 8 +-- pnpm-lock.yaml | 104 ++++++++++++++++++++++++++---- pnpm-workspace.yaml | 8 ++- scripts/lib/d1-schema-verifier.ts | 8 +-- 4 files changed, 100 insertions(+), 28 deletions(-) diff --git a/docs/shared-checkin.md b/docs/shared-checkin.md index 6b11b928..7f92146d 100644 --- a/docs/shared-checkin.md +++ b/docs/shared-checkin.md @@ -26,10 +26,8 @@ Its recorded scheduled-run outcome already enters the daily summary. ## Release -This draft requires the unpublished `@harlan-zw/nuxt-checkin@0.1.0` release from harlan-nuxt PR 143. -After release, confirm the version and run `pnpm install` to update the lockfile. -The draft deliberately leaves the registry lockfile unchanged. -Local checks use the built package tarball, without committed file dependencies. +The check-in uses published `@harlan-zw/nuxt-checkin@0.1.0` and `@harlan-zw/nuxt-sentry@0.1.5`. +The lockfile resolves registry packages. ## External daily check-in @@ -44,5 +42,3 @@ The existing detailed Sentry probe runs only when the shared check finds unresol It keeps recurrence and impact details for triage. That extra request remains an optimization opportunity. Live credentials and the environment mapping remain unverified. -The draft uses `nuxt-sentry@0.1.5` as a release placeholder for the public checks export. -Confirm its actual release version before updating the lockfile. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20a29aaf..ee621612 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,14 +19,20 @@ catalogs: specifier: 0.1.5 version: 0.1.5 '@harlan-zw/nuxt-cf-jobs': - specifier: 0.2.2 - version: 0.2.2 + specifier: 0.2.3 + version: 0.2.3 + '@harlan-zw/nuxt-checkin': + specifier: 0.1.0 + version: 0.1.0 '@harlan-zw/nuxt-cloudflare': - specifier: ^0.3.1 - version: 0.3.1 + specifier: 0.4.1 + version: 0.4.1 '@harlan-zw/nuxt-dx': specifier: 0.1.2 version: 0.1.2 + '@harlan-zw/nuxt-sentry': + specifier: 0.1.5 + version: 0.1.5 '@harlan-zw/nuxt-use-query': specifier: 0.2.1 version: 0.2.1 @@ -212,7 +218,10 @@ importers: version: 0.1.5(98d5e2cee3315275cd7393fa02452bc5) '@harlan-zw/nuxt-cf-jobs': specifier: 'catalog:' - version: 0.2.2(eb08aac2723832fea99bb083d3e5de66) + version: 0.2.3(7d0ed70cf4c00b5041e45380a979dbc5) + '@harlan-zw/nuxt-checkin': + specifier: 'catalog:' + version: 0.1.0(e385fb48e32147193c05e7ca84503583) '@harlan-zw/nuxt-use-query': specifier: 'catalog:' version: 0.2.1(c8e02c8c74d77c0aa02a701b775491e2) @@ -327,10 +336,13 @@ importers: version: 5.20260901.1 '@harlan-zw/nuxt-cloudflare': specifier: 'catalog:' - version: 0.3.1(de4565feb4d455d9556f7b5b61faeaa1) + version: 0.4.1(86f60b01955541ac2ab799ea25f82069) '@harlan-zw/nuxt-dx': specifier: 'catalog:' version: 0.1.2(530d2fb91dd879c8eaebd3f75fc3ff20) + '@harlan-zw/nuxt-sentry': + specifier: 'catalog:' + version: 0.1.5(b92f4956c1d351dde2153cf2bca45bda) '@nuxt/a11y': specifier: 'catalog:' version: 1.0.0-alpha.1(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)))(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)) @@ -1495,25 +1507,38 @@ packages: nuxt: '>=4.5.0 <6.0.0' vue: '>=3.5.0 <4.0.0' - '@harlan-zw/nuxt-cf-jobs@0.2.2': - resolution: {integrity: sha512-LfVUNndp43Uzswh5lzKJ3JoAQTcuxCFhGqu/aSBfHzuenemxbpTzMsruI0NmMU8CnIQXbuBifbMLIyTLw+a+Yw==} + '@harlan-zw/nuxt-cf-jobs@0.2.3': + resolution: {integrity: sha512-fh9B7OyagbTuqNnI24XY9IfjrK+D/ypOUTLpHGj9U70H0lzSIXBbk45AbpE5/zihiWK0iHWeg3FwPLWOQG7s+w==} engines: {node: '>=22.12.0'} hasBin: true peerDependencies: + '@harlan-zw/nuxt-checkin': ^0.1.0 '@sentry/cloudflare': ^10.70.0 nuxt: '>=4.5.0 <6.0.0' vite: ^8.0.0 peerDependenciesMeta: + '@harlan-zw/nuxt-checkin': + optional: true '@sentry/cloudflare': optional: true - '@harlan-zw/nuxt-cloudflare@0.3.1': - resolution: {integrity: sha512-mNqNHX8Zt0Bwi6U6apJJf+u1RioDLeWUrtA8l/C7ItIUSz/twGW+Xnn2lyXr3IsjUUPl2tiVfmV5ixMbUpx4pA==} + '@harlan-zw/nuxt-checkin@0.1.0': + resolution: {integrity: sha512-APUDTVzj2jRs1qDj9l39jtXOLJzKXVd2UnnW4F2LI9QlbXHAOKM7VYe3Oj3Orwmw1Vbxp6GMwujLeVhv1O468A==} + engines: {node: '>=22.12.0'} + peerDependencies: + nuxt: '>=4.5.0 <6.0.0' + + '@harlan-zw/nuxt-cloudflare@0.4.1': + resolution: {integrity: sha512-fosZ5OB73Kq5nOkwTfHa6jiWdn+1ka2s6TTbTdjr3Z5RwKwqj0Z3vtmfED5iqOeqZxp2nOf2jYzJSJWKAHCdkA==} engines: {node: '>=22.12.0'} hasBin: true peerDependencies: + '@harlan-zw/nuxt-checkin': ^0.1.0 nuxt: '>=4.5.0 <6.0.0' wrangler: '>=4.120.0 <5.0.0' + peerDependenciesMeta: + '@harlan-zw/nuxt-checkin': + optional: true '@harlan-zw/nuxt-dx@0.1.2': resolution: {integrity: sha512-Py9m04a2TFwEbUCRtrjeSVp9FcEl6fB4X23zHRUktmdKH/xS58G5MCg0czbrH1lEWXfp8l/0WjUMnc5smc7COQ==} @@ -1523,6 +1548,20 @@ packages: nuxt: '>=4.5.0 <6.0.0' vite: ^8.0.0 + '@harlan-zw/nuxt-sentry@0.1.5': + resolution: {integrity: sha512-0FR4/JuapsrP98UvkJKgii98w7HKljYyPw26Vel37YFBkeGLcldLRad/egQGkyFPBZ+G87j3UkbRBAfwOni57Q==} + engines: {node: '>=22.12.0'} + peerDependencies: + '@harlan-zw/nuxt-checkin': ^0.1.0 + '@sentry/cloudflare': ^10.70.0 + '@sentry/nuxt': ^10.70.0 + nuxt: '>=4.5.0 <6.0.0' + peerDependenciesMeta: + '@harlan-zw/nuxt-checkin': + optional: true + '@sentry/cloudflare': + optional: true + '@harlan-zw/nuxt-use-query@0.2.1': resolution: {integrity: sha512-Tn7f8VPbxlNM268P6dDJph7iwM53SKQK4J2ncz4HQNAAWFr56QO/VR0/Si6aCjxAZv16922yaGvBku1IaG83pw==} engines: {node: '>=22.12.0'} @@ -8063,6 +8102,11 @@ packages: type-level-regexp@0.1.17: resolution: {integrity: sha512-wTk4DH3cxwk196uGLK/E9pE45aLfeKJacKmcEgEOA/q5dnPGNxXt0cfYdFxb57L+sEpf1oJH4Dnx/pnRcku9jg==} + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -9687,9 +9731,9 @@ snapshots: - shiki - unplugin - '@harlan-zw/nuxt-cf-jobs@0.2.2(eb08aac2723832fea99bb083d3e5de66)': + '@harlan-zw/nuxt-cf-jobs@0.2.3(7d0ed70cf4c00b5041e45380a979dbc5)': dependencies: - '@harlan-zw/nuxt-cloudflare': 0.3.1(de4565feb4d455d9556f7b5b61faeaa1) + '@harlan-zw/nuxt-cloudflare': 0.4.1(86f60b01955541ac2ab799ea25f82069) '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) '@nuxt/schema': 4.5.2 citty: 0.2.2 @@ -9697,6 +9741,7 @@ snapshots: nuxt: 4.5.2(@babel/plugin-proposal-decorators@8.0.2)(@babel/plugin-syntax-jsx@7.29.7)(@babel/plugin-syntax-typescript@7.29.7)(@oxc-project/types@0.144.0)(@types/node@26.4.0)(@vue/compiler-sfc@3.5.42)(better-sqlite3@13.0.3)(commander@14.0.3)(db0@0.3.4(better-sqlite3@13.0.3))(esbuild@0.28.2)(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.4)(rollup@4.62.4))(rollup@4.62.4)(srvx@0.12.5)(supports-color@10.2.2)(terser@5.50.0)(tsx@4.23.13)(typescript@6.0.3)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0) vite: 8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0) optionalDependencies: + '@harlan-zw/nuxt-checkin': 0.1.0(e385fb48e32147193c05e7ca84503583) '@sentry/cloudflare': 10.73.0(@cloudflare/workers-types@5.20260901.1)(wrangler@4.127.1(@cloudflare/workers-types@5.20260901.1)) transitivePeerDependencies: - '@aws-sdk/client-rds-data' @@ -9784,7 +9829,20 @@ snapshots: - xml2js - zod - '@harlan-zw/nuxt-cloudflare@0.3.1(de4565feb4d455d9556f7b5b61faeaa1)': + '@harlan-zw/nuxt-checkin@0.1.0(e385fb48e32147193c05e7ca84503583)': + dependencies: + '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) + nuxt: 4.5.2(@babel/plugin-proposal-decorators@8.0.2)(@babel/plugin-syntax-jsx@7.29.7)(@babel/plugin-syntax-typescript@7.29.7)(@oxc-project/types@0.144.0)(@types/node@26.4.0)(@vue/compiler-sfc@3.5.42)(better-sqlite3@13.0.3)(commander@14.0.3)(db0@0.3.4(better-sqlite3@13.0.3))(esbuild@0.28.2)(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.4)(rollup@4.62.4))(rollup@4.62.4)(srvx@0.12.5)(supports-color@10.2.2)(terser@5.50.0)(tsx@4.23.13)(typescript@6.0.3)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0) + pathe: 2.0.3 + typescript: 5.9.3 + transitivePeerDependencies: + - magic-string + - magicast + - oxc-parser + - rolldown + - unplugin + + '@harlan-zw/nuxt-cloudflare@0.4.1(86f60b01955541ac2ab799ea25f82069)': dependencies: '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) citty: 0.2.2 @@ -9796,6 +9854,8 @@ snapshots: pathe: 2.0.3 unstorage: 1.17.5(db0@0.3.4(better-sqlite3@13.0.3))(ioredis@5.11.1(supports-color@10.2.2)) wrangler: 4.127.1(@cloudflare/workers-types@5.20260901.1) + optionalDependencies: + '@harlan-zw/nuxt-checkin': 0.1.0(e385fb48e32147193c05e7ca84503583) transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -9859,6 +9919,22 @@ snapshots: - unplugin - vue + '@harlan-zw/nuxt-sentry@0.1.5(b92f4956c1d351dde2153cf2bca45bda)': + dependencies: + '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) + '@sentry/nuxt': 10.73.0(433b670f0f22b940727aa0d678e7208d) + nuxt: 4.5.2(@babel/plugin-proposal-decorators@8.0.2)(@babel/plugin-syntax-jsx@7.29.7)(@babel/plugin-syntax-typescript@7.29.7)(@oxc-project/types@0.144.0)(@types/node@26.4.0)(@vue/compiler-sfc@3.5.42)(better-sqlite3@13.0.3)(commander@14.0.3)(db0@0.3.4(better-sqlite3@13.0.3))(esbuild@0.28.2)(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.4)(rollup@4.62.4))(rollup@4.62.4)(srvx@0.12.5)(supports-color@10.2.2)(terser@5.50.0)(tsx@4.23.13)(typescript@6.0.3)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0) + pathe: 2.0.3 + optionalDependencies: + '@harlan-zw/nuxt-checkin': 0.1.0(e385fb48e32147193c05e7ca84503583) + '@sentry/cloudflare': 10.73.0(@cloudflare/workers-types@5.20260901.1)(wrangler@4.127.1(@cloudflare/workers-types@5.20260901.1)) + transitivePeerDependencies: + - magic-string + - magicast + - oxc-parser + - rolldown + - unplugin + '@harlan-zw/nuxt-use-query@0.2.1(c8e02c8c74d77c0aa02a701b775491e2)': dependencies: '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) @@ -17464,6 +17540,8 @@ snapshots: type-level-regexp@0.1.17: {} + typescript@5.9.3: {} + typescript@6.0.3: {} uc.micro@2.1.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 64bfc686..ffc197c4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,6 @@ minimumReleaseAgeExclude: + - '@harlan-zw/nuxt-sentry@0.1.5' + - '@harlan-zw/nuxt-checkin' - '@comark/nuxt' - '@comark/vue' - comark @@ -75,11 +77,11 @@ catalog: '@cloudflare/workers-types': ^5.20260901.1 '@comark/nuxt': 0.6.2 '@harlan-zw/comark-content': 0.1.5 - '@harlan-zw/nuxt-cf-jobs': 0.2.2 + '@harlan-zw/nuxt-cf-jobs': 0.2.3 '@harlan-zw/nuxt-checkin': 0.1.0 - '@harlan-zw/nuxt-cloudflare': ^0.3.1 + '@harlan-zw/nuxt-cloudflare': 0.4.1 '@harlan-zw/nuxt-dx': 0.1.2 - '@harlan-zw/nuxt-sentry': 0.1.5 # Release placeholder: requires the public checks export. + '@harlan-zw/nuxt-sentry': 0.1.5 '@harlan-zw/nuxt-use-query': 0.2.1 '@harlan-zw/nuxt-wide-events': ^0.1.4 '@iconify-json/lucide': ^1.2.128 diff --git a/scripts/lib/d1-schema-verifier.ts b/scripts/lib/d1-schema-verifier.ts index 5dace27c..0d4c9015 100644 --- a/scripts/lib/d1-schema-verifier.ts +++ b/scripts/lib/d1-schema-verifier.ts @@ -129,12 +129,8 @@ export type CfJobsPackageVerificationResult | { _tag: 'fail', issues: PackageContractIssue[] } export const CF_JOBS_SCHEMA_CONTRACT = { - // Bumped 0.0.5 -> 0.2.2 after checking the distributed schema rather than - // just the version string: all five indexes below appear in 0.2.2's - // `dist/runtime/server/d1.js` with byte-identical SQL, including the - // partial-index predicates. The jobs, job_batches and failed_jobs columns - // in 0.2.2 also match the migration replay exactly, so no migration is due. - version: '0.2.2', + // The 0.2.3 distributed schema preserves these five indexes and predicates. + version: '0.2.3', indexes: [ { name: 'idx_jobs_dispatchable', From df06e9e38ba351e970c20bd22664bfe112d6da98 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Tue, 15 Sep 2026 01:06:21 +1000 Subject: [PATCH 05/11] refactor(checkin): register checks with the shared CLI --- .claude/skills/daily-checkin/SKILL.md | 9 +- checks/_helpers/collectors.mjs | 301 ++++++++++ .../_helpers/observability.mjs | 0 checks/_helpers/policy.mjs | 33 ++ checks/_helpers/sentry.ts | 11 + checks/external/ci.ts | 10 + checks/external/database.ts | 10 + checks/external/deploy.ts | 10 + checks/external/git.ts | 9 + checks/external/health-email.ts | 18 + checks/external/home.ts | 8 + checks/external/sentry-details.ts | 34 ++ checks/external/sentry.ts | 7 + checks/external/skills.ts | 8 + checks/external/workers.ts | 10 + docs/shared-checkin.md | 10 + nuxt.config.ts | 2 + package.json | 3 +- scripts/tools/daily-checkin-checks.mjs | 26 - scripts/tools/daily-checkin-data.mjs | 543 ------------------ scripts/tools/daily-checkin-process.mjs | 62 -- shared/checkin-external.ts | 23 + test/unit/daily-checkin-checks.test.ts | 19 +- test/unit/daily-checkin-observability.test.ts | 2 +- test/unit/daily-checkin-process.test.ts | 78 --- test/unit/external-checkin-collectors.test.ts | 68 +++ test/unit/external-checkin-policy.test.ts | 23 + 27 files changed, 621 insertions(+), 716 deletions(-) create mode 100644 checks/_helpers/collectors.mjs rename scripts/tools/daily-checkin-observability.mjs => checks/_helpers/observability.mjs (100%) create mode 100644 checks/_helpers/policy.mjs create mode 100644 checks/_helpers/sentry.ts create mode 100644 checks/external/ci.ts create mode 100644 checks/external/database.ts create mode 100644 checks/external/deploy.ts create mode 100644 checks/external/git.ts create mode 100644 checks/external/health-email.ts create mode 100644 checks/external/home.ts create mode 100644 checks/external/sentry-details.ts create mode 100644 checks/external/sentry.ts create mode 100644 checks/external/skills.ts create mode 100644 checks/external/workers.ts delete mode 100644 scripts/tools/daily-checkin-checks.mjs delete mode 100644 scripts/tools/daily-checkin-data.mjs delete mode 100644 scripts/tools/daily-checkin-process.mjs create mode 100644 shared/checkin-external.ts delete mode 100644 test/unit/daily-checkin-process.test.ts create mode 100644 test/unit/external-checkin-collectors.test.ts create mode 100644 test/unit/external-checkin-policy.test.ts diff --git a/.claude/skills/daily-checkin/SKILL.md b/.claude/skills/daily-checkin/SKILL.md index 5e116740..ffd9224f 100644 --- a/.claude/skills/daily-checkin/SKILL.md +++ b/.claude/skills/daily-checkin/SKILL.md @@ -9,10 +9,13 @@ Produce one read-only morning report that answers: what changed, what broke, wha ## Workflow -1. Run `node scripts/tools/daily-checkin-data.mjs --save` from the repo root. Preserve every probe error as a finding. The command archives raw evidence in `docs/ops/checkins/YYYY-MM-DD.json`; same-day reruns use a timestamped sibling and do not move the next baseline. -2. Read the newest prior JSON archive and `docs/ops/triage-ledger.md`. Compare fingerprints and rates, not only totals. Treat a missing prior key as a new probe with no baseline. +1. Run `pnpm checkin --save` from the repo root. Preserve every probe error as a finding. The command archives raw evidence in `docs/ops/checkins/YYYY-MM-DD.json`; same-day reruns use a timestamped sibling and do not move the next baseline. +2. Read check evidence from `results`: `skilld.git`, `skilld.deploy`, `skilld.ci`, `skilld.database`, and `skilld.workers`. + Map the previous `git`, `deploy`, `ci`, `d1`, and `workers` fields to each result's `result.evidence`. + Read detailed Sentry evidence from `skilld.sentry-details`. Read HTTP results from `skilld.home` and `skilld.skills`. + Read the newest prior JSON archive and `docs/ops/triage-ledger.md`. Compare fingerprints and rates, not only totals. Treat a missing prior key as a new probe with no baseline. 3. Verify these gates first: - - Read `checkin.severity`, `checkin.coverage`, and every result. Incomplete coverage cannot support GREEN. + - Read `severity`, `coverage`, and every result. Incomplete coverage cannot support GREEN. - `sentry.skilld` covers the retained unresolved backlog. Missing credentials or partial pagination remain findings. - `skilld.health-email` verifies the saved report against the current Worker version and a 36-hour age limit. - If backlog IDs lack archived triage details, fetch their details read-only and append them to the archive. diff --git a/checks/_helpers/collectors.mjs b/checks/_helpers/collectors.mjs new file mode 100644 index 00000000..4c0a05a8 --- /dev/null +++ b/checks/_helpers/collectors.mjs @@ -0,0 +1,301 @@ +import { readFileSync } from 'node:fs' +import { readdir, readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { runCheckCommand } from '@harlan-zw/nuxt-checkin/external' +import { buildWorkersQuery, parseHealthEmailRows, parseWorkflowName, summarizeWorkflowRuns } from './observability.mjs' + +const resources = Object.fromEntries(['github-auth', 'production-ref', 'git', 'deploy', 'ci', 'd1', 'workers'].map(key => [key, {}])) + +function collect(context, key, load) { + return context.collect(resources[key], `skilld.${key}`, async signal => ({ value: await load({ ...context, signal }) })) +} + +async function run(context, command, args) { + const baseEnv = { ...context.env, NO_COLOR: '1' } + const env = command === 'git' || command === 'gh' + ? await collect(context, 'github-auth', async (context) => { + if (!(baseEnv.GITHUB_TOKEN || baseEnv.GH_TOKEN)) + return baseEnv + const probe = await runCheckCommand({ ...context, env: baseEnv }, 'gh', ['api', 'user']) + if (probe._tag === 'Ok') + return baseEnv + // A stale environment token overrides a valid keyring login. + const { GITHUB_TOKEN: _token, GH_TOKEN: _altToken, ...keyringEnv } = baseEnv + return keyringEnv + }) + : baseEnv + const result = await runCheckCommand({ ...context, env }, command, args, { maxBytes: 20 * 1024 * 1024 }) + if (result._tag === 'Err') + throw new Error(result.reason) + return result.stdout.trim() +} +async function commandJson(context, command, args) { + return JSON.parse(await run(context, command, args)) +} +export function collectProduction(context) { + return collect(context, 'production-ref', async (context) => { + await run(context, 'git', ['fetch', 'origin', 'main']) + return { ref: 'origin/main' } + }) +} +export function collectGit(context) { + return collect(context, 'git', async (context) => { + const production = await collectProduction(context) + const commits = (await run(context, 'git', ['log', `--since=${context.since.toISOString()}`, '--pretty=format:%H%x09%aI%x09%s'])).split('\n').filter(Boolean).map((line) => { + const [sha, authoredAt, ...subject] = line.split('\t') + return { sha, authoredAt, subject: subject.join('\t') } + }) + return { branch: await run(context, 'git', ['branch', '--show-current']), head: await run(context, 'git', ['rev-parse', 'HEAD']), dirtyFiles: (await run(context, 'git', ['status', '--short'])).split('\n').filter(Boolean), commitsSinceLastRun: commits, productionRef: production } + }) +} +export function collectDeploy(context) { + return collect(context, 'deploy', async (context) => { + const production = await collectProduction(context) + const deployments = await commandJson(context, join(context.rootDir, 'node_modules/.bin/wrangler'), ['deployments', 'list', '--json', '--config', 'wrangler.jsonc']) + const latest = [...deployments].sort((a, b) => String(b.created_on).localeCompare(String(a.created_on)))[0] + return { latest: latest ? { id: latest.id, createdOn: latest.created_on, versionId: latest.versions?.find(version => version.percentage === 100)?.version_id ?? latest.versions?.[0]?.version_id ?? null, message: latest.annotations?.['workers/message'] ?? null, approxDeployedSha: latest.created_on ? await run(context, 'git', ['rev-list', '-1', `--before=${latest.created_on}`, production.ref]) : null } : null } + }) +} +async function readMigrations(context, production) { + const migrationFiles = text => text.split('\n').map(path => path.slice('migrations/'.length)).filter(file => /^\d.*\.sql$/.test(file)).sort() + const productionFiles = migrationFiles(await run(context, 'git', ['ls-tree', '--name-only', production.ref, 'migrations/'])) + const current = migrationFiles(await run(context, 'git', ['ls-tree', '--name-only', 'HEAD', 'migrations/'])) + const working = (await readdir(join(context.rootDir, 'migrations'))).filter(file => /^\d.*\.sql$/.test(file)).sort() + return { localHead: productionFiles.at(-1) ?? null, uncommitted: working.filter(file => !current.includes(file)) } +} +export function collectCI(context) { + return collect(context, 'ci', async (context) => { + const directory = join(context.rootDir, '.github/workflows') + const names = (await Promise.all((await readdir(directory)).filter(file => /\.ya?ml$/.test(file)).map(async file => parseWorkflowName(await readFile(join(directory, file), 'utf8'))))).filter(Boolean).sort() + const fields = 'databaseId,workflowName,displayTitle,headSha,status,conclusion,createdAt,updatedAt,url' + const rows = [] + for (const name of names) { + let page = await commandJson(context, 'gh', ['run', 'list', '--workflow', name, '--limit', '10', '--json', fields]) + if (!page.some(row => row.status === 'completed' && row.conclusion !== 'skipped') && page.length === 10) + page = await commandJson(context, 'gh', ['run', 'list', '--workflow', name, '--limit', '100', '--json', fields]) + rows.push(...page) + } + return { workflows: summarizeWorkflowRuns(rows, names), recent: (await commandJson(context, 'gh', ['run', 'list', '--limit', '20', '--json', fields])).slice(0, 10) } + }) +} +function readDailyBudget(root) { + try { + const src = readFileSync(join(root, 'shared/server/x-ingest.ts'), 'utf8') + const match = src.match(/DAILY_DISCOVERY_READ_BUDGET\s*=\s*(\d+)/) + return match ? Number(match[1]) : null + } + catch (error) { + throw new Error('X discovery budget source is unavailable.', { cause: error }) + } +} +export function collectD1(context) { + return collect(context, 'd1', async (context) => { + const root = context.rootDir + const now = context.now + const sinceSec = Math.floor(context.since.getTime() / 1000) + const sinceMs = context.since.getTime() + const utcDay = now.toISOString().slice(0, 10) + async function d1Query(sql) { + const output = await commandJson(context, join(root, 'node_modules/.bin/wrangler'), ['d1', 'execute', 'DB', '--remote', '--json', '--config', 'wrangler.jsonc', '--command', sql]) + if (!output[0]?.success) + throw new Error('D1 query did not succeed.') + return output[0].results ?? [] + } + const tableRows = (await d1Query(`SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name`)) + const tables = new Set(tableRows.map(row => row.name)) + const has = table => tables.has(table) + /** + * Column-level guard. `has(table)` is not enough: a migration that adds a + * column ships in the code before it is applied to production, and a query + * naming the new column fails the whole d1 probe. That silently blanked the + * entire checkin the first time `skills_scanned_at` was referenced, which is + * the worst possible failure for the tool that reports failures. + */ + const columnCache = new Map() + const hasColumn = async (table, column) => { + if (!tables.has(table)) + return false + if (!columnCache.has(table)) { + columnCache.set(table, new Set((await d1Query(`SELECT name FROM pragma_table_info('${table}')`)).map(row => row.name))) + } + return columnCache.get(table).has(column) + } + const inventoryParts = [ + has('skills') ? `(SELECT COUNT(*) FROM skills) AS skills` : 'NULL AS skills', + has('repos') ? `(SELECT COUNT(*) FROM repos) AS repos` : 'NULL AS repos', + has('owners') ? `(SELECT COUNT(*) FROM owners) AS owners` : 'NULL AS owners', + has('users') ? `(SELECT COUNT(*) FROM users) AS users` : 'NULL AS users', + has('collections_v2') ? `(SELECT COUNT(*) FROM collections_v2 WHERE deleted_at IS NULL) AS collections` : 'NULL AS collections', + has('repos') ? `(SELECT COUNT(*) FROM repos WHERE broken_since IS NOT NULL) AS broken_repos` : 'NULL AS broken_repos', + ] + const activityParts = [ + has('skills') ? `(SELECT COUNT(*) FROM skills WHERE first_seen_at >= ${sinceSec}) AS new_skills` : 'NULL AS new_skills', + has('activity') ? `(SELECT COUNT(DISTINCT owner || '/' || repo) FROM activity WHERE occurred_at >= ${sinceSec}) AS changed_repos` : 'NULL AS changed_repos', + has('install_events') ? `(SELECT COUNT(*) FROM install_events WHERE occurred_at >= ${sinceMs}) AS install_events` : 'NULL AS install_events', + has('users') ? `(SELECT COUNT(*) FROM users WHERE created_at >= ${sinceSec}) AS new_users` : 'NULL AS new_users', + has('digest_runs') ? `(SELECT COUNT(*) FROM digest_runs WHERE status = 'sent' AND sent_at >= ${sinceSec}) AS digests_sent` : 'NULL AS digests_sent', + has('digest_runs') ? `(SELECT COUNT(*) FROM digest_runs WHERE status = 'failed' AND window_end >= ${sinceSec}) AS digests_failed` : 'NULL AS digests_failed', + ] + // A count alone gives the 3-repo AMBER gate nothing to repair or prune, so + // the identities behind `newly_broken_repos_impacted` are archived too: one + // row per impacted repo with the reasons it still backs. The filter is shared + // with the count so the number and the rows can never disagree. + const impactedBrokenReposGuarded = has('repos') && has('skills') && has('user_starred_repos') && has('skill_subscriptions') && has('collection_skills_v2') && has('activity') && has('install_events') + const impactedBrokenReposWhere = `r.broken_since >= ${sinceSec} AND (EXISTS (SELECT 1 FROM skills s WHERE s.owner = r.owner AND s.repo = r.repo) OR EXISTS (SELECT 1 FROM user_starred_repos usr WHERE usr.owner = r.owner AND usr.repo = r.repo) OR EXISTS (SELECT 1 FROM skill_subscriptions sub WHERE sub.owner = r.owner AND sub.repo = r.repo) OR EXISTS (SELECT 1 FROM collection_skills_v2 cs WHERE cs.owner = r.owner AND cs.repo = r.repo) OR EXISTS (SELECT 1 FROM activity a JOIN install_events ie ON ie.slug = a.owner || '/' || a.name WHERE a.owner = r.owner AND a.repo = r.repo))` + const pipelineParts = [ + has('repos') ? `(SELECT COUNT(*) FROM repos WHERE broken_since >= ${sinceSec}) AS newly_broken_repos_total` : 'NULL AS newly_broken_repos_total', + impactedBrokenReposGuarded ? `(SELECT COUNT(*) FROM repos r WHERE ${impactedBrokenReposWhere}) AS newly_broken_repos_impacted` : 'NULL AS newly_broken_repos_impacted', + has('skills') ? `(SELECT COUNT(*) FROM skills WHERE sync_status IS NOT NULL AND sync_status != 'ok' AND last_synced_at >= ${sinceSec}) AS skill_sync_failures` : 'NULL AS skill_sync_failures', + has('skill_dirty') ? `(SELECT COUNT(*) FROM skill_dirty WHERE queued_at < ${Math.floor(now.getTime() / 1000) - 3600}) AS stale_dirty_skills` : 'NULL AS stale_dirty_skills', + // `ai_batches` covers the Anthropic batch pipeline only, and embeddings never + // touch it, so these three read 0 while `HAIKU_GENERATION_PAUSED` holds. The + // embedding counters below are the ones that move; reading an AI-pipeline + // verdict off `ai_submitted` alone produced a wrong diagnosis on 2026-08-03. + has('ai_batches') ? `(SELECT COUNT(*) FROM ai_batches WHERE status = 'submitted') AS ai_submitted` : 'NULL AS ai_submitted', + has('ai_batches') ? `(SELECT COUNT(*) FROM ai_batches WHERE status = 'submitted' AND submitted_at < ${Math.floor(now.getTime() / 1000) - 10800}) AS ai_stuck` : 'NULL AS ai_stuck', + has('ai_batches') ? `(SELECT COUNT(*) FROM ai_batches WHERE status IN ('failed','expired') AND COALESCE(completed_at, submitted_at) >= ${sinceSec}) AS ai_failed` : 'NULL AS ai_failed', + has('embedding_attempts') ? `(SELECT COUNT(*) FROM embedding_attempts WHERE state = 'completed' AND started_at >= ${sinceSec}) AS embeddings_completed` : 'NULL AS embeddings_completed', + has('embedding_attempts') ? `(SELECT COUNT(*) FROM embedding_attempts WHERE state IN ('provider_failed','rejected','vector_succeeded_marker_failed') AND started_at >= ${sinceSec}) AS embeddings_failed` : 'NULL AS embeddings_failed', + // Bounded below at 7 days: `started` rows abandoned by a closed incident + // otherwise read as a live fault forever (6 rows from Jul 23/29 did exactly + // that on 2026-08-04). A row stuck past 7 days is history, not an incident. + has('embedding_attempts') ? `(SELECT COUNT(*) FROM embedding_attempts WHERE state = 'started' AND started_at < ${Math.floor(now.getTime() / 1000) - 3600} AND started_at >= ${Math.floor(now.getTime() / 1000) - 604800}) AS embeddings_stuck` : 'NULL AS embeddings_stuck', + has('failed_jobs') ? `(SELECT COUNT(*) FROM failed_jobs WHERE failed_at >= ${sinceSec}) AS failed_jobs` : 'NULL AS failed_jobs', + has('jobs') ? `(SELECT COUNT(*) FROM jobs WHERE reserved_at IS NOT NULL AND reserved_at < ${Math.floor(now.getTime() / 1000) - 900} AND completed_at IS NULL AND failed_at IS NULL) AS stale_reserved_jobs` : 'NULL AS stale_reserved_jobs', + ] + /** + * Turn raw X read counts into the number that matters: dollars, and whether + * the month is on track. Reads are $0.005 each with no included allowance. + * + * The budget is read out of shared/server/x-ingest.ts rather than copied. + * This script is plain node and cannot import the app's module graph, and a + * hand-copied constant had already gone stale once, reporting 22 while the + * real budget was 400 and so claiming the ingest was massively over budget. + */ + function withXSpend(costRow) { + const USD_PER_READ = 0.005 + const budgetTarget = readDailyBudget(root) + const row = costRow ?? {} + const discoveryToday = Number(row.x_discovery_reads_today ?? 0) + const hot = Number(row.x_hot_posts ?? 0) + // Refresh only pays for a hot post again when its window crosses midnight. + // Treating every hot post as one more charge is the pessimistic bound. + const projectedDaily = discoveryToday + hot + return { + ...row, + x_budget_target: budgetTarget, + x_budget_used_pct: budgetTarget > 0 ? Math.round((discoveryToday / budgetTarget) * 100) : null, + x_projected_daily_reads: projectedDaily, + x_projected_monthly_usd: Math.round(projectedDaily * 30 * USD_PER_READ * 100) / 100, + x_over_budget: discoveryToday > budgetTarget, + // The value question: reads bought per skill actually verified. Rising + // means the query is getting less precise or the stream is noisier. + x_reads_per_verified_skill: Number(row.x_verified_skills ?? 0) > 0 + ? Math.round(Number(row.x_discovery_reads_total ?? 0) / Number(row.x_verified_skills)) + : null, + // Budget spent without finishing the window means the stream is + // outrunning the ingest and the backlog grows daily. + x_falling_behind: discoveryToday >= budgetTarget, + } + } + const costParts = [ + has('ai_batch_costs') ? `(SELECT COALESCE(SUM(est_cost_usd), 0) FROM ai_batch_costs WHERE submitted_at >= ${sinceSec}) AS ai_cost_usd` : 'NULL AS ai_cost_usd', + // X is pay-per-use with no included allowance: every post read is a real + // $0.005 invoice line, so it belongs in the health report rather than only + // in task logs. `budget_spent` counts today only when `budget_day` matches; + // a stale day reads as 0, matching how the ingest resets it. + has('x_ingest_cursor') ? `(SELECT COALESCE(SUM(CASE WHEN budget_day = '${utcDay}' THEN budget_spent ELSE 0 END), 0) FROM x_ingest_cursor) AS x_discovery_reads_today` : 'NULL AS x_discovery_reads_today', + has('x_ingest_cursor') ? `(SELECT COALESCE(SUM(posts_read_total), 0) FROM x_ingest_cursor) AS x_discovery_reads_total` : 'NULL AS x_discovery_reads_total', + // Hot posts are refresh's remaining exposure: each can cost at most one + // more read, when its window crosses midnight UTC. + has('x_posts') ? `(SELECT COUNT(*) FROM x_posts WHERE refresh_tier = 'hot') AS x_hot_posts` : 'NULL AS x_hot_posts', + has('x_posts') ? `(SELECT COUNT(*) FROM x_posts WHERE first_seen_at >= ${sinceSec}) AS x_posts_24h` : 'NULL AS x_posts_24h', + // How stale the freshest ingested post is. Climbs when discovery lags. + has('x_posts') ? `(SELECT CAST((${Math.floor(now.getTime() / 1000)} - MAX(posted_at)) / 3600 AS INTEGER) FROM x_posts) AS x_newest_post_age_hours` : 'NULL AS x_newest_post_age_hours', + has('x_post_skills') ? `(SELECT COUNT(*) FROM x_post_skills) AS x_verified_skills` : 'NULL AS x_verified_skills', + has('x_post_skills') ? `(SELECT COUNT(*) FROM x_post_skills WHERE verified_at >= ${sinceSec}) AS x_verified_skills_24h` : 'NULL AS x_verified_skills_24h', + (await hasColumn('x_posts', 'skills_scanned_at')) ? `(SELECT COUNT(*) FROM x_posts WHERE skills_scanned_at IS NULL) AS x_posts_unscanned` : 'NULL AS x_posts_unscanned', + (await hasColumn('discovery_ledger', 'held_reason')) ? `(SELECT COUNT(*) FROM discovery_ledger WHERE status = 'pending' AND held_reason IS NULL) AS x_ledger_pending` : 'NULL AS x_ledger_pending', + (await hasColumn('discovery_ledger', 'held_reason')) ? `(SELECT COUNT(*) FROM discovery_ledger WHERE held_reason IS NOT NULL) AS x_ledger_held` : 'NULL AS x_ledger_held', + has('discovery_ledger') ? `(SELECT COUNT(*) FROM discovery_ledger WHERE status = 'submitted' AND submitted_at IS NOT NULL AND submitted_at < ${Math.floor(now.getTime() / 1000) - 21600}) AS x_ledger_stalled` : 'NULL AS x_ledger_stalled', + has('discovery_ledger') ? `(SELECT COUNT(*) FROM discovery_ledger WHERE status = 'indexed') AS x_ledger_indexed` : 'NULL AS x_ledger_indexed', + ] + const impactedBrokenRepos = impactedBrokenReposGuarded + ? (await d1Query(`SELECT r.owner, r.repo, rtrim((CASE WHEN EXISTS (SELECT 1 FROM skills s WHERE s.owner = r.owner AND s.repo = r.repo) THEN 'skill ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM user_starred_repos usr WHERE usr.owner = r.owner AND usr.repo = r.repo) THEN 'star ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM skill_subscriptions sub WHERE sub.owner = r.owner AND sub.repo = r.repo) THEN 'subscription ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM collection_skills_v2 cs WHERE cs.owner = r.owner AND cs.repo = r.repo) THEN 'collection ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM activity a JOIN install_events ie ON ie.slug = a.owner || '/' || a.name WHERE a.owner = r.owner AND a.repo = r.repo) THEN 'install ' ELSE '' END)) AS reason FROM repos r WHERE ${impactedBrokenReposWhere} ORDER BY r.broken_since DESC`)) + : null + const syncJobs = has('sync_jobs') + ? (await d1Query(`SELECT name, cron, stale_after_seconds, last_run_at, last_status, last_error FROM sync_jobs WHERE enabled = 1 ORDER BY name`)) + : null + const failedJobFingerprints = has('failed_jobs') + ? (await d1Query(`SELECT queue, job_type, substr(exception, 1, 160) exception, COUNT(*) count, MIN(failed_at) first_failed_at, MAX(failed_at) last_failed_at FROM failed_jobs WHERE failed_at >= ${sinceSec} GROUP BY queue, job_type, substr(exception, 1, 160) ORDER BY count DESC LIMIT 10`)) + : null + const healthEmail = has('daily_health_checks') + ? parseHealthEmailRows((await d1Query(`SELECT report_date, health_status, delivery_status, recipient, sent_at, error, summary_json FROM daily_health_checks ORDER BY report_date DESC LIMIT 2`))) + : null + const recentJobBatches = has('job_batches') + ? (await d1Query(`SELECT id, name, total_jobs, pending_jobs, failed_jobs, created_at, updated_at, finished_at FROM job_batches ORDER BY updated_at DESC LIMIT 10`)) + : null + const registryMaintenance = has('registry_maintenance') + ? (await d1Query(`SELECT name, status, batch_id, updated_at, last_error FROM registry_maintenance ORDER BY updated_at DESC`)) + : null + const prodMigrationHead = has('d1_migrations') + ? (await d1Query(`SELECT MAX(name) name FROM d1_migrations`))[0]?.name ?? null + : null + // Production can be checked from a feature worktree. Compare D1 with the + // production branch, while keeping worktree-only migrations visible. + // + // A failed `git fetch` leaves `migrations` as `{ error }` with neither + // `localHead` nor `prodHead`, so the drift gate cannot misread a fetch + // failure as drift. A 2026-09-03 silently failed fetch archived drift that + // did not exist. + const migrations = await collectProduction(context).then(async production => ({ ...(await readMigrations(context, production)), prodHead: prodMigrationHead })).catch(error => ({ error: error.message })) + return { + tables: [...tables], + inventory: (await d1Query(`SELECT ${inventoryParts.join(', ')}`))[0], + activity: (await d1Query(`SELECT ${activityParts.join(', ')}`))[0], + pipeline: { + ...(await d1Query(`SELECT ${pipelineParts.join(', ')}`))[0], + newly_broken_repos_impacted_identities: impactedBrokenRepos, + }, + cost: withXSpend((await d1Query(`SELECT ${costParts.join(', ')}`))[0]), + syncJobs, + failedJobFingerprints, + healthEmail, + recentJobBatches, + registryMaintenance, + migrations, + missingExpectedTables: [ + 'skills', + 'repos', + 'owners', + 'users', + 'activity', + 'install_events', + 'sync_jobs', + 'jobs', + 'failed_jobs', + 'daily_health_checks', + ].filter(table => !has(table)), + } + }) +} +export function collectWorkers(context) { + return collect(context, 'workers', async (context) => { + const token = context.env.CLOUDFLARE_USAGE_TOKEN || context.env.CLOUDFLARE_API_TOKEN || context.env.CF_API_TOKEN || (await commandJson(context, join(context.rootDir, 'node_modules/.bin/wrangler'), ['auth', 'token', '--json'])).token + if (!token) + throw new Error('Cloudflare token is unavailable.') + const response = await fetch('https://api.cloudflare.com/client/v4/graphql', { method: 'POST', redirect: 'error', signal: context.signal, headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ query: buildWorkersQuery(context.since.toISOString(), context.now.toISOString()) }) }) + const body = await response.json() + if (!response.ok || body.errors?.length) + throw new Error(`Cloudflare GraphQL returned HTTP ${response.status}.`) + const rows = body.data?.viewer?.accounts?.[0]?.workersInvocationsAdaptive + if (!Array.isArray(rows)) + throw new Error('Cloudflare Worker evidence is unavailable.') + const outcomes = {} + for (const row of rows) { + if (row.dimensions.scriptName === 'skilld-dev') + outcomes[row.dimensions.status] = (outcomes[row.dimensions.status] ?? 0) + row.sum.requests + } + return { outcomes, nonOk: Object.entries(outcomes).filter(([status]) => !['success', 'clientDisconnected', 'responseStreamDisconnected'].includes(status)).map(([status, requests]) => ({ status, requests })) } + }) +} diff --git a/scripts/tools/daily-checkin-observability.mjs b/checks/_helpers/observability.mjs similarity index 100% rename from scripts/tools/daily-checkin-observability.mjs rename to checks/_helpers/observability.mjs diff --git a/checks/_helpers/policy.mjs b/checks/_helpers/policy.mjs new file mode 100644 index 00000000..1a6a3d88 --- /dev/null +++ b/checks/_helpers/policy.mjs @@ -0,0 +1,33 @@ +import { fail, pass, unavailable, warn } from '@harlan-zw/nuxt-checkin/external' + +export function evaluateCI(data) { + if (!data.workflows.length || data.workflows.some(workflow => workflow.state._tag === 'missing')) + return unavailable('Workflow evidence is missing.', data) + if (data.workflows.some(workflow => workflow.state._tag === 'failure')) + return fail('A workflow is failing.', data) + if (data.workflows.some(workflow => workflow.state._tag === 'pending' && workflow.state.consecutiveFailures > 0)) + return warn('A workflow is pending after a failure.', data) + return pass(data) +} + +export function evaluateD1(data, now, since) { + if (data.missingExpectedTables.length) + return unavailable('Expected database tables are missing.', data) + if (data.migrations.error) + return unavailable('Production migration evidence is unavailable.', data) + if (data.cost.x_budget_target === null) + return unavailable('X discovery budget is unavailable.', data) + if (data.cost.x_projected_monthly_usd > 60) + return fail('Projected X spend exceeds $60 per month.', data) + const failed = data.activity.digests_failed > 0 + || data.pipeline.stale_reserved_jobs > 0 + || data.syncJobs?.some(job => job.last_status === 'error' || (job.stale_after_seconds && Number(job.last_run_at) + Number(job.stale_after_seconds) < now.getTime() / 1000)) + if (failed) + return fail('Delivery or scheduled work needs attention.', data) + const warning = data.cost.x_over_budget || data.cost.x_projected_monthly_usd > 40 + || data.migrations.localHead !== data.migrations.prodHead + || ['newly_broken_repos_impacted', 'skill_sync_failures', 'stale_dirty_skills', 'ai_stuck', 'ai_failed', 'embeddings_failed', 'embeddings_stuck', 'failed_jobs'].some(key => data.pipeline[key] > 0) + || data.syncJobs?.some(job => job.last_status === 'partial' && job.last_run_at >= since.getTime() / 1000) + || data.healthEmail?.[0]?.deliveryStatus !== 'sent' + return warning ? warn('Database evidence needs attention.', data) : pass(data) +} diff --git a/checks/_helpers/sentry.ts b/checks/_helpers/sentry.ts new file mode 100644 index 00000000..0dba4bac --- /dev/null +++ b/checks/_helpers/sentry.ts @@ -0,0 +1,11 @@ +import type { ExternalCheckContext } from '@harlan-zw/nuxt-checkin/external' +import { defineSentryCheck } from '@harlan-zw/nuxt-sentry/checks' + +export function collectSentry(context: ExternalCheckContext) { + return context.collect(collectSentry, 'skilld.sentry-backlog', async signal => ({ value: await defineSentryCheck({ + id: 'sentry.skilld', + org: 'harlan-zw', + project: 'skilld', + environment: context.env.SENTRY_ENVIRONMENT, + }).run({ ...context, signal }) })) +} diff --git a/checks/external/ci.ts b/checks/external/ci.ts new file mode 100644 index 00000000..0d4283b3 --- /dev/null +++ b/checks/external/ci.ts @@ -0,0 +1,10 @@ +import { defineExternalCheck } from '@harlan-zw/nuxt-checkin/external' +import { collectCI } from '../_helpers/collectors.mjs' +import { evaluateCI } from '../_helpers/policy.mjs' + +export default defineExternalCheck({ + id: 'skilld.ci', + async run(context) { + return evaluateCI(await collectCI(context)) + }, +}) diff --git a/checks/external/database.ts b/checks/external/database.ts new file mode 100644 index 00000000..425b5319 --- /dev/null +++ b/checks/external/database.ts @@ -0,0 +1,10 @@ +import { defineExternalCheck } from '@harlan-zw/nuxt-checkin/external' +import { collectD1 } from '../_helpers/collectors.mjs' +import { evaluateD1 } from '../_helpers/policy.mjs' + +export default defineExternalCheck({ + id: 'skilld.database', + async run(context) { + return evaluateD1(await collectD1(context), context.now, context.since) + }, +}) diff --git a/checks/external/deploy.ts b/checks/external/deploy.ts new file mode 100644 index 00000000..d410dac3 --- /dev/null +++ b/checks/external/deploy.ts @@ -0,0 +1,10 @@ +import { defineExternalCheck, pass, unavailable } from '@harlan-zw/nuxt-checkin/external' +import { collectDeploy } from '../_helpers/collectors.mjs' + +export default defineExternalCheck({ + id: 'skilld.deploy', + async run(context) { + const data = await collectDeploy(context) + return data.latest?.versionId ? pass(data) : unavailable('Deployed Worker identity is unavailable.', data) + }, +}) diff --git a/checks/external/git.ts b/checks/external/git.ts new file mode 100644 index 00000000..1d9f73e5 --- /dev/null +++ b/checks/external/git.ts @@ -0,0 +1,9 @@ +import { defineExternalCheck, pass } from '@harlan-zw/nuxt-checkin/external' +import { collectGit } from '../_helpers/collectors.mjs' + +export default defineExternalCheck({ + id: 'skilld.git', + async run(context) { + return pass(await collectGit(context)) + }, +}) diff --git a/checks/external/health-email.ts b/checks/external/health-email.ts new file mode 100644 index 00000000..37500bb5 --- /dev/null +++ b/checks/external/health-email.ts @@ -0,0 +1,18 @@ +import { checkReport, defineExternalCheck, unavailable } from '@harlan-zw/nuxt-checkin/external' +import { collectD1, collectDeploy } from '../_helpers/collectors.mjs' + +export default defineExternalCheck({ + id: 'skilld.health-email', + async run(context) { + const deployment = (await collectDeploy(context)).latest?.versionId + const healthReport = (await collectD1(context)).healthEmail?.[0]?.checkin + return deployment + ? checkReport(healthReport, { + now: context.clock(), + identity: { site: 'skilld.dev', environment: 'production', deployment }, + required: ['skilld.daily-health', 'skilld.daily-health-coverage'], + maxAgeMs: 36 * 60 * 60 * 1000, + }) + : unavailable('Deployed Worker identity is unavailable.') + }, +}) diff --git a/checks/external/home.ts b/checks/external/home.ts new file mode 100644 index 00000000..50ed3cb7 --- /dev/null +++ b/checks/external/home.ts @@ -0,0 +1,8 @@ +import { defineHttpCheck } from '@harlan-zw/nuxt-checkin/external' + +export default defineHttpCheck({ + id: 'skilld.home', + url: 'https://skilld.dev/', + status: 200, + attempts: 2, +}) diff --git a/checks/external/sentry-details.ts b/checks/external/sentry-details.ts new file mode 100644 index 00000000..5317c60d --- /dev/null +++ b/checks/external/sentry-details.ts @@ -0,0 +1,34 @@ +import { defineExternalCheck, pass, readBoundedResponseText, unavailable, warn } from '@harlan-zw/nuxt-checkin/external' +import { parseSentryIssuesResponse } from '../../scripts/tools/sentry-observability.mjs' +import { collectSentry } from '../_helpers/sentry' + +export default defineExternalCheck({ + id: 'skilld.sentry-details', + async run(context) { + const backlog = await collectSentry(context) + if (backlog._tag === 'Pass') + return pass({ newIssues: [], recurringIssues: [], truncatedAtLimit: false }) + if (backlog._tag !== 'Warn' && backlog._tag !== 'Fail') + return unavailable('Sentry backlog evidence is unavailable.') + const token = context.credentials.sentry + if (!token) + return unavailable('Sentry credential is unavailable.') + const sinceIso = context.since.toISOString() + const url = new URL('https://sentry.io/api/0/organizations/harlan-zw/issues/') + url.searchParams.set('query', `project:skilld is:unresolved lastSeen:>${sinceIso.slice(0, 19)}`) + url.searchParams.set('sort', 'freq') + url.searchParams.set('limit', '25') + if (context.env.SENTRY_ENVIRONMENT) + url.searchParams.set('environment', context.env.SENTRY_ENVIRONMENT) + const response = await fetch(url, { headers: { Authorization: `Bearer ${token}` }, signal: context.signal, redirect: 'error' }) + const body: unknown = JSON.parse(await readBoundedResponseText(response, 2_097_152)) + const details = parseSentryIssuesResponse(response.status, body, 'runtime credential', sinceIso, 25) + if (details._tag !== 'available') + return unavailable(details.diagnostic, details) + if (details.truncatedAtLimit) + return unavailable('Sentry detail results reached the page limit.', details) + return details.newIssues.length || details.recurringIssues.length + ? warn('Sentry issues occurred during this window.', details) + : pass(details) + }, +}) diff --git a/checks/external/sentry.ts b/checks/external/sentry.ts new file mode 100644 index 00000000..5bd8467f --- /dev/null +++ b/checks/external/sentry.ts @@ -0,0 +1,7 @@ +import { defineExternalCheck } from '@harlan-zw/nuxt-checkin/external' +import { collectSentry } from '../_helpers/sentry' + +export default defineExternalCheck({ + id: 'sentry.skilld', + run: collectSentry, +}) diff --git a/checks/external/skills.ts b/checks/external/skills.ts new file mode 100644 index 00000000..89d8c82e --- /dev/null +++ b/checks/external/skills.ts @@ -0,0 +1,8 @@ +import { defineHttpCheck } from '@harlan-zw/nuxt-checkin/external' + +export default defineHttpCheck({ + id: 'skilld.skills', + url: 'https://skilld.dev/skills', + status: 200, + attempts: 2, +}) diff --git a/checks/external/workers.ts b/checks/external/workers.ts new file mode 100644 index 00000000..4543ae76 --- /dev/null +++ b/checks/external/workers.ts @@ -0,0 +1,10 @@ +import { defineExternalCheck, pass, warn } from '@harlan-zw/nuxt-checkin/external' +import { collectWorkers } from '../_helpers/collectors.mjs' + +export default defineExternalCheck({ + id: 'skilld.workers', + async run(context) { + const data = await collectWorkers(context) + return data.nonOk.length ? warn('Worker requests have non-success outcomes.', data) : pass(data) + }, +}) diff --git a/docs/shared-checkin.md b/docs/shared-checkin.md index 7f92146d..81eb3b33 100644 --- a/docs/shared-checkin.md +++ b/docs/shared-checkin.md @@ -42,3 +42,13 @@ The existing detailed Sentry probe runs only when the shared check finds unresol It keeps recurrence and impact details for triage. That extra request remains an optimization opportunity. Live credentials and the environment mapping remain unverified. + +## Shared CLI + +Run `pnpm checkin --save` to prepare and execute registered external checks. +The shared CLI writes daily archives and keeps successful state boundaries. +Same-day reruns preserve the morning baseline. +Read `results[].result.evidence` for each registered check. +The `skilld.database` result retains inventory, activity, pipeline, cost, and operator-email evidence. +The `skilld.sentry-details` result retains new and recurring issue details. +The pending `@harlan-zw/nuxt-checkin@0.2.0-alpha.0` release supplies the CLI. diff --git a/nuxt.config.ts b/nuxt.config.ts index b9c1c826..ea066292 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -3,12 +3,14 @@ import { fileURLToPath } from 'node:url' import { unpublishedAgentPaths } from './layers/marketing/app/utils/agent-pages' import { dependencyPluginCompat } from './scripts/lib/dependency-plugin-compat' import { withBuildAssetMissFallthrough } from './scripts/lib/static-asset-fallthrough' +import { externalCheckin } from './shared/checkin-external' import { SENTRY_DSN, sentryRelease, sentryReportingEnabled } from './shared/sentry' const hasSentryAuthToken = Boolean(process.env.SENTRY_AUTH_TOKEN) || existsSync('.env.sentry-build-plugin') export default defineNuxtConfig({ + checkin: { external: externalCheckin }, extends: ['./layers/admin', './layers/artifact-delivery', './layers/identity', './layers/registry', './layers/marketing', './layers/mcp'], hooks: { diff --git a/package.json b/package.json index 62f3d4b2..3390ff90 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,8 @@ "cli:grammar": "tsx scripts/check-published-cli-grammar.ts", "start:playwright:webserver": "NODE_ENV=test nuxi preview --port 5678", "db:migrations:dev": "wrangler d1 migrations apply DB --local", - "db:migrations:prod": "wrangler d1 migrations apply DB --remote" + "db:migrations:prod": "wrangler d1 migrations apply DB --remote", + "checkin": "nuxt-checkin prepare && nuxt-checkin" }, "dependencies": { "@comark/nuxt": "catalog:", diff --git a/scripts/tools/daily-checkin-checks.mjs b/scripts/tools/daily-checkin-checks.mjs deleted file mode 100644 index a2b3c9dd..00000000 --- a/scripts/tools/daily-checkin-checks.mjs +++ /dev/null @@ -1,26 +0,0 @@ -import { checkReport, defineCheck, runChecks, unavailable } from '@harlan-zw/nuxt-checkin/server' -import { defineSentryCheck } from '@harlan-zw/nuxt-sentry/checks' - -export function runDailyOperatorChecks({ now, token, deployment, healthReport, environment, clock = () => new Date(), request = fetch }) { - return runChecks([ - defineSentryCheck({ id: 'sentry.skilld', org: 'harlan-zw', project: 'skilld', environment }, request), - defineCheck({ - id: 'skilld.health-email', - run: () => deployment - ? checkReport(healthReport, { - now: clock(), - identity: { site: 'skilld.dev', environment: 'production', deployment }, - required: ['skilld.daily-health', 'skilld.daily-health-coverage'], - maxAgeMs: 36 * 60 * 60 * 1000, - }) - : unavailable('Deployed Worker identity is unavailable.'), - }), - ], { - now, - identity: { site: 'skilld.dev', environment: 'production', deployment: deployment ?? 'unknown' }, - required: ['sentry.skilld', 'skilld.health-email'], - credentials: token ? { sentry: token } : {}, - timeoutMs: 30_000, - totalTimeoutMs: 30_000, - }) -} diff --git a/scripts/tools/daily-checkin-data.mjs b/scripts/tools/daily-checkin-data.mjs deleted file mode 100644 index a18c4218..00000000 --- a/scripts/tools/daily-checkin-data.mjs +++ /dev/null @@ -1,543 +0,0 @@ -#!/usr/bin/env node - -import { spawnSync } from 'node:child_process' -import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' -import { homedir } from 'node:os' -import { dirname, join } from 'node:path' -import { fileURLToPath } from 'node:url' -import { runDailyOperatorChecks } from './daily-checkin-checks.mjs' -import { - approximateDeployedSha, - buildWorkersQuery, - collectWorkflowRuns, - parseHealthEmailRows, - parseWorkflowName, - readMigrationState, - refreshProductionRef, - summarizeWorkflowRuns, -} from './daily-checkin-observability.mjs' -import { ghEnv, runReadOnlyProcess, subprocessEnv } from './daily-checkin-process.mjs' -import { parseSentryIssuesResponse } from './sentry-observability.mjs' - -const root = join(dirname(fileURLToPath(import.meta.url)), '../..') -const checkinDir = join(root, 'docs/ops/checkins') -const statePath = join(checkinDir, 'state.json') -const save = process.argv.includes('--save') -const now = new Date() -const defaultSince = new Date(now.getTime() - 24 * 60 * 60 * 1000) -const state = existsSync(statePath) ? JSON.parse(readFileSync(statePath, 'utf8')) : null -const since = new Date(state?.lastRunAt || defaultSince) -const sinceIso = since.toISOString() -const sinceSec = Math.floor(since.getTime() / 1000) -const sinceMs = since.getTime() -// X deduplicates read charges per UTC day, so its budget counter is keyed on -// the UTC date rather than on the check-in window. -const utcDay = now.toISOString().slice(0, 10) -const wrangler = join(root, 'node_modules/.bin/wrangler') - -// The gh environment is resolved lazily and once per process: deciding it -// costs a real API call, and the ci probe alone invokes gh once per workflow. -let ghEnvironment -function run(command, args, options = {}) { - const baseEnv = { ...process.env, NO_COLOR: '1' } - return runReadOnlyProcess(spawnSync, command, args, { - cwd: options.cwd ?? root, - encoding: 'utf8', - maxBuffer: 20 * 1024 * 1024, - env: subprocessEnv(command, baseEnv, () => (ghEnvironment ??= ghEnv(baseEnv, spawnSync))), - }) -} - -function probe(load) { - try { - return load() - } - catch (error) { - return { error: error instanceof Error ? error.message : String(error) } - } -} - -async function probeAsync(load) { - return await load().catch(error => ({ error: error instanceof Error ? error.message : String(error) })) -} - -function commandJson(command, args) { - return JSON.parse(run(command, args)) -} - -function d1Query(sql) { - const output = commandJson(wrangler, [ - 'd1', - 'execute', - 'DB', - '--remote', - '--json', - '--config', - 'wrangler.jsonc', - '--command', - sql, - ]) - const statement = output[0] - if (!statement?.success) - throw new Error('D1 query did not succeed') - return statement.results ?? [] -} - -// One fetch per run, shared by every production read. A failed fetch becomes -// a probe error here, so no read below can compare against a stale ref. -const production = probe(() => refreshProductionRef(args => run('git', args))) - -const git = probe(() => { - const dirty = run('git', ['status', '--short']).split('\n').filter(Boolean) - const commits = run('git', ['log', `--since=${sinceIso}`, '--pretty=format:%H%x09%aI%x09%s']) - .split('\n') - .filter(Boolean) - .map((line) => { - const [sha, authoredAt, ...subject] = line.split('\t') - return { sha, authoredAt, subject: subject.join('\t') } - }) - return { - branch: run('git', ['branch', '--show-current']), - head: run('git', ['rev-parse', 'HEAD']), - dirtyFiles: dirty, - commitsSinceLastRun: commits, - productionRef: production, - } -}) - -const deploy = probe(() => { - const deployments = commandJson(wrangler, ['deployments', 'list', '--json', '--config', 'wrangler.jsonc']) - const latest = [...deployments].sort((a, b) => String(b.created_on).localeCompare(String(a.created_on)))[0] ?? null - return { - latest: latest && { - id: latest.id, - createdOn: latest.created_on, - versionId: latest.versions?.find(version => version.percentage === 100)?.version_id ?? latest.versions?.[0]?.version_id ?? null, - message: latest.annotations?.['workers/message'] ?? null, - approxDeployedSha: 'error' in production ? null : approximateDeployedSha(args => run('git', args), production, latest.created_on), - }, - } -}) - -const runFields = 'databaseId,workflowName,displayTitle,headSha,status,conclusion,createdAt,updatedAt,url' - -const ci = probe(() => { - const workflowDir = join(root, '.github/workflows') - const definedWorkflows = readdirSync(workflowDir) - .filter(file => /\.ya?ml$/.test(file)) - .map(file => parseWorkflowName(readFileSync(join(workflowDir, file), 'utf8'))) - .filter(Boolean) - .sort() - // A low-cadence workflow can fall outside a flat recent-runs page, and an - // absent row reads as `missing`, which is an observability gap rather than a - // health signal. Each workflow is therefore paged on its own name, and paged - // deeper when a run of skipped guard runs hides the last verdict. - const perWorkflowRows = collectWorkflowRuns( - (name, limit) => commandJson('gh', ['run', 'list', '--workflow', name, '--limit', String(limit), '--json', runFields]), - definedWorkflows, - ) - const recent = commandJson('gh', ['run', 'list', '--limit', '20', '--json', runFields]) - return { - workflows: summarizeWorkflowRuns(perWorkflowRows, definedWorkflows), - recent: recent.slice(0, 10), - } -}) - -// Loop 1 front door only. `/guides` was probed until 2026-08-04 and was dropped -// when guides were retired; it now answers 410 by design, so a 200 there would be -// the regression. Removing a probe silently reads as a passing probe, so any future -// removal belongs here in writing. -const http = await probeAsync(async () => { - const entries = await Promise.all([ - 'https://skilld.dev/', - 'https://skilld.dev/skills', - ].map(async (url) => { - // A transport failure is itself a front-door result, so it becomes a null - // status the report prints rather than an exception that kills the probe. - const hit = async () => await fetch(url, { - redirect: 'follow', - signal: AbortSignal.timeout(15_000), - }).then(response => response.status).catch((error) => { - console.warn(`[http] ${url} ${error instanceof Error ? error.message : String(error)}`) - return null - }) - const firstAttempt = await hit() - if (firstAttempt === 200) - return [url, 200] - const status = await hit() - return [url, status === firstAttempt ? status : { status, firstAttempt }] - })) - return Object.fromEntries(entries) -}) - -/** - * Pull DAILY_DISCOVERY_READ_BUDGET out of the source of truth. Returns null - * rather than a guess when the constant cannot be found, so a rename surfaces - * as a missing number instead of a confidently wrong one. - */ -function readDailyBudget() { - try { - const src = readFileSync(join(root, 'shared/server/x-ingest.ts'), 'utf8') - const match = src.match(/DAILY_DISCOVERY_READ_BUDGET\s*=\s*(\d+)/) - return match ? Number(match[1]) : null - } - catch { - return null - } -} - -const d1 = probe(() => { - const tableRows = d1Query(`SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name`) - const tables = new Set(tableRows.map(row => row.name)) - const has = table => tables.has(table) - - /** - * Column-level guard. `has(table)` is not enough: a migration that adds a - * column ships in the code before it is applied to production, and a query - * naming the new column fails the whole d1 probe. That silently blanked the - * entire checkin the first time `skills_scanned_at` was referenced, which is - * the worst possible failure for the tool that reports failures. - */ - const columnCache = new Map() - const hasColumn = (table, column) => { - if (!tables.has(table)) - return false - if (!columnCache.has(table)) { - columnCache.set( - table, - new Set(d1Query(`SELECT name FROM pragma_table_info('${table}')`).map(row => row.name)), - ) - } - return columnCache.get(table).has(column) - } - - const inventoryParts = [ - has('skills') ? `(SELECT COUNT(*) FROM skills) AS skills` : 'NULL AS skills', - has('repos') ? `(SELECT COUNT(*) FROM repos) AS repos` : 'NULL AS repos', - has('owners') ? `(SELECT COUNT(*) FROM owners) AS owners` : 'NULL AS owners', - has('users') ? `(SELECT COUNT(*) FROM users) AS users` : 'NULL AS users', - has('collections_v2') ? `(SELECT COUNT(*) FROM collections_v2 WHERE deleted_at IS NULL) AS collections` : 'NULL AS collections', - has('repos') ? `(SELECT COUNT(*) FROM repos WHERE broken_since IS NOT NULL) AS broken_repos` : 'NULL AS broken_repos', - ] - const activityParts = [ - has('skills') ? `(SELECT COUNT(*) FROM skills WHERE first_seen_at >= ${sinceSec}) AS new_skills` : 'NULL AS new_skills', - has('activity') ? `(SELECT COUNT(DISTINCT owner || '/' || repo) FROM activity WHERE occurred_at >= ${sinceSec}) AS changed_repos` : 'NULL AS changed_repos', - has('install_events') ? `(SELECT COUNT(*) FROM install_events WHERE occurred_at >= ${sinceMs}) AS install_events` : 'NULL AS install_events', - has('users') ? `(SELECT COUNT(*) FROM users WHERE created_at >= ${sinceSec}) AS new_users` : 'NULL AS new_users', - has('digest_runs') ? `(SELECT COUNT(*) FROM digest_runs WHERE status = 'sent' AND sent_at >= ${sinceSec}) AS digests_sent` : 'NULL AS digests_sent', - has('digest_runs') ? `(SELECT COUNT(*) FROM digest_runs WHERE status = 'failed' AND window_end >= ${sinceSec}) AS digests_failed` : 'NULL AS digests_failed', - ] - // A count alone gives the 3-repo AMBER gate nothing to repair or prune, so - // the identities behind `newly_broken_repos_impacted` are archived too: one - // row per impacted repo with the reasons it still backs. The filter is shared - // with the count so the number and the rows can never disagree. - const impactedBrokenReposGuarded = has('repos') && has('skills') && has('user_starred_repos') && has('skill_subscriptions') && has('collection_skills_v2') && has('activity') && has('install_events') - const impactedBrokenReposWhere = `r.broken_since >= ${sinceSec} AND (EXISTS (SELECT 1 FROM skills s WHERE s.owner = r.owner AND s.repo = r.repo) OR EXISTS (SELECT 1 FROM user_starred_repos usr WHERE usr.owner = r.owner AND usr.repo = r.repo) OR EXISTS (SELECT 1 FROM skill_subscriptions sub WHERE sub.owner = r.owner AND sub.repo = r.repo) OR EXISTS (SELECT 1 FROM collection_skills_v2 cs WHERE cs.owner = r.owner AND cs.repo = r.repo) OR EXISTS (SELECT 1 FROM activity a JOIN install_events ie ON ie.slug = a.owner || '/' || a.name WHERE a.owner = r.owner AND a.repo = r.repo))` - const pipelineParts = [ - has('repos') ? `(SELECT COUNT(*) FROM repos WHERE broken_since >= ${sinceSec}) AS newly_broken_repos_total` : 'NULL AS newly_broken_repos_total', - impactedBrokenReposGuarded ? `(SELECT COUNT(*) FROM repos r WHERE ${impactedBrokenReposWhere}) AS newly_broken_repos_impacted` : 'NULL AS newly_broken_repos_impacted', - has('skills') ? `(SELECT COUNT(*) FROM skills WHERE sync_status IS NOT NULL AND sync_status != 'ok' AND last_synced_at >= ${sinceSec}) AS skill_sync_failures` : 'NULL AS skill_sync_failures', - has('skill_dirty') ? `(SELECT COUNT(*) FROM skill_dirty WHERE queued_at < ${Math.floor(now.getTime() / 1000) - 3600}) AS stale_dirty_skills` : 'NULL AS stale_dirty_skills', - // `ai_batches` covers the Anthropic batch pipeline only, and embeddings never - // touch it, so these three read 0 while `HAIKU_GENERATION_PAUSED` holds. The - // embedding counters below are the ones that move; reading an AI-pipeline - // verdict off `ai_submitted` alone produced a wrong diagnosis on 2026-08-03. - has('ai_batches') ? `(SELECT COUNT(*) FROM ai_batches WHERE status = 'submitted') AS ai_submitted` : 'NULL AS ai_submitted', - has('ai_batches') ? `(SELECT COUNT(*) FROM ai_batches WHERE status = 'submitted' AND submitted_at < ${Math.floor(now.getTime() / 1000) - 10800}) AS ai_stuck` : 'NULL AS ai_stuck', - has('ai_batches') ? `(SELECT COUNT(*) FROM ai_batches WHERE status IN ('failed','expired') AND COALESCE(completed_at, submitted_at) >= ${sinceSec}) AS ai_failed` : 'NULL AS ai_failed', - has('embedding_attempts') ? `(SELECT COUNT(*) FROM embedding_attempts WHERE state = 'completed' AND started_at >= ${sinceSec}) AS embeddings_completed` : 'NULL AS embeddings_completed', - has('embedding_attempts') ? `(SELECT COUNT(*) FROM embedding_attempts WHERE state IN ('provider_failed','rejected','vector_succeeded_marker_failed') AND started_at >= ${sinceSec}) AS embeddings_failed` : 'NULL AS embeddings_failed', - // Bounded below at 7 days: `started` rows abandoned by a closed incident - // otherwise read as a live fault forever (6 rows from Jul 23/29 did exactly - // that on 2026-08-04). A row stuck past 7 days is history, not an incident. - has('embedding_attempts') ? `(SELECT COUNT(*) FROM embedding_attempts WHERE state = 'started' AND started_at < ${Math.floor(now.getTime() / 1000) - 3600} AND started_at >= ${Math.floor(now.getTime() / 1000) - 604800}) AS embeddings_stuck` : 'NULL AS embeddings_stuck', - has('failed_jobs') ? `(SELECT COUNT(*) FROM failed_jobs WHERE failed_at >= ${sinceSec}) AS failed_jobs` : 'NULL AS failed_jobs', - has('jobs') ? `(SELECT COUNT(*) FROM jobs WHERE reserved_at IS NOT NULL AND reserved_at < ${Math.floor(now.getTime() / 1000) - 900} AND completed_at IS NULL AND failed_at IS NULL) AS stale_reserved_jobs` : 'NULL AS stale_reserved_jobs', - ] - /** - * Turn raw X read counts into the number that matters: dollars, and whether - * the month is on track. Reads are $0.005 each with no included allowance. - * - * The budget is read out of shared/server/x-ingest.ts rather than copied. - * This script is plain node and cannot import the app's module graph, and a - * hand-copied constant had already gone stale once, reporting 22 while the - * real budget was 400 and so claiming the ingest was massively over budget. - */ - function withXSpend(costRow) { - const USD_PER_READ = 0.005 - const budgetTarget = readDailyBudget() - const row = costRow ?? {} - const discoveryToday = Number(row.x_discovery_reads_today ?? 0) - const hot = Number(row.x_hot_posts ?? 0) - // Refresh only pays for a hot post again when its window crosses midnight. - // Treating every hot post as one more charge is the pessimistic bound. - const projectedDaily = discoveryToday + hot - return { - ...row, - x_budget_target: budgetTarget, - x_budget_used_pct: budgetTarget > 0 ? Math.round((discoveryToday / budgetTarget) * 100) : null, - x_projected_daily_reads: projectedDaily, - x_projected_monthly_usd: Math.round(projectedDaily * 30 * USD_PER_READ * 100) / 100, - x_over_budget: discoveryToday > budgetTarget, - // The value question: reads bought per skill actually verified. Rising - // means the query is getting less precise or the stream is noisier. - x_reads_per_verified_skill: Number(row.x_verified_skills ?? 0) > 0 - ? Math.round(Number(row.x_discovery_reads_total ?? 0) / Number(row.x_verified_skills)) - : null, - // Budget spent without finishing the window means the stream is - // outrunning the ingest and the backlog grows daily. - x_falling_behind: discoveryToday >= budgetTarget, - } - } - - const costParts = [ - has('ai_batch_costs') ? `(SELECT COALESCE(SUM(est_cost_usd), 0) FROM ai_batch_costs WHERE submitted_at >= ${sinceSec}) AS ai_cost_usd` : 'NULL AS ai_cost_usd', - // X is pay-per-use with no included allowance: every post read is a real - // $0.005 invoice line, so it belongs in the health report rather than only - // in task logs. `budget_spent` counts today only when `budget_day` matches; - // a stale day reads as 0, matching how the ingest resets it. - has('x_ingest_cursor') ? `(SELECT COALESCE(SUM(CASE WHEN budget_day = '${utcDay}' THEN budget_spent ELSE 0 END), 0) FROM x_ingest_cursor) AS x_discovery_reads_today` : 'NULL AS x_discovery_reads_today', - has('x_ingest_cursor') ? `(SELECT COALESCE(SUM(posts_read_total), 0) FROM x_ingest_cursor) AS x_discovery_reads_total` : 'NULL AS x_discovery_reads_total', - // Hot posts are refresh's remaining exposure: each can cost at most one - // more read, when its window crosses midnight UTC. - has('x_posts') ? `(SELECT COUNT(*) FROM x_posts WHERE refresh_tier = 'hot') AS x_hot_posts` : 'NULL AS x_hot_posts', - has('x_posts') ? `(SELECT COUNT(*) FROM x_posts WHERE first_seen_at >= ${sinceSec}) AS x_posts_24h` : 'NULL AS x_posts_24h', - // How stale the freshest ingested post is. Climbs when discovery lags. - has('x_posts') ? `(SELECT CAST((${Math.floor(now.getTime() / 1000)} - MAX(posted_at)) / 3600 AS INTEGER) FROM x_posts) AS x_newest_post_age_hours` : 'NULL AS x_newest_post_age_hours', - has('x_post_skills') ? `(SELECT COUNT(*) FROM x_post_skills) AS x_verified_skills` : 'NULL AS x_verified_skills', - has('x_post_skills') ? `(SELECT COUNT(*) FROM x_post_skills WHERE verified_at >= ${sinceSec}) AS x_verified_skills_24h` : 'NULL AS x_verified_skills_24h', - hasColumn('x_posts', 'skills_scanned_at') ? `(SELECT COUNT(*) FROM x_posts WHERE skills_scanned_at IS NULL) AS x_posts_unscanned` : 'NULL AS x_posts_unscanned', - hasColumn('discovery_ledger', 'held_reason') ? `(SELECT COUNT(*) FROM discovery_ledger WHERE status = 'pending' AND held_reason IS NULL) AS x_ledger_pending` : 'NULL AS x_ledger_pending', - hasColumn('discovery_ledger', 'held_reason') ? `(SELECT COUNT(*) FROM discovery_ledger WHERE held_reason IS NOT NULL) AS x_ledger_held` : 'NULL AS x_ledger_held', - has('discovery_ledger') ? `(SELECT COUNT(*) FROM discovery_ledger WHERE status = 'submitted' AND submitted_at IS NOT NULL AND submitted_at < ${Math.floor(now.getTime() / 1000) - 21600}) AS x_ledger_stalled` : 'NULL AS x_ledger_stalled', - has('discovery_ledger') ? `(SELECT COUNT(*) FROM discovery_ledger WHERE status = 'indexed') AS x_ledger_indexed` : 'NULL AS x_ledger_indexed', - ] - - const impactedBrokenRepos = impactedBrokenReposGuarded - ? d1Query(`SELECT r.owner, r.repo, rtrim((CASE WHEN EXISTS (SELECT 1 FROM skills s WHERE s.owner = r.owner AND s.repo = r.repo) THEN 'skill ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM user_starred_repos usr WHERE usr.owner = r.owner AND usr.repo = r.repo) THEN 'star ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM skill_subscriptions sub WHERE sub.owner = r.owner AND sub.repo = r.repo) THEN 'subscription ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM collection_skills_v2 cs WHERE cs.owner = r.owner AND cs.repo = r.repo) THEN 'collection ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM activity a JOIN install_events ie ON ie.slug = a.owner || '/' || a.name WHERE a.owner = r.owner AND a.repo = r.repo) THEN 'install ' ELSE '' END)) AS reason FROM repos r WHERE ${impactedBrokenReposWhere} ORDER BY r.broken_since DESC`) - : null - const syncJobs = has('sync_jobs') - ? d1Query(`SELECT name, cron, stale_after_seconds, last_run_at, last_status, last_error FROM sync_jobs WHERE enabled = 1 ORDER BY name`) - : null - const failedJobFingerprints = has('failed_jobs') - ? d1Query(`SELECT queue, job_type, substr(exception, 1, 160) exception, COUNT(*) count, MIN(failed_at) first_failed_at, MAX(failed_at) last_failed_at FROM failed_jobs WHERE failed_at >= ${sinceSec} GROUP BY queue, job_type, substr(exception, 1, 160) ORDER BY count DESC LIMIT 10`) - : null - const healthEmail = has('daily_health_checks') - ? parseHealthEmailRows(d1Query(`SELECT report_date, health_status, delivery_status, recipient, sent_at, error, summary_json FROM daily_health_checks ORDER BY report_date DESC LIMIT 2`)) - : null - const recentJobBatches = has('job_batches') - ? d1Query(`SELECT id, name, total_jobs, pending_jobs, failed_jobs, created_at, updated_at, finished_at FROM job_batches ORDER BY updated_at DESC LIMIT 10`) - : null - const registryMaintenance = has('registry_maintenance') - ? d1Query(`SELECT name, status, batch_id, updated_at, last_error FROM registry_maintenance ORDER BY updated_at DESC`) - : null - const prodMigrationHead = has('d1_migrations') - ? d1Query(`SELECT MAX(name) name FROM d1_migrations`)[0]?.name ?? null - : null - // Production can be checked from a feature worktree. Compare D1 with the - // production branch, while keeping worktree-only migrations visible. - // - // A failed `git fetch` leaves `migrations` as `{ error }` with neither - // `localHead` nor `prodHead`, so the drift gate cannot misread a fetch - // failure as drift. A 2026-09-03 silently failed fetch archived drift that - // did not exist. - const migrations = 'error' in production - ? { error: production.error } - : { - ...readMigrationState( - args => run('git', args), - production, - readdirSync(join(root, 'migrations')), - ), - prodHead: prodMigrationHead, - } - - return { - tables: [...tables], - inventory: d1Query(`SELECT ${inventoryParts.join(', ')}`)[0], - activity: d1Query(`SELECT ${activityParts.join(', ')}`)[0], - pipeline: { - ...d1Query(`SELECT ${pipelineParts.join(', ')}`)[0], - newly_broken_repos_impacted_identities: impactedBrokenRepos, - }, - cost: withXSpend(d1Query(`SELECT ${costParts.join(', ')}`)[0]), - syncJobs, - failedJobFingerprints, - healthEmail, - recentJobBatches, - registryMaintenance, - migrations, - missingExpectedTables: [ - 'skills', - 'repos', - 'owners', - 'users', - 'activity', - 'install_events', - 'sync_jobs', - 'jobs', - 'failed_jobs', - 'daily_health_checks', - ].filter(table => !has(table)), - } -}) - -function cloudflareToken() { - const environmentToken = process.env.CLOUDFLARE_USAGE_TOKEN || process.env.CLOUDFLARE_API_TOKEN || process.env.CF_API_TOKEN - if (environmentToken) - return environmentToken - const auth = commandJson(wrangler, ['auth', 'token', '--json']) - if (!auth.token) - throw new Error('Cloudflare token unavailable') - return auth.token -} - -const workers = await probeAsync(async () => { - const query = buildWorkersQuery(sinceIso, now.toISOString()) - const response = await fetch('https://api.cloudflare.com/client/v4/graphql', { - method: 'POST', - headers: { 'Authorization': `Bearer ${cloudflareToken()}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ query }), - }) - const body = await response.json() - if (!response.ok || body.errors?.length) - throw new Error(`Cloudflare GraphQL ${response.status}: ${JSON.stringify(body.errors ?? body).slice(0, 300)}`) - const rows = body.data?.viewer?.accounts?.[0]?.workersInvocationsAdaptive ?? [] - const outcomes = {} - for (const row of rows) { - if (row.dimensions.scriptName !== 'skilld-dev') - continue - const status = row.dimensions.status - outcomes[status] = (outcomes[status] ?? 0) + row.sum.requests - } - return { - outcomes, - nonOk: Object.entries(outcomes) - .filter(([status]) => !['success', 'clientDisconnected', 'responseStreamDisconnected'].includes(status)) - .map(([status, requests]) => ({ status, requests })), - } -}) - -// Token sources in descending order of read scope. `.env.sentry-build-plugin` -// holds the build-plugin token, which only carries source-map upload scope and -// 403s on the issues API, so it is the last resort rather than the first. -function sentryToken() { - if (process.env.SENTRY_AUTH_TOKEN) - return { token: process.env.SENTRY_AUTH_TOKEN, source: 'SENTRY_AUTH_TOKEN env' } - - const rcPath = join(homedir(), '.sentryclirc') - if (existsSync(rcPath)) { - const rcToken = readFileSync(rcPath, 'utf8').match(/^token\s*=\s*(\S+)/m)?.[1] - if (rcToken) - return { token: rcToken, source: '~/.sentryclirc' } - } - - const buildPluginPath = join(root, '.env.sentry-build-plugin') - if (existsSync(buildPluginPath)) { - const buildToken = readFileSync(buildPluginPath, 'utf8').match(/^SENTRY_AUTH_TOKEN=(\S+)/m)?.[1] - if (buildToken) - return { token: buildToken, source: '.env.sentry-build-plugin' } - } - - return null -} - -// Recurrences now share the page with new issues, so the cap has to hold -// both. `truncatedAtLimit` reports when it did not. -const SENTRY_ISSUE_LIMIT = 25 - -const resolvedSentry = sentryToken() -const checkin = await runDailyOperatorChecks({ - now, - token: resolvedSentry?.token, - deployment: deploy.latest?.versionId, - healthReport: d1.healthEmail?.[0]?.checkin, - environment: process.env.SENTRY_ENVIRONMENT, -}) - -const sentry = await (async () => { - const backlog = checkin.results.find(check => check.id === 'sentry.skilld')?.result - if (backlog?._tag === 'Pass') - return { _tag: 'available', newIssues: [], recurringIssues: [], truncatedAtLimit: false } - if (backlog?._tag !== 'Warn' && backlog?._tag !== 'Fail') - return { _tag: 'missing_observability', status: null, diagnostic: backlog?.reason ?? 'Sentry evidence is unavailable.' } - const resolved = resolvedSentry - if (!resolved) { - return { - _tag: 'missing_observability', - status: null, - diagnostic: 'No Sentry token found in SENTRY_AUTH_TOKEN, ~/.sentryclirc, or .env.sentry-build-plugin.', - } - } - const { token, source: tokenSource } = resolved - // `lastSeen` catches an issue that fired again on an id we already know; - // `firstSeen` could only ever report births, which is how 686 post-deploy - // errors reached the 2026-08-06 archive as silence. - const query = encodeURIComponent(`project:skilld is:unresolved lastSeen:>${sinceIso.slice(0, 19)}`) - const response = await fetch(`https://sentry.io/api/0/organizations/harlan-zw/issues/?query=${query}&sort=freq&limit=${SENTRY_ISSUE_LIMIT}`, { - headers: { Authorization: `Bearer ${token}` }, - }).catch(error => ({ - networkError: error instanceof Error ? error.message : String(error), - })) - if ('networkError' in response) { - return { - _tag: 'provider_failure', - status: null, - diagnostic: `Sentry issues request failed: ${response.networkError}`, - } - } - const issues = await response.json().catch(error => ({ - parseError: error instanceof Error ? error.message : String(error), - })) - if (issues?.parseError) { - return { - _tag: 'parse_failure', - status: response.status, - diagnostic: `Sentry issues response was not JSON: ${issues.parseError}`, - } - } - return parseSentryIssuesResponse(response.status, issues, tokenSource, sinceIso, SENTRY_ISSUE_LIMIT) -})().catch(error => ({ - _tag: 'provider_failure', - status: null, - diagnostic: error instanceof Error ? error.message : String(error), -})) - -const doc = { - generatedAt: now.toISOString(), - since: sinceIso, - git, - deploy, - ci, - http, - d1, - workers, - sentry, - checkin, -} - -console.log(JSON.stringify(doc, null, 2)) - -const failedProbes = Object.entries({ git, deploy, ci, http, d1, workers, sentry, checkin }).filter(([name, value]) => - value?.error || (name === 'sentry' && value?._tag !== 'available') || (name === 'checkin' && value?.coverage !== 'complete'), -) -if (failedProbes.length) - console.error(`WARN ${failedProbes.length} probes failed: ${failedProbes.map(([name]) => name).join(', ')}. Missing data is not health.`) - -if (save) { - mkdirSync(checkinDir, { recursive: true }) - const archivePath = join(checkinDir, `${now.toISOString().slice(0, 10)}.json`) - if (existsSync(archivePath)) { - const time = now.toISOString().slice(11, 16).replace(':', '') - const rerunPath = archivePath.replace(/\.json$/, `.rerun-${time}.json`) - writeFileSync(rerunPath, `${JSON.stringify(doc, null, 2)}\n`) - console.error(`Same-day rerun wrote ${rerunPath}. The morning baseline and state were not changed.`) - } - else { - writeFileSync(archivePath, `${JSON.stringify(doc, null, 2)}\n`) - if (!d1.error) - writeFileSync(statePath, `${JSON.stringify({ lastRunAt: now.toISOString() }, null, 2)}\n`) - else - console.error('State was not advanced because the D1 probe failed.') - } -} diff --git a/scripts/tools/daily-checkin-process.mjs b/scripts/tools/daily-checkin-process.mjs deleted file mode 100644 index 6b873901..00000000 --- a/scripts/tools/daily-checkin-process.mjs +++ /dev/null @@ -1,62 +0,0 @@ -function failureMessage(command, result) { - const output = String(result.stderr || result.stdout || '').trim() - if (output) - return output - if (result.error instanceof Error) - return result.error.message - if (result.signal) - return `${command} terminated by ${result.signal}` - return `${command} exited ${String(result.status)}` -} - -/** - * Check-in probes are read-only, so a process terminated by the host can retry - * once without duplicating a mutation. Ordinary nonzero exits remain final. - */ -export function runReadOnlyProcess(spawn, command, args, options = {}) { - let result = spawn(command, args, options) - if (result.status === null) - result = spawn(command, args, options) - if (result.status !== 0) - throw new Error(failureMessage(command, result).slice(0, 800)) - return String(result.stdout).trim() -} - -/** - * gh resolves credentials from the environment before its keyring, so one - * stale GITHUB_TOKEN or GH_TOKEN overrides a valid `gh auth` login and every - * call 401s. When a token is present, validate it once with a cheap - * authenticated call; when gh rejects it, drop both variables so gh falls - * back to the keyring. - */ -export function ghEnv(env, spawn) { - if (!(env.GITHUB_TOKEN || env.GH_TOKEN)) - return env - try { - runReadOnlyProcess(spawn, 'gh', ['api', 'user'], { - encoding: 'utf8', - env: { ...env, NO_COLOR: '1' }, - }) - return env - } - catch { - // gh rejected the environment token (or the check itself could not run), - // so the keyring login is the better credential source either way. - const { GITHUB_TOKEN: _staleToken, GH_TOKEN: _staleAltToken, ...keyringEnv } = env - return keyringEnv - } -} - -/** - * Commands whose subprocesses authenticate to GitHub share the sanitized - * environment; everything else inherits the caller's environment untouched. - * git serves github.com credentials through the `gh auth git-credential` - * helper, which reads the same environment variables gh does, so a stale - * token breaks `git fetch` exactly the way it breaks gh. Resolution is - * handed over lazily so commands that never touch GitHub never pay for it. - */ -export function subprocessEnv(command, baseEnv, resolveGitHubEnv) { - if (command === 'gh' || command === 'git') - return resolveGitHubEnv() - return baseEnv -} diff --git a/shared/checkin-external.ts b/shared/checkin-external.ts new file mode 100644 index 00000000..ede201c2 --- /dev/null +++ b/shared/checkin-external.ts @@ -0,0 +1,23 @@ +import type { ExternalOptions } from '@harlan-zw/nuxt-checkin/external' + +export const externalCheckin = { + required: ['skilld.git', 'skilld.deploy', 'skilld.ci', 'skilld.home', 'skilld.skills', 'skilld.database', 'skilld.workers', 'skilld.health-email', 'sentry.skilld', 'skilld.sentry-details'], + credentials: { + sentry: { + env: 'SENTRY_AUTH_TOKEN', + files: [ + { path: '~/.sentryclirc', key: 'token' }, + { path: '.env.sentry-build-plugin', key: 'SENTRY_AUTH_TOKEN' }, + ], + }, + }, + timeoutMs: 180_000, + totalTimeoutMs: 240_000, + save: { + dir: 'docs/ops/checkins', + stateFile: 'state.json', + timestampKey: 'lastRunAt', + baseline: 'daily', + defaultWindowMs: 86_400_000, + }, +} satisfies ExternalOptions diff --git a/test/unit/daily-checkin-checks.test.ts b/test/unit/daily-checkin-checks.test.ts index 5f337152..cd45a01c 100644 --- a/test/unit/daily-checkin-checks.test.ts +++ b/test/unit/daily-checkin-checks.test.ts @@ -1,7 +1,22 @@ // @vitest-environment node -import { defineCheck, pass, runChecks } from '@harlan-zw/nuxt-checkin/server' +import { defineCheck, pass, runChecks, runExternalChecks } from '@harlan-zw/nuxt-checkin/external' import { describe, expect, it, vi } from 'vitest' -import { runDailyOperatorChecks } from '../../scripts/tools/daily-checkin-checks.mjs' +import healthEmailCheck from '../../checks/external/health-email' +import sentryCheck from '../../checks/external/sentry' + +const evidence = vi.hoisted(() => ({ deployment: undefined as string | undefined, healthReport: undefined as unknown })) +vi.mock('../../checks/_helpers/collectors.mjs', () => ({ + collectDeploy: async () => ({ latest: { versionId: evidence.deployment } }), + collectD1: async () => ({ healthEmail: [{ checkin: evidence.healthReport }] }), +})) + +async function runDailyOperatorChecks({ now, token, deployment, healthReport, environment, clock, request }: any) { + evidence.deployment = deployment + evidence.healthReport = healthReport + vi.stubGlobal('fetch', request) + const { report } = await runExternalChecks([sentryCheck, healthEmailCheck], { required: ['sentry.skilld', 'skilld.health-email'], credentials: { sentry: 'SENTRY_AUTH_TOKEN' } }, { now, clock, env: { SENTRY_AUTH_TOKEN: token, SENTRY_ENVIRONMENT: environment } }) + return report +} const now = new Date('2026-09-14T22:00:00Z') const identity = { site: 'skilld.dev', environment: 'production', deployment: 'worker-123' } diff --git a/test/unit/daily-checkin-observability.test.ts b/test/unit/daily-checkin-observability.test.ts index 6aa880c2..f39dc0d4 100644 --- a/test/unit/daily-checkin-observability.test.ts +++ b/test/unit/daily-checkin-observability.test.ts @@ -10,7 +10,7 @@ import { readMigrationState, refreshProductionRef, summarizeWorkflowRuns, -} from '../../scripts/tools/daily-checkin-observability.mjs' +} from '../../checks/_helpers/observability.mjs' function run( workflowName: string, diff --git a/test/unit/daily-checkin-process.test.ts b/test/unit/daily-checkin-process.test.ts deleted file mode 100644 index e922a1ea..00000000 --- a/test/unit/daily-checkin-process.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -// @vitest-environment node -import { describe, expect, it, vi } from 'vitest' -import { ghEnv, runReadOnlyProcess, subprocessEnv } from '../../scripts/tools/daily-checkin-process.mjs' - -describe('daily check-in process runner', () => { - it('retries one transient signal termination', () => { - const spawn = vi.fn() - .mockReturnValueOnce({ status: null, signal: 'SIGTERM', stdout: '', stderr: '' }) - .mockReturnValueOnce({ status: 0, signal: null, stdout: '[]\n', stderr: '' }) - - expect(runReadOnlyProcess(spawn, 'wrangler', ['deployments', 'list'])).toBe('[]') - expect(spawn).toHaveBeenCalledTimes(2) - }) - - it('reports the terminating signal when the retry also fails', () => { - const spawn = vi.fn() - .mockReturnValue({ status: null, signal: 'SIGKILL', stdout: '', stderr: '' }) - - expect(() => runReadOnlyProcess(spawn, 'wrangler', ['deployments', 'list'])) - .toThrow('wrangler terminated by SIGKILL') - }) -}) - -describe('daily check-in gh environment', () => { - it('validates the environment token with gh and keeps it when gh accepts it', () => { - const spawn = vi.fn() - .mockReturnValue({ status: 0, stdout: '{"login":"harlan-zw"}', stderr: '' }) - const env = { GITHUB_TOKEN: 'gho_valid', HOME: '/home/harlan' } - - expect(ghEnv(env, spawn)).toEqual({ GITHUB_TOKEN: 'gho_valid', HOME: '/home/harlan' }) - expect(spawn).toHaveBeenCalledWith( - 'gh', - ['api', 'user'], - expect.objectContaining({ env: expect.objectContaining({ GITHUB_TOKEN: 'gho_valid' }) }), - ) - }) - - it('strips stale tokens so gh falls back to its keyring login', () => { - const spawn = vi.fn() - .mockReturnValue({ status: 1, stdout: '', stderr: 'gh: HTTP 401: Bad credentials (https://api.github.com/user)' }) - const env = { GITHUB_TOKEN: 'gho_stale', GH_TOKEN: 'ghp_stale', HOME: '/home/harlan' } - - expect(ghEnv(env, spawn)).toEqual({ HOME: '/home/harlan' }) - }) - - it('leaves the environment untouched when no github token is set', () => { - const spawn = vi.fn() - const env = { HOME: '/home/harlan' } - - expect(ghEnv(env, spawn)).toEqual({ HOME: '/home/harlan' }) - expect(spawn).not.toHaveBeenCalled() - }) -}) - -describe('daily check-in subprocess environment', () => { - const staleEnv = { GITHUB_TOKEN: 'gho_stale', GH_TOKEN: 'ghp_stale', HOME: '/home/harlan' } - const rejectToken = () => vi.fn() - .mockReturnValue({ status: 1, stdout: '', stderr: 'gh: HTTP 401: Bad credentials (https://api.github.com/user)' }) - - it('serves gh the sanitized environment', () => { - const spawn = rejectToken() - - expect(subprocessEnv('gh', staleEnv, () => ghEnv(staleEnv, spawn))).toEqual({ HOME: '/home/harlan' }) - }) - - it('serves git the sanitized environment so a stale token cannot reach the credential helper', () => { - const spawn = rejectToken() - - expect(subprocessEnv('git', staleEnv, () => ghEnv(staleEnv, spawn))).toEqual({ HOME: '/home/harlan' }) - }) - - it('leaves other commands on the inherited environment without paying for token resolution', () => { - const spawn = rejectToken() - - expect(subprocessEnv('wrangler', staleEnv, () => ghEnv(staleEnv, spawn))).toEqual(staleEnv) - expect(spawn).not.toHaveBeenCalled() - }) -}) diff --git a/test/unit/external-checkin-collectors.test.ts b/test/unit/external-checkin-collectors.test.ts new file mode 100644 index 00000000..9fa24296 --- /dev/null +++ b/test/unit/external-checkin-collectors.test.ts @@ -0,0 +1,68 @@ +// @vitest-environment node +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { defineCheck, pass, runChecks, runExternalChecks } from '@harlan-zw/nuxt-checkin/external' +import { afterEach, expect, it, vi } from 'vitest' +import databaseCheck from '../../checks/external/database' +import deployCheck from '../../checks/external/deploy' +import gitCheck from '../../checks/external/git' +import healthCheck from '../../checks/external/health-email' + +const boundary = vi.hoisted(() => ({ command: vi.fn() })) +vi.mock('@harlan-zw/nuxt-checkin/external', async (original) => { + const actual = await original() + return { ...actual, runCheckCommand: boundary.command } +}) +const roots: string[] = [] +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true }))) + boundary.command.mockReset() +}) + +it('shares deployment and database evidence without blocking nested collectors', async () => { + const now = new Date('2026-09-14T00:00:00Z') + const root = await mkdtemp(join(tmpdir(), 'skilld-checkin-')) + roots.push(root) + await mkdir(join(root, 'shared/server'), { recursive: true }) + await mkdir(join(root, 'migrations')) + await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400') + await writeFile(join(root, 'migrations/001.sql'), '') + const health = await runChecks(['skilld.daily-health', 'skilld.daily-health-coverage'].map(id => defineCheck({ id, run: () => pass() })), { now, identity: { site: 'skilld.dev', environment: 'production', deployment: 'release-1' } }) + boundary.command.mockImplementation(async (_context, command: string, args: string[]) => { + let output: unknown = '' + if (command === 'git') { + output = args[0] === 'ls-tree' ? 'migrations/001.sql' : args[0] === 'branch' ? 'main' : args[0] === 'rev-parse' ? 'abc' : '' + return { _tag: 'Ok', stdout: output, stderr: '' } + } + if (args[0] === 'deployments') { + output = [{ id: 'deployment', created_on: now.toISOString(), versions: [{ percentage: 100, version_id: 'release-1' }] }] + } + else { + const sql = args[args.indexOf('--command') + 1]! + let rows: unknown[] = [] + if (sql.includes('sqlite_master')) + rows = ['skills', 'repos', 'owners', 'users', 'activity', 'install_events', 'sync_jobs', 'jobs', 'failed_jobs', 'daily_health_checks', 'd1_migrations'].map(name => ({ name })) + else if (sql.includes('MAX(name)')) + rows = [{ name: '001.sql' }] + else if (sql.includes('summary_json')) + rows = [{ report_date: '2026-09-14', delivery_status: 'sent', summary_json: JSON.stringify({ window: {}, reasons: [], warnings: [], checkin: health }) }] + else if (sql.includes('AS skills')) + rows = [{ skills: 23, broken_repos: 99 }] + else if (sql.includes('AS new_skills')) + rows = [{ new_skills: 2, digests_failed: 0 }] + else if (sql.includes('AS newly_broken_repos_total')) + rows = [{ newly_broken_repos_total: 0, stale_reserved_jobs: 0 }] + else if (sql.includes('AS ai_cost_usd')) + rows = [{ x_discovery_reads_today: 10, x_hot_posts: 0 }] + output = [{ success: true, results: rows }] + } + return { _tag: 'Ok', stdout: JSON.stringify(output), stderr: '' } + }) + const checks = [gitCheck, deployCheck, databaseCheck, healthCheck] + const result = await runExternalChecks(checks, { required: checks.map(check => check.id), timeoutMs: 1000, totalTimeoutMs: 1500 }, { rootDir: root, env: {}, clock: () => now }) + expect(result.report).toMatchObject({ severity: 'pass', coverage: 'complete' }) + expect(result.report.results.find(check => check.id === 'skilld.database')?.result).toMatchObject({ evidence: { inventory: { skills: 23 } } }) + expect(boundary.command.mock.calls.filter(([, command, args]) => command === 'git' && args[0] === 'fetch')).toHaveLength(1) + expect(boundary.command.mock.calls.filter(([, , args]) => args.includes('SELECT name FROM sqlite_master WHERE type = \'table\' ORDER BY name'))).toHaveLength(1) +}) diff --git a/test/unit/external-checkin-policy.test.ts b/test/unit/external-checkin-policy.test.ts new file mode 100644 index 00000000..590aa2d4 --- /dev/null +++ b/test/unit/external-checkin-policy.test.ts @@ -0,0 +1,23 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { evaluateCI, evaluateD1 } from '../../checks/_helpers/policy.mjs' + +const now = new Date('2026-09-14T00:00:00Z') +const since = new Date(now.getTime() - 86_400_000) +function healthy() { + return { missingExpectedTables: [], migrations: { localHead: '001.sql', prodHead: '001.sql' }, inventory: { broken_repos: 999 }, cost: { x_budget_target: 400, x_projected_monthly_usd: 40, x_budget_used_pct: 100, x_falling_behind: true }, activity: { digests_failed: 0 }, pipeline: {}, syncJobs: [], healthEmail: [{ deliveryStatus: 'sent' }] } +} +it('keeps accepted X spend and cumulative broken inventory healthy', () => { + expect(evaluateD1(healthy(), now, since)._tag).toBe('Pass') +}) +it.each([[40.01, 'Warn'], [60, 'Warn'], [60.01, 'Fail']])('gates projected monthly X spend at %s', (spend, tag) => { + const data = healthy() + data.cost.x_projected_monthly_usd = Number(spend) + expect(evaluateD1(data, now, since)._tag).toBe(tag) +}) +it('keeps missing schema evidence incomplete', () => { + expect(evaluateD1({ ...healthy(), missingExpectedTables: ['jobs'] }, now, since)._tag).toBe('Unavailable') +}) +it('keeps a pending workflow visible after a failure', () => { + expect(evaluateCI({ workflows: [{ state: { _tag: 'pending', consecutiveFailures: 1 } }] })._tag).toBe('Warn') +}) From 028a178e2723364984cfcb35ab8734baf97b5f52 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Tue, 15 Sep 2026 01:08:14 +1000 Subject: [PATCH 06/11] fix(checkin): bound Worker analytics responses --- checks/_helpers/collectors.mjs | 4 ++-- test/unit/external-checkin-collectors.test.ts | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/checks/_helpers/collectors.mjs b/checks/_helpers/collectors.mjs index 4c0a05a8..a25e25b4 100644 --- a/checks/_helpers/collectors.mjs +++ b/checks/_helpers/collectors.mjs @@ -1,7 +1,7 @@ import { readFileSync } from 'node:fs' import { readdir, readFile } from 'node:fs/promises' import { join } from 'node:path' -import { runCheckCommand } from '@harlan-zw/nuxt-checkin/external' +import { readBoundedResponseText, runCheckCommand } from '@harlan-zw/nuxt-checkin/external' import { buildWorkersQuery, parseHealthEmailRows, parseWorkflowName, summarizeWorkflowRuns } from './observability.mjs' const resources = Object.fromEntries(['github-auth', 'production-ref', 'git', 'deploy', 'ci', 'd1', 'workers'].map(key => [key, {}])) @@ -285,7 +285,7 @@ export function collectWorkers(context) { if (!token) throw new Error('Cloudflare token is unavailable.') const response = await fetch('https://api.cloudflare.com/client/v4/graphql', { method: 'POST', redirect: 'error', signal: context.signal, headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ query: buildWorkersQuery(context.since.toISOString(), context.now.toISOString()) }) }) - const body = await response.json() + const body = JSON.parse(await readBoundedResponseText(response, 2_097_152)) if (!response.ok || body.errors?.length) throw new Error(`Cloudflare GraphQL returned HTTP ${response.status}.`) const rows = body.data?.viewer?.accounts?.[0]?.workersInvocationsAdaptive diff --git a/test/unit/external-checkin-collectors.test.ts b/test/unit/external-checkin-collectors.test.ts index 9fa24296..67e2b627 100644 --- a/test/unit/external-checkin-collectors.test.ts +++ b/test/unit/external-checkin-collectors.test.ts @@ -8,6 +8,7 @@ import databaseCheck from '../../checks/external/database' import deployCheck from '../../checks/external/deploy' import gitCheck from '../../checks/external/git' import healthCheck from '../../checks/external/health-email' +import workersCheck from '../../checks/external/workers' const boundary = vi.hoisted(() => ({ command: vi.fn() })) vi.mock('@harlan-zw/nuxt-checkin/external', async (original) => { @@ -18,6 +19,7 @@ const roots: string[] = [] afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true }))) boundary.command.mockReset() + vi.unstubAllGlobals() }) it('shares deployment and database evidence without blocking nested collectors', async () => { @@ -66,3 +68,10 @@ it('shares deployment and database evidence without blocking nested collectors', expect(boundary.command.mock.calls.filter(([, command, args]) => command === 'git' && args[0] === 'fetch')).toHaveLength(1) expect(boundary.command.mock.calls.filter(([, , args]) => args.includes('SELECT name FROM sqlite_master WHERE type = \'table\' ORDER BY name'))).toHaveLength(1) }) + +it('rejects oversized Worker analytics evidence', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json({ data: { viewer: { accounts: [{ workersInvocationsAdaptive: [] }] } }, padding: 'x'.repeat(2_097_153) }))) + const { report } = await runExternalChecks([workersCheck], { required: [workersCheck.id] }, { env: { CLOUDFLARE_USAGE_TOKEN: 'test-token' } }) + expect(report.coverage).toBe('incomplete') + expect(report.results[0]?.result._tag).toBe('Unavailable') +}) From e3e2f42f50a2ef26471b64221d092fdda73c2dcc Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Tue, 15 Sep 2026 01:20:09 +1000 Subject: [PATCH 07/11] chore(checkin): install published shared CLI prerelease --- docs/shared-checkin.md | 4 ++-- pnpm-lock.yaml | 34 ++++++++++++++++++---------------- pnpm-workspace.yaml | 8 +++++++- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/docs/shared-checkin.md b/docs/shared-checkin.md index 81eb3b33..c7980fce 100644 --- a/docs/shared-checkin.md +++ b/docs/shared-checkin.md @@ -26,7 +26,7 @@ Its recorded scheduled-run outcome already enters the daily summary. ## Release -The check-in uses published `@harlan-zw/nuxt-checkin@0.1.0` and `@harlan-zw/nuxt-sentry@0.1.5`. +The check-in uses published `@harlan-zw/nuxt-checkin@0.2.0-alpha.0` and `@harlan-zw/nuxt-sentry@0.1.5`. The lockfile resolves registry packages. ## External daily check-in @@ -51,4 +51,4 @@ Same-day reruns preserve the morning baseline. Read `results[].result.evidence` for each registered check. The `skilld.database` result retains inventory, activity, pipeline, cost, and operator-email evidence. The `skilld.sentry-details` result retains new and recurring issue details. -The pending `@harlan-zw/nuxt-checkin@0.2.0-alpha.0` release supplies the CLI. +The published `@harlan-zw/nuxt-checkin@0.2.0-alpha.0` prerelease supplies the CLI. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ee621612..22560498 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,8 +22,8 @@ catalogs: specifier: 0.2.3 version: 0.2.3 '@harlan-zw/nuxt-checkin': - specifier: 0.1.0 - version: 0.1.0 + specifier: 0.2.0-alpha.0 + version: 0.2.0-alpha.0 '@harlan-zw/nuxt-cloudflare': specifier: 0.4.1 version: 0.4.1 @@ -218,10 +218,10 @@ importers: version: 0.1.5(98d5e2cee3315275cd7393fa02452bc5) '@harlan-zw/nuxt-cf-jobs': specifier: 'catalog:' - version: 0.2.3(7d0ed70cf4c00b5041e45380a979dbc5) + version: 0.2.3(5d31fc731279723bd3098f63e96d2e8d) '@harlan-zw/nuxt-checkin': specifier: 'catalog:' - version: 0.1.0(e385fb48e32147193c05e7ca84503583) + version: 0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583) '@harlan-zw/nuxt-use-query': specifier: 'catalog:' version: 0.2.1(c8e02c8c74d77c0aa02a701b775491e2) @@ -336,13 +336,13 @@ importers: version: 5.20260901.1 '@harlan-zw/nuxt-cloudflare': specifier: 'catalog:' - version: 0.4.1(86f60b01955541ac2ab799ea25f82069) + version: 0.4.1(a9eee12ab58b94d72eb85df2e8853afb) '@harlan-zw/nuxt-dx': specifier: 'catalog:' version: 0.1.2(530d2fb91dd879c8eaebd3f75fc3ff20) '@harlan-zw/nuxt-sentry': specifier: 'catalog:' - version: 0.1.5(b92f4956c1d351dde2153cf2bca45bda) + version: 0.1.5(6e534e5f1d0945427f5c4edc8068e342) '@nuxt/a11y': specifier: 'catalog:' version: 1.0.0-alpha.1(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)))(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)) @@ -1522,9 +1522,10 @@ packages: '@sentry/cloudflare': optional: true - '@harlan-zw/nuxt-checkin@0.1.0': - resolution: {integrity: sha512-APUDTVzj2jRs1qDj9l39jtXOLJzKXVd2UnnW4F2LI9QlbXHAOKM7VYe3Oj3Orwmw1Vbxp6GMwujLeVhv1O468A==} + '@harlan-zw/nuxt-checkin@0.2.0-alpha.0': + resolution: {integrity: sha512-oBMJXH8pt6RXniG5xlxSbqdc3M2uQd/cfJiJ6Z0FuM6BTeYdn0w5S1UyrWNrUjlKieQFxAjpM8zY5NCaXjR+NA==} engines: {node: '>=22.12.0'} + hasBin: true peerDependencies: nuxt: '>=4.5.0 <6.0.0' @@ -9731,9 +9732,9 @@ snapshots: - shiki - unplugin - '@harlan-zw/nuxt-cf-jobs@0.2.3(7d0ed70cf4c00b5041e45380a979dbc5)': + '@harlan-zw/nuxt-cf-jobs@0.2.3(5d31fc731279723bd3098f63e96d2e8d)': dependencies: - '@harlan-zw/nuxt-cloudflare': 0.4.1(86f60b01955541ac2ab799ea25f82069) + '@harlan-zw/nuxt-cloudflare': 0.4.1(a9eee12ab58b94d72eb85df2e8853afb) '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) '@nuxt/schema': 4.5.2 citty: 0.2.2 @@ -9741,7 +9742,7 @@ snapshots: nuxt: 4.5.2(@babel/plugin-proposal-decorators@8.0.2)(@babel/plugin-syntax-jsx@7.29.7)(@babel/plugin-syntax-typescript@7.29.7)(@oxc-project/types@0.144.0)(@types/node@26.4.0)(@vue/compiler-sfc@3.5.42)(better-sqlite3@13.0.3)(commander@14.0.3)(db0@0.3.4(better-sqlite3@13.0.3))(esbuild@0.28.2)(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.4)(rollup@4.62.4))(rollup@4.62.4)(srvx@0.12.5)(supports-color@10.2.2)(terser@5.50.0)(tsx@4.23.13)(typescript@6.0.3)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0) vite: 8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0) optionalDependencies: - '@harlan-zw/nuxt-checkin': 0.1.0(e385fb48e32147193c05e7ca84503583) + '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583) '@sentry/cloudflare': 10.73.0(@cloudflare/workers-types@5.20260901.1)(wrangler@4.127.1(@cloudflare/workers-types@5.20260901.1)) transitivePeerDependencies: - '@aws-sdk/client-rds-data' @@ -9829,11 +9830,12 @@ snapshots: - xml2js - zod - '@harlan-zw/nuxt-checkin@0.1.0(e385fb48e32147193c05e7ca84503583)': + '@harlan-zw/nuxt-checkin@0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583)': dependencies: '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) nuxt: 4.5.2(@babel/plugin-proposal-decorators@8.0.2)(@babel/plugin-syntax-jsx@7.29.7)(@babel/plugin-syntax-typescript@7.29.7)(@oxc-project/types@0.144.0)(@types/node@26.4.0)(@vue/compiler-sfc@3.5.42)(better-sqlite3@13.0.3)(commander@14.0.3)(db0@0.3.4(better-sqlite3@13.0.3))(esbuild@0.28.2)(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.4)(rollup@4.62.4))(rollup@4.62.4)(srvx@0.12.5)(supports-color@10.2.2)(terser@5.50.0)(tsx@4.23.13)(typescript@6.0.3)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0) pathe: 2.0.3 + rollup: 4.62.4 typescript: 5.9.3 transitivePeerDependencies: - magic-string @@ -9842,7 +9844,7 @@ snapshots: - rolldown - unplugin - '@harlan-zw/nuxt-cloudflare@0.4.1(86f60b01955541ac2ab799ea25f82069)': + '@harlan-zw/nuxt-cloudflare@0.4.1(a9eee12ab58b94d72eb85df2e8853afb)': dependencies: '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) citty: 0.2.2 @@ -9855,7 +9857,7 @@ snapshots: unstorage: 1.17.5(db0@0.3.4(better-sqlite3@13.0.3))(ioredis@5.11.1(supports-color@10.2.2)) wrangler: 4.127.1(@cloudflare/workers-types@5.20260901.1) optionalDependencies: - '@harlan-zw/nuxt-checkin': 0.1.0(e385fb48e32147193c05e7ca84503583) + '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583) transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -9919,14 +9921,14 @@ snapshots: - unplugin - vue - '@harlan-zw/nuxt-sentry@0.1.5(b92f4956c1d351dde2153cf2bca45bda)': + '@harlan-zw/nuxt-sentry@0.1.5(6e534e5f1d0945427f5c4edc8068e342)': dependencies: '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) '@sentry/nuxt': 10.73.0(433b670f0f22b940727aa0d678e7208d) nuxt: 4.5.2(@babel/plugin-proposal-decorators@8.0.2)(@babel/plugin-syntax-jsx@7.29.7)(@babel/plugin-syntax-typescript@7.29.7)(@oxc-project/types@0.144.0)(@types/node@26.4.0)(@vue/compiler-sfc@3.5.42)(better-sqlite3@13.0.3)(commander@14.0.3)(db0@0.3.4(better-sqlite3@13.0.3))(esbuild@0.28.2)(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.4)(rollup@4.62.4))(rollup@4.62.4)(srvx@0.12.5)(supports-color@10.2.2)(terser@5.50.0)(tsx@4.23.13)(typescript@6.0.3)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0) pathe: 2.0.3 optionalDependencies: - '@harlan-zw/nuxt-checkin': 0.1.0(e385fb48e32147193c05e7ca84503583) + '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583) '@sentry/cloudflare': 10.73.0(@cloudflare/workers-types@5.20260901.1)(wrangler@4.127.1(@cloudflare/workers-types@5.20260901.1)) transitivePeerDependencies: - magic-string diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ffc197c4..b3915944 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,6 @@ minimumReleaseAgeExclude: + - '@harlan-zw/nuxt-checkin@0.1.0' + - '@harlan-zw/nuxt-checkin@0.2.0-alpha.0' - '@harlan-zw/nuxt-sentry@0.1.5' - '@harlan-zw/nuxt-checkin' - '@comark/nuxt' @@ -78,7 +80,7 @@ catalog: '@comark/nuxt': 0.6.2 '@harlan-zw/comark-content': 0.1.5 '@harlan-zw/nuxt-cf-jobs': 0.2.3 - '@harlan-zw/nuxt-checkin': 0.1.0 + '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0 '@harlan-zw/nuxt-cloudflare': 0.4.1 '@harlan-zw/nuxt-dx': 0.1.2 '@harlan-zw/nuxt-sentry': 0.1.5 @@ -157,3 +159,7 @@ allowBuilds: unrs-resolver: true vue-demi: true workerd: true + +peerDependencyRules: + allowedVersions: + '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0 From e48bf937684c62bb0dd5f0902d9a04457862b38a Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Tue, 15 Sep 2026 01:56:29 +1000 Subject: [PATCH 08/11] chore(checkin): adopt stable shared CLI and providers --- checks/_helpers/policy.mjs | 2 + docs/shared-checkin.md | 4 +- pnpm-lock.yaml | 62 +++++++++++++------------- pnpm-workspace.yaml | 19 ++++---- scripts/tools/sentry-observability.mjs | 10 ++--- 5 files changed, 48 insertions(+), 49 deletions(-) diff --git a/checks/_helpers/policy.mjs b/checks/_helpers/policy.mjs index d97920e2..2ca44ecc 100644 --- a/checks/_helpers/policy.mjs +++ b/checks/_helpers/policy.mjs @@ -1,5 +1,6 @@ import { fail, pass, warn } from '@harlan-zw/nuxt-checkin/external' +/** @returns {import('@harlan-zw/nuxt-checkin/external').CheckResult} */ export function evaluateCI(data) { if (!data.workflows.length || data.workflows.some(workflow => workflow.state._tag === 'missing')) return { _tag: 'Warn', reason: 'Workflow evidence is missing.', evidence: data, coverage: 'incomplete' } @@ -10,6 +11,7 @@ export function evaluateCI(data) { return pass(data) } +/** @returns {import('@harlan-zw/nuxt-checkin/external').CheckResult} */ export function evaluateD1(data, now, since) { if (data.missingExpectedTables.length) return { _tag: 'Warn', reason: 'Expected database tables are missing.', evidence: data, coverage: 'incomplete' } diff --git a/docs/shared-checkin.md b/docs/shared-checkin.md index 0413df64..7e58a277 100644 --- a/docs/shared-checkin.md +++ b/docs/shared-checkin.md @@ -26,7 +26,7 @@ Its recorded scheduled-run outcome already enters the daily summary. ## Release -The check-in uses published `@harlan-zw/nuxt-checkin@0.2.0-alpha.0` and `@harlan-zw/nuxt-sentry@0.1.5`. +The check-in uses published `@harlan-zw/nuxt-checkin@0.2.0` and `@harlan-zw/nuxt-sentry@0.1.6`. The lockfile resolves registry packages. ## External daily check-in @@ -51,7 +51,7 @@ Same-day reruns preserve the morning baseline. Read `results[].result.evidence` for each registered check. The `skilld.database` result retains inventory, activity, pipeline, cost, and operator-email evidence. The `skilld.sentry-details` result retains new and recurring issue details. -The published `@harlan-zw/nuxt-checkin@0.2.0-alpha.0` prerelease supplies the CLI. +The published `@harlan-zw/nuxt-checkin@0.2.0` supplies the CLI. The `skilld.baseline` result records fresh or stale comparison windows. Invalid saved state stops the CLI before collection and never advances the saved baseline. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 22560498..4af6b995 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,20 +19,20 @@ catalogs: specifier: 0.1.5 version: 0.1.5 '@harlan-zw/nuxt-cf-jobs': - specifier: 0.2.3 - version: 0.2.3 + specifier: 0.2.4 + version: 0.2.4 '@harlan-zw/nuxt-checkin': - specifier: 0.2.0-alpha.0 - version: 0.2.0-alpha.0 + specifier: 0.2.0 + version: 0.2.0 '@harlan-zw/nuxt-cloudflare': - specifier: 0.4.1 - version: 0.4.1 + specifier: 0.4.2 + version: 0.4.2 '@harlan-zw/nuxt-dx': specifier: 0.1.2 version: 0.1.2 '@harlan-zw/nuxt-sentry': - specifier: 0.1.5 - version: 0.1.5 + specifier: 0.1.6 + version: 0.1.6 '@harlan-zw/nuxt-use-query': specifier: 0.2.1 version: 0.2.1 @@ -218,10 +218,10 @@ importers: version: 0.1.5(98d5e2cee3315275cd7393fa02452bc5) '@harlan-zw/nuxt-cf-jobs': specifier: 'catalog:' - version: 0.2.3(5d31fc731279723bd3098f63e96d2e8d) + version: 0.2.4(a2b172cf2857179621cbbf1f6c6fa8f8) '@harlan-zw/nuxt-checkin': specifier: 'catalog:' - version: 0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583) + version: 0.2.0(e385fb48e32147193c05e7ca84503583) '@harlan-zw/nuxt-use-query': specifier: 'catalog:' version: 0.2.1(c8e02c8c74d77c0aa02a701b775491e2) @@ -336,13 +336,13 @@ importers: version: 5.20260901.1 '@harlan-zw/nuxt-cloudflare': specifier: 'catalog:' - version: 0.4.1(a9eee12ab58b94d72eb85df2e8853afb) + version: 0.4.2(ac7d691632b3cf8cbedfa5110c1172e0) '@harlan-zw/nuxt-dx': specifier: 'catalog:' version: 0.1.2(530d2fb91dd879c8eaebd3f75fc3ff20) '@harlan-zw/nuxt-sentry': specifier: 'catalog:' - version: 0.1.5(6e534e5f1d0945427f5c4edc8068e342) + version: 0.1.6(413e11a0b686d151b08c92e68297d580) '@nuxt/a11y': specifier: 'catalog:' version: 1.0.0-alpha.1(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)))(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)) @@ -1507,12 +1507,12 @@ packages: nuxt: '>=4.5.0 <6.0.0' vue: '>=3.5.0 <4.0.0' - '@harlan-zw/nuxt-cf-jobs@0.2.3': - resolution: {integrity: sha512-fh9B7OyagbTuqNnI24XY9IfjrK+D/ypOUTLpHGj9U70H0lzSIXBbk45AbpE5/zihiWK0iHWeg3FwPLWOQG7s+w==} + '@harlan-zw/nuxt-cf-jobs@0.2.4': + resolution: {integrity: sha512-yScNkMcyKAMyPJtItY5YK3avJmAvkLhEmTaFZ3bTgoQu1rTkYiFd5EoveyDKoa+EAVmfEICbCZdbEvFFhSH/Mw==} engines: {node: '>=22.12.0'} hasBin: true peerDependencies: - '@harlan-zw/nuxt-checkin': ^0.1.0 + '@harlan-zw/nuxt-checkin': ^0.1.0 || ^0.2.0-0 '@sentry/cloudflare': ^10.70.0 nuxt: '>=4.5.0 <6.0.0' vite: ^8.0.0 @@ -1522,19 +1522,19 @@ packages: '@sentry/cloudflare': optional: true - '@harlan-zw/nuxt-checkin@0.2.0-alpha.0': - resolution: {integrity: sha512-oBMJXH8pt6RXniG5xlxSbqdc3M2uQd/cfJiJ6Z0FuM6BTeYdn0w5S1UyrWNrUjlKieQFxAjpM8zY5NCaXjR+NA==} + '@harlan-zw/nuxt-checkin@0.2.0': + resolution: {integrity: sha512-1c7YBwovlMSN5cxe9jYt5Kfh7crZxlZciWvoyqdn0r/acE/Zb5lYE+byYVFVD2EZ82IOfLpecdQGijnXTSOVBw==} engines: {node: '>=22.12.0'} hasBin: true peerDependencies: nuxt: '>=4.5.0 <6.0.0' - '@harlan-zw/nuxt-cloudflare@0.4.1': - resolution: {integrity: sha512-fosZ5OB73Kq5nOkwTfHa6jiWdn+1ka2s6TTbTdjr3Z5RwKwqj0Z3vtmfED5iqOeqZxp2nOf2jYzJSJWKAHCdkA==} + '@harlan-zw/nuxt-cloudflare@0.4.2': + resolution: {integrity: sha512-800IClygYpN4U0GiFS1tW3OmF5boa14B6EDbkZkPcrJLF3bq4eN6Zo2UE0y5g+QFLma3o91VNAw74LupV3uy1A==} engines: {node: '>=22.12.0'} hasBin: true peerDependencies: - '@harlan-zw/nuxt-checkin': ^0.1.0 + '@harlan-zw/nuxt-checkin': ^0.1.0 || ^0.2.0-0 nuxt: '>=4.5.0 <6.0.0' wrangler: '>=4.120.0 <5.0.0' peerDependenciesMeta: @@ -1549,11 +1549,11 @@ packages: nuxt: '>=4.5.0 <6.0.0' vite: ^8.0.0 - '@harlan-zw/nuxt-sentry@0.1.5': - resolution: {integrity: sha512-0FR4/JuapsrP98UvkJKgii98w7HKljYyPw26Vel37YFBkeGLcldLRad/egQGkyFPBZ+G87j3UkbRBAfwOni57Q==} + '@harlan-zw/nuxt-sentry@0.1.6': + resolution: {integrity: sha512-Z0vdTmqzO7kk5KYjhrh7f9sH6PUkq+YTaaGEMlo3CRrpEn3AqlTw0EmFDtgG9Nhc8ScpjHBzDKISo2a/JE1lMw==} engines: {node: '>=22.12.0'} peerDependencies: - '@harlan-zw/nuxt-checkin': ^0.1.0 + '@harlan-zw/nuxt-checkin': ^0.1.0 || ^0.2.0-0 '@sentry/cloudflare': ^10.70.0 '@sentry/nuxt': ^10.70.0 nuxt: '>=4.5.0 <6.0.0' @@ -9732,9 +9732,9 @@ snapshots: - shiki - unplugin - '@harlan-zw/nuxt-cf-jobs@0.2.3(5d31fc731279723bd3098f63e96d2e8d)': + '@harlan-zw/nuxt-cf-jobs@0.2.4(a2b172cf2857179621cbbf1f6c6fa8f8)': dependencies: - '@harlan-zw/nuxt-cloudflare': 0.4.1(a9eee12ab58b94d72eb85df2e8853afb) + '@harlan-zw/nuxt-cloudflare': 0.4.2(ac7d691632b3cf8cbedfa5110c1172e0) '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) '@nuxt/schema': 4.5.2 citty: 0.2.2 @@ -9742,7 +9742,7 @@ snapshots: nuxt: 4.5.2(@babel/plugin-proposal-decorators@8.0.2)(@babel/plugin-syntax-jsx@7.29.7)(@babel/plugin-syntax-typescript@7.29.7)(@oxc-project/types@0.144.0)(@types/node@26.4.0)(@vue/compiler-sfc@3.5.42)(better-sqlite3@13.0.3)(commander@14.0.3)(db0@0.3.4(better-sqlite3@13.0.3))(esbuild@0.28.2)(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.4)(rollup@4.62.4))(rollup@4.62.4)(srvx@0.12.5)(supports-color@10.2.2)(terser@5.50.0)(tsx@4.23.13)(typescript@6.0.3)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0) vite: 8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0) optionalDependencies: - '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583) + '@harlan-zw/nuxt-checkin': 0.2.0(e385fb48e32147193c05e7ca84503583) '@sentry/cloudflare': 10.73.0(@cloudflare/workers-types@5.20260901.1)(wrangler@4.127.1(@cloudflare/workers-types@5.20260901.1)) transitivePeerDependencies: - '@aws-sdk/client-rds-data' @@ -9830,7 +9830,7 @@ snapshots: - xml2js - zod - '@harlan-zw/nuxt-checkin@0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583)': + '@harlan-zw/nuxt-checkin@0.2.0(e385fb48e32147193c05e7ca84503583)': dependencies: '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) nuxt: 4.5.2(@babel/plugin-proposal-decorators@8.0.2)(@babel/plugin-syntax-jsx@7.29.7)(@babel/plugin-syntax-typescript@7.29.7)(@oxc-project/types@0.144.0)(@types/node@26.4.0)(@vue/compiler-sfc@3.5.42)(better-sqlite3@13.0.3)(commander@14.0.3)(db0@0.3.4(better-sqlite3@13.0.3))(esbuild@0.28.2)(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.4)(rollup@4.62.4))(rollup@4.62.4)(srvx@0.12.5)(supports-color@10.2.2)(terser@5.50.0)(tsx@4.23.13)(typescript@6.0.3)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0) @@ -9844,7 +9844,7 @@ snapshots: - rolldown - unplugin - '@harlan-zw/nuxt-cloudflare@0.4.1(a9eee12ab58b94d72eb85df2e8853afb)': + '@harlan-zw/nuxt-cloudflare@0.4.2(ac7d691632b3cf8cbedfa5110c1172e0)': dependencies: '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) citty: 0.2.2 @@ -9857,7 +9857,7 @@ snapshots: unstorage: 1.17.5(db0@0.3.4(better-sqlite3@13.0.3))(ioredis@5.11.1(supports-color@10.2.2)) wrangler: 4.127.1(@cloudflare/workers-types@5.20260901.1) optionalDependencies: - '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583) + '@harlan-zw/nuxt-checkin': 0.2.0(e385fb48e32147193c05e7ca84503583) transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -9921,14 +9921,14 @@ snapshots: - unplugin - vue - '@harlan-zw/nuxt-sentry@0.1.5(6e534e5f1d0945427f5c4edc8068e342)': + '@harlan-zw/nuxt-sentry@0.1.6(413e11a0b686d151b08c92e68297d580)': dependencies: '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.4)(rollup@4.62.4)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) '@sentry/nuxt': 10.73.0(433b670f0f22b940727aa0d678e7208d) nuxt: 4.5.2(@babel/plugin-proposal-decorators@8.0.2)(@babel/plugin-syntax-jsx@7.29.7)(@babel/plugin-syntax-typescript@7.29.7)(@oxc-project/types@0.144.0)(@types/node@26.4.0)(@vue/compiler-sfc@3.5.42)(better-sqlite3@13.0.3)(commander@14.0.3)(db0@0.3.4(better-sqlite3@13.0.3))(esbuild@0.28.2)(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.33.0)(magicast@0.5.4)(optionator@0.9.4)(oxc-parser@0.143.0)(rolldown@1.2.4)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.4)(rollup@4.62.4))(rollup@4.62.4)(srvx@0.12.5)(supports-color@10.2.2)(terser@5.50.0)(tsx@4.23.13)(typescript@6.0.3)(vite@8.2.1(@types/node@26.4.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))(vue-tsc@3.3.11(typescript@6.0.3))(yaml@2.9.0) pathe: 2.0.3 optionalDependencies: - '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0(e385fb48e32147193c05e7ca84503583) + '@harlan-zw/nuxt-checkin': 0.2.0(e385fb48e32147193c05e7ca84503583) '@sentry/cloudflare': 10.73.0(@cloudflare/workers-types@5.20260901.1)(wrangler@4.127.1(@cloudflare/workers-types@5.20260901.1)) transitivePeerDependencies: - magic-string diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b3915944..8449a132 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,7 +1,8 @@ minimumReleaseAgeExclude: - - '@harlan-zw/nuxt-checkin@0.1.0' - - '@harlan-zw/nuxt-checkin@0.2.0-alpha.0' - - '@harlan-zw/nuxt-sentry@0.1.5' + - '@harlan-zw/nuxt-checkin@0.2.0' + - '@harlan-zw/nuxt-cf-jobs@0.2.4' + - '@harlan-zw/nuxt-cloudflare@0.4.2' + - '@harlan-zw/nuxt-sentry@0.1.6' - '@harlan-zw/nuxt-checkin' - '@comark/nuxt' - '@comark/vue' @@ -79,11 +80,11 @@ catalog: '@cloudflare/workers-types': ^5.20260901.1 '@comark/nuxt': 0.6.2 '@harlan-zw/comark-content': 0.1.5 - '@harlan-zw/nuxt-cf-jobs': 0.2.3 - '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0 - '@harlan-zw/nuxt-cloudflare': 0.4.1 + '@harlan-zw/nuxt-cf-jobs': 0.2.4 + '@harlan-zw/nuxt-checkin': 0.2.0 + '@harlan-zw/nuxt-cloudflare': 0.4.2 '@harlan-zw/nuxt-dx': 0.1.2 - '@harlan-zw/nuxt-sentry': 0.1.5 + '@harlan-zw/nuxt-sentry': 0.1.6 '@harlan-zw/nuxt-use-query': 0.2.1 '@harlan-zw/nuxt-wide-events': ^0.1.4 '@iconify-json/lucide': ^1.2.128 @@ -159,7 +160,3 @@ allowBuilds: unrs-resolver: true vue-demi: true workerd: true - -peerDependencyRules: - allowedVersions: - '@harlan-zw/nuxt-checkin': 0.2.0-alpha.0 diff --git a/scripts/tools/sentry-observability.mjs b/scripts/tools/sentry-observability.mjs index 9449c87e..cacee35b 100644 --- a/scripts/tools/sentry-observability.mjs +++ b/scripts/tools/sentry-observability.mjs @@ -24,7 +24,7 @@ export function parseSentryIssuesResponse(status, body, tokenSource, sinceIso, l if (status === 401 || status === 403) { const origin = tokenSource ? ` Token came from ${tokenSource}.` : '' return { - _tag: 'missing_observability', + _tag: /** @type {const} */ ('missing_observability'), status, diagnostic: status === 403 ? `Sentry token lacks issue-read permission (HTTP 403).${origin} Use a token with event:read + org:read, such as the one sentry-cli writes to ~/.sentryclirc.` @@ -33,14 +33,14 @@ export function parseSentryIssuesResponse(status, body, tokenSource, sinceIso, l } if (status < 200 || status >= 300) { return { - _tag: 'provider_failure', + _tag: /** @type {const} */ ('provider_failure'), status, diagnostic: `Sentry issues request failed with HTTP ${status}.`, } } if (!Array.isArray(body)) { return { - _tag: 'parse_failure', + _tag: /** @type {const} */ ('parse_failure'), status, diagnostic: 'Sentry issues response was not an array.', } @@ -63,7 +63,7 @@ export function parseSentryIssuesResponse(status, body, tokenSource, sinceIso, l || typeof issue.firstSeen !== 'string' || typeof issue.lastSeen !== 'string') { return { - _tag: 'parse_failure', + _tag: /** @type {const} */ ('parse_failure'), status, diagnostic: 'Sentry issues response contained an invalid issue.', } @@ -88,7 +88,7 @@ export function parseSentryIssuesResponse(status, body, tokenSource, sinceIso, l recurringIssues.push(parsed) } return { - _tag: 'available', + _tag: /** @type {const} */ ('available'), newIssues, recurringIssues, // A full page means Sentry had more to say. Reporting the cap keeps a From 75dfd7281eeb033b3f70506c80f447fd516f7f3c Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Tue, 15 Sep 2026 02:13:24 +1000 Subject: [PATCH 09/11] refactor(checkin): replace health emails with live reports --- .claude/skills/daily-checkin/SKILL.md | 7 +- CRON.md | 3 +- checks/_helpers/collectors.mjs | 9 +- checks/_helpers/observability.mjs | 52 --- checks/_helpers/policy.mjs | 1 - checks/external/health-email.ts | 18 - checks/external/report.ts | 21 ++ docs/shared-checkin.md | 22 +- .../server/api/internal/checkin.get.ts | 22 ++ layers/identity/server/checks/daily-health.ts | 2 +- .../server/tasks/daily-health-check.ts | 86 ----- .../server/utils/daily-health-check.spec.ts | 148 +------- .../server/utils/daily-health-check.ts | 324 +----------------- .../server/utils/daily-health-checkin.ts | 22 +- .../server/tasks/embedding-parity-audit.ts | 4 +- .../server/tasks/refresh-x-engagement.ts | 27 -- .../registry/server/tasks/sync-x-mentions.ts | 28 -- nuxt.config.ts | 2 +- scripts/lib/d1-schema-verifier.ts | 4 +- server/utils/x-api-alert.ts | 156 --------- shared/checkin-external.ts | 2 +- shared/schedule-policy.ts | 1 - test/unit/checkin-route.test.ts | 36 ++ test/unit/daily-checkin-checks.test.ts | 8 +- test/unit/daily-checkin-observability.test.ts | 69 ---- test/unit/external-checkin-collectors.test.ts | 6 +- test/unit/schedule-parity.test.ts | 2 +- test/unit/x-api-alert.test.ts | 151 -------- 28 files changed, 119 insertions(+), 1114 deletions(-) delete mode 100644 checks/external/health-email.ts create mode 100644 checks/external/report.ts create mode 100644 layers/identity/server/api/internal/checkin.get.ts delete mode 100644 layers/identity/server/tasks/daily-health-check.ts delete mode 100644 server/utils/x-api-alert.ts create mode 100644 test/unit/checkin-route.test.ts delete mode 100644 test/unit/x-api-alert.test.ts diff --git a/.claude/skills/daily-checkin/SKILL.md b/.claude/skills/daily-checkin/SKILL.md index a7da8e8a..a52f0de5 100644 --- a/.claude/skills/daily-checkin/SKILL.md +++ b/.claude/skills/daily-checkin/SKILL.md @@ -17,13 +17,12 @@ Produce one read-only morning report that answers: what changed, what broke, wha 3. Verify these gates first: - Read `severity`, `coverage`, and every result. Incomplete coverage cannot support GREEN. - `sentry.skilld` covers the retained unresolved backlog. Missing credentials or partial pagination remain findings. - - `skilld.health-email` verifies the saved report against the current Worker version and a 36-hour age limit. + - `skilld.report` collects live protected health checks and verifies the current Worker version with a five-minute age limit. - If backlog IDs lack archived triage details, fetch their details read-only and append them to the archive. - Any stable non-200 `skilld.dev` front door response is RED. A retry flap is a note. - Read every `ci.workflows` state. `failure` means the gate is broken. `pending` after a prior failure must be followed until complete. `missing` is an observability gap. Never infer overall CI health from only the deploy workflow. - Any `workers.nonOk`, new Sentry issue, failed digest, failed job, stale reserved job, stale scheduled task, or missing expected table needs an explicit verdict. - `d1.migrations.localHead !== d1.migrations.prodHead` is migration drift. Never assume deployment applied D1 migrations. - - `d1.healthEmail` must show one sent operator report per Melbourne date after the feature is deployed. Missing, failed, or stuck `sending` rows are findings. Always report its archived `reasons`; the color alone is insufficient. - `skilld.baseline` result evidence records the window against the daily cadence. `stale` means the routine skipped days; report the `gapHours` figure and compare rates per day, never as overnight step changes. The shared CLI rejects invalid or future state before collecting evidence. Repair the state or supply an explicit `--since` boundary. - Use failed-job `first_failed_at` and `last_failed_at`, plus `d1.recentJobBatches` and `d1.registryMaintenance`, to distinguish an active incident from a recovered burst. Do not call a window clean because the latest batch passed, or active because an older batch failed. - Sentry issues must include their archived permalink and culprit in the proposed action. Archive any extra detail reads. @@ -51,3 +50,7 @@ Produce one read-only morning report that answers: what changed, what broke, wha - Lead with failed probes. Missing data cannot support GREEN. - Compare step changes over equivalent windows. Do not turn cumulative queues or known broken inventory into overnight failures. - Keep the last 30 days of paired JSON and Markdown reports. Never delete `state.json`. + +Record every non-green or incomplete result in the agent tracking issue. +Use stable site, check, and cause identities. Link existing issues and state the next action. +Record verified recovery evidence. The controller owns issue publication; do not create issues from checks or cron jobs. diff --git a/CRON.md b/CRON.md index fa5bf798..2c9bb64d 100644 --- a/CRON.md +++ b/CRON.md @@ -2,7 +2,7 @@ Generated from literal `defineScheduledTask` declarations. Run `pnpm cron:docs` after schedule changes. -Tasks: 18. Unique Cloudflare triggers: 15. +Tasks: 17. Unique Cloudflare triggers: 14. | Task | Cron | Source | | --- | --- | --- | @@ -10,7 +10,6 @@ Tasks: 18. Unique Cloudflare triggers: 15. | `ai-generate-submit` | `15 * * * *` | `layers/registry/server/tasks/ai-generate-submit.ts` | | `ai-ready:cron` | `*/5 * * * *` | `server/tasks/ai-ready-cron.ts` | | `backfill-skill-assets` | `*/5 * * * *` | `layers/registry/server/tasks/backfill-skill-assets.ts` | -| `daily-health-check` | `0 22 * * *` | `layers/identity/server/tasks/daily-health-check.ts` | | `detect-star-surges` | `30 4 * * *` | `layers/registry/server/tasks/detect-star-surges.ts` | | `drain-skill-dirty` | `*/5 * * * *` | `layers/registry/server/tasks/drain-skill-dirty.ts` | | `embedding-parity-audit` | `0 21 * * *` | `layers/registry/server/tasks/embedding-parity-audit.ts` | diff --git a/checks/_helpers/collectors.mjs b/checks/_helpers/collectors.mjs index e708850b..18e9809a 100644 --- a/checks/_helpers/collectors.mjs +++ b/checks/_helpers/collectors.mjs @@ -2,7 +2,7 @@ import { readFileSync } from 'node:fs' import { readdir, readFile } from 'node:fs/promises' import { join } from 'node:path' import { readBoundedResponseText, runCheckCommand } from '@harlan-zw/nuxt-checkin/external' -import { buildHealthEmailQuery, buildWorkersQuery, parseHealthEmailRows, parseWorkflowName, runListArgs, summarizeWorkflowRuns } from './observability.mjs' +import { buildWorkersQuery, parseWorkflowName, runListArgs, summarizeWorkflowRuns } from './observability.mjs' const resources = Object.fromEntries(['github-auth', 'production-ref', 'git', 'deploy', 'ci', 'd1', 'workers'].map(key => [key, {}])) @@ -223,14 +223,11 @@ export function collectD1(context) { ? (await d1Query(`SELECT r.owner, r.repo, rtrim((CASE WHEN EXISTS (SELECT 1 FROM skills s WHERE s.owner = r.owner AND s.repo = r.repo) THEN 'skill ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM user_starred_repos usr WHERE usr.owner = r.owner AND usr.repo = r.repo) THEN 'star ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM skill_subscriptions sub WHERE sub.owner = r.owner AND sub.repo = r.repo) THEN 'subscription ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM collection_skills_v2 cs WHERE cs.owner = r.owner AND cs.repo = r.repo) THEN 'collection ' ELSE '' END) || (CASE WHEN EXISTS (SELECT 1 FROM activity a JOIN install_events ie ON ie.slug = a.owner || '/' || a.name WHERE a.owner = r.owner AND a.repo = r.repo) THEN 'install ' ELSE '' END)) AS reason FROM repos r WHERE ${impactedBrokenReposWhere} ORDER BY r.broken_since DESC`)) : null const syncJobs = has('sync_jobs') - ? (await d1Query(`SELECT name, cron, stale_after_seconds, last_run_at, last_status, last_error FROM sync_jobs WHERE enabled = 1 ORDER BY name`)) + ? (await d1Query(`SELECT name, cron, stale_after_seconds, last_run_at, last_status, last_error FROM sync_jobs WHERE enabled = 1 AND name != 'daily-health-check' ORDER BY name`)) : null const failedJobFingerprints = has('failed_jobs') ? (await d1Query(`SELECT queue, job_type, substr(exception, 1, 160) exception, COUNT(*) count, MIN(failed_at) first_failed_at, MAX(failed_at) last_failed_at FROM failed_jobs WHERE failed_at >= ${sinceSec} GROUP BY queue, job_type, substr(exception, 1, 160) ORDER BY count DESC LIMIT 10`)) : null - const healthEmail = has('daily_health_checks') - ? parseHealthEmailRows((await d1Query(buildHealthEmailQuery()))) - : null const recentJobBatches = has('job_batches') ? (await d1Query(`SELECT id, name, total_jobs, pending_jobs, failed_jobs, created_at, updated_at, finished_at FROM job_batches ORDER BY updated_at DESC LIMIT 10`)) : null @@ -259,7 +256,6 @@ export function collectD1(context) { cost: withXSpend((await d1Query(`SELECT ${costParts.join(', ')}`))[0]), syncJobs, failedJobFingerprints, - healthEmail, recentJobBatches, registryMaintenance, migrations, @@ -273,7 +269,6 @@ export function collectD1(context) { 'sync_jobs', 'jobs', 'failed_jobs', - 'daily_health_checks', ].filter(table => !has(table)), } }) diff --git a/checks/_helpers/observability.mjs b/checks/_helpers/observability.mjs index 30b4327e..f768a293 100644 --- a/checks/_helpers/observability.mjs +++ b/checks/_helpers/observability.mjs @@ -1,7 +1,3 @@ -function record(value) { - return value !== null && typeof value === 'object' && !Array.isArray(value) ? value : null -} - const PRODUCTION_REF = 'origin/main' const isMigration = file => /^\d.*\.sql$/.test(file) @@ -204,51 +200,3 @@ export function summarizeWorkflowRuns(rows, requiredWorkflowNames) { } }) } - -function stringArray(value, field) { - if (!Array.isArray(value) || value.some(item => typeof item !== 'string')) - throw new Error(`daily health check ${field} must be a string array`) - return value -} - -// The gate is one sent operator report per Melbourne date, so the read has to -// show enough days to verify continuity. The old two-row page could only ever -// verify the two newest dates, which is how a gap three days back stayed -// invisible. Melbourne runs up to one day ahead of UTC, so eight UTC days is -// the smallest bound that always covers seven Melbourne dates. `report_date` -// is the table's primary key, so a row cap equal to the window can never hide -// an in-window date. -export function buildHealthEmailQuery() { - return `SELECT report_date, health_status, delivery_status, recipient, sent_at, error, summary_json FROM daily_health_checks WHERE report_date >= date('now', '-8 days') ORDER BY report_date DESC LIMIT 8` -} - -export function parseHealthEmailRows(rows) { - return rows.map((row) => { - const summary = (() => { - try { - return record(JSON.parse(row.summary_json)) - } - catch { - throw new Error(`daily health check ${row.report_date} has invalid summary_json`) - } - })() - if (!summary) - throw new Error(`daily health check ${row.report_date} summary_json must contain an object`) - const window = record(summary.window) - if (!window) - throw new Error(`daily health check ${row.report_date} summary_json is missing its window`) - - return { - reportDate: row.report_date, - healthStatus: row.health_status, - deliveryStatus: row.delivery_status, - recipient: row.recipient, - sentAt: row.sent_at, - error: row.error, - reasons: stringArray(summary.reasons, 'reasons'), - warnings: stringArray(summary.warnings, 'warnings'), - window, - checkin: summary.checkin, - } - }) -} diff --git a/checks/_helpers/policy.mjs b/checks/_helpers/policy.mjs index 2ca44ecc..d5b133fa 100644 --- a/checks/_helpers/policy.mjs +++ b/checks/_helpers/policy.mjs @@ -30,6 +30,5 @@ export function evaluateD1(data, now, since) { || data.migrations.localHead !== data.migrations.prodHead || ['newly_broken_repos_impacted', 'skill_sync_failures', 'stale_dirty_skills', 'ai_stuck', 'ai_failed', 'embeddings_failed', 'embeddings_stuck', 'failed_jobs'].some(key => data.pipeline[key] > 0) || data.syncJobs?.some(job => job.last_status === 'partial' && job.last_run_at >= since.getTime() / 1000) - || data.healthEmail?.[0]?.deliveryStatus !== 'sent' return warning ? warn('Database evidence needs attention.', data) : pass(data) } diff --git a/checks/external/health-email.ts b/checks/external/health-email.ts deleted file mode 100644 index 37500bb5..00000000 --- a/checks/external/health-email.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { checkReport, defineExternalCheck, unavailable } from '@harlan-zw/nuxt-checkin/external' -import { collectD1, collectDeploy } from '../_helpers/collectors.mjs' - -export default defineExternalCheck({ - id: 'skilld.health-email', - async run(context) { - const deployment = (await collectDeploy(context)).latest?.versionId - const healthReport = (await collectD1(context)).healthEmail?.[0]?.checkin - return deployment - ? checkReport(healthReport, { - now: context.clock(), - identity: { site: 'skilld.dev', environment: 'production', deployment }, - required: ['skilld.daily-health', 'skilld.daily-health-coverage'], - maxAgeMs: 36 * 60 * 60 * 1000, - }) - : unavailable('Deployed Worker identity is unavailable.') - }, -}) diff --git a/checks/external/report.ts b/checks/external/report.ts new file mode 100644 index 00000000..4a578d6f --- /dev/null +++ b/checks/external/report.ts @@ -0,0 +1,21 @@ +import { defineExternalCheck, defineReportCheck, unavailable } from '@harlan-zw/nuxt-checkin/external' +import { collectDeploy } from '../_helpers/collectors.mjs' + +export default defineExternalCheck({ + id: 'skilld.report', + async run(context) { + const deployment = (await collectDeploy(context)).latest?.versionId + if (!deployment) + return unavailable('Deployed Worker identity is unavailable.') + return defineReportCheck({ + id: 'skilld.report', + url: 'https://skilld.dev/api/internal/checkin', + tokenEnv: 'NUXT_CHECKIN_TOKEN', + deploymentEnv: 'CHECKIN_DEPLOYMENT', + site: 'skilld.dev', + environment: 'production', + required: ['skilld.daily-health', 'skilld.daily-health-coverage'], + maxAgeMs: 300_000, + }).run({ ...context, event: { rootDir: context.rootDir, since: context.since, previous: context.previous, clock: context.clock, env: { ...context.env, CHECKIN_DEPLOYMENT: deployment } } }) + }, +}) diff --git a/docs/shared-checkin.md b/docs/shared-checkin.md index 7e58a277..d079a993 100644 --- a/docs/shared-checkin.md +++ b/docs/shared-checkin.md @@ -1,23 +1,20 @@ # Daily health check -The operator email uses the public Nuxt Check-in runner. +The protected report route uses the public Nuxt Check-in runner. The module discovers checks in `layers/identity/server/checks` during the build. The existing summary builder and health evaluator keep their current policy. Both checks share one summary collection and one observation time. The independent required list detects missing check files. A run has a 60-second deadline. -If collection fails, the sender fails before claiming or sending an email. -The scheduled task records that failure through its existing reporting path. +If collection fails, the report records incomplete coverage. Active reads may continue because D1 cannot cancel an active query. -Each saved summary includes the versioned check report, deployment identity, severity, and evidence coverage. -The email shows coverage in HTML and text. +Each report includes deployment identity, severity, evidence coverage, and the full health summary. A known failure stays RED when another source is unavailable. Unavailable evidence can never produce GREEN. -Detailed evidence stays in the existing private report storage and operator email. +Detailed evidence stays in the private report and agent archive. -Recipients, email schedules, delivery claims, retry behavior, and pause gates stay unchanged. The agent keeps its 07:40 daily check-in and removes its separate 06:20 Sentry run. `send-digests` sends subscriber content, not system health emails. Its outcome already enters the daily summary. @@ -31,7 +28,7 @@ The lockfile resolves registry packages. ## External daily check-in -The external daily script runs the public [Sentry](https://sentry.io) backlog check with the saved health report. +The external daily script runs the public [Sentry](https://sentry.io) backlog check with the live health report. It validates report age, required checks, site, environment, and the deployed Worker version. Missing credentials or stale reports produce unavailable evidence. Sentry credentials stay in the existing external locations. @@ -49,9 +46,16 @@ Run `pnpm checkin --save` to prepare and execute registered external checks. The shared CLI writes daily archives and keeps successful state boundaries. Same-day reruns preserve the morning baseline. Read `results[].result.evidence` for each registered check. -The `skilld.database` result retains inventory, activity, pipeline, cost, and operator-email evidence. +The `skilld.database` result retains inventory, activity, pipeline, and cost evidence. The `skilld.sentry-details` result retains new and recurring issue details. The published `@harlan-zw/nuxt-checkin@0.2.0` supplies the CLI. The `skilld.baseline` result records fresh or stale comparison windows. Invalid saved state stops the CLI before collection and never advances the saved baseline. + +The agent tracking issue replaces system-health email delivery. +Set the private `NUXT_CHECKIN_TOKEN` Worker secret and the same token in the agent environment. +The protected `/api/internal/checkin` route runs the existing health checks on demand. +The report must match the deployed Worker and be less than five minutes old. +The daily health email cron and X API failure emails are removed. Customer digests remain unchanged. +Historical email records stay in D1, but checks no longer depend on them. diff --git a/layers/identity/server/api/internal/checkin.get.ts b/layers/identity/server/api/internal/checkin.get.ts new file mode 100644 index 00000000..4ec811c8 --- /dev/null +++ b/layers/identity/server/api/internal/checkin.get.ts @@ -0,0 +1,22 @@ +import { resolveCloudflareBindings } from '@harlan-zw/nuxt-cloudflare/bindings' +import { createError, defineEventHandler, getHeader, setHeader } from 'h3' +import { useRuntimeConfig } from 'nitropack/runtime' +import { buildDailyHealthCheck, frontDoorFetcher } from '../../utils/daily-health-check' +import { runDailyHealthChecks } from '../../utils/daily-health-checkin' + +export default defineEventHandler(async (event) => { + setHeader(event, 'cache-control', 'no-store') + const config = useRuntimeConfig(event) + if (!config.checkinToken || getHeader(event, 'authorization') !== `Bearer ${config.checkinToken}`) + throw createError({ statusCode: 401, message: 'Check-in authorization failed.' }) + const env = resolveCloudflareBindings(event.context) + const deployment = env?.CF_VERSION_METADATA?.id + if (!env?.DB || !deployment) + throw createError({ statusCode: 503, message: 'Check-in bindings or deployment identity are missing.' }) + return runDailyHealthChecks(env.DB, (db, options) => buildDailyHealthCheck(db, { + ...options, + fetcher: frontDoorFetcher(env), + githubToken: env.GITHUB_TOKEN, + workerVersion: deployment, + }), new Date(), deployment) +}) diff --git a/layers/identity/server/checks/daily-health.ts b/layers/identity/server/checks/daily-health.ts index 7138bc4e..53b4bb34 100644 --- a/layers/identity/server/checks/daily-health.ts +++ b/layers/identity/server/checks/daily-health.ts @@ -6,7 +6,7 @@ export default defineCheck({ id: 'skilld.daily-health', async run(context) { const summary = await collectDailyHealth(context) - const evidence = { reasons: summary.reasons } + const evidence = { ...summary } const result = summary.status === 'RED' ? fail(summary.reasons.join(' '), evidence) : summary.status === 'AMBER' ? warn(summary.reasons.join(' '), evidence) : pass(evidence) diff --git a/layers/identity/server/tasks/daily-health-check.ts b/layers/identity/server/tasks/daily-health-check.ts deleted file mode 100644 index 3d500396..00000000 --- a/layers/identity/server/tasks/daily-health-check.ts +++ /dev/null @@ -1,86 +0,0 @@ -/// - -import { resolveCloudflareBindings } from '@harlan-zw/nuxt-cloudflare/bindings' -import { runObservedScheduledTask } from '~~/server/utils/scheduled-run' -import { reportJobRun } from '~~/server/utils/sync-job-reporter' -import { observedSchedulePolicy } from '#shared/schedule-policy' -import { buildDailyHealthCheck, frontDoorFetcher, sendDailyHealthCheck } from '../utils/daily-health-check' -import { sendEmailWithEnv } from '../utils/email' - -const CRON = '0 22 * * *' -const DEFAULT_OPERATOR_EMAIL = 'harlan@harlanzw.com' - -function workerVersion(env: Cloudflare.Env): string | null { - const metadata = env.CF_VERSION_METADATA - if (!metadata?.id) - return null - return `${metadata.id.slice(0, 8)} deployed ${metadata.timestamp}` -} - -export default defineScheduledTask({ - name: 'daily-health-check', - cron: '0 22 * * *', - description: 'Send the operator a daily health check for skilld traffic, delivery, pipeline freshness, job failures, and AI cost.', - async run({ context }) { - if (import.meta.dev) - return { result: { _tag: 'SkippedInDev' as const } } - - const env = resolveCloudflareBindings(context) - const db = env?.DB as D1Database | undefined - if (!env || !db) { - emitOperationalEvent(createWideEvent({ operation: 'daily-health-check', outcome: 'binding-missing' })) - return { result: { _tag: 'MissingBindings' as const } } - } - - return await runObservedScheduledTask({ - db, - env, - context, - policy: observedSchedulePolicy('daily-health-check'), - }, async () => { - const config = useRuntimeConfig() - const to = String(config.healthCheckNotifyTo || DEFAULT_OPERATOR_EMAIL).trim() - const from = config.email.from as EmailAddress - const now = new Date() - const startedAt = Date.now() - - const attempt = await sendDailyHealthCheck(db, { - now, - deployment: env.CF_VERSION_METADATA?.id, - to, - build: (database, options) => buildDailyHealthCheck(database, { - ...options, - fetcher: frontDoorFetcher(env), - githubToken: env.GITHUB_TOKEN, - workerVersion: workerVersion(env), - }), - send: input => sendEmailWithEnv(env, { ...input, from }), - }) - .then(result => ({ _tag: 'Completed' as const, result })) - .catch(error => ({ - _tag: 'InfrastructureFailure' as const, - error: error instanceof Error ? error.message : String(error), - })) - - const deliveryFailed = attempt._tag === 'Completed' - && (attempt.result._tag === 'SendFailed' || attempt.result._tag === 'Uncertain') - const error = attempt._tag === 'InfrastructureFailure' - ? attempt.error - : deliveryFailed - ? attempt.result.error - : null - - await reportJobRun(db, 'daily-health-check', { - cron: CRON, - status: error ? 'error' : 'ok', - durationMs: Date.now() - startedAt, - error, - staleAfterSeconds: 36 * 60 * 60, - }) - - if (attempt._tag === 'InfrastructureFailure') - throw new Error(attempt.error) - return { result: attempt } - }) - }, -}) diff --git a/layers/identity/server/utils/daily-health-check.spec.ts b/layers/identity/server/utils/daily-health-check.spec.ts index ceb2b138..44378433 100644 --- a/layers/identity/server/utils/daily-health-check.spec.ts +++ b/layers/identity/server/utils/daily-health-check.spec.ts @@ -1,6 +1,6 @@ import type { DailyHealthCheckSummary } from './daily-health-check' import Database from 'better-sqlite3' -import { afterEach, describe, expect, it, vi } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { SCHEDULE_POLICY } from '#shared/schedule-policy' import { buildDailyHealthCheck, @@ -8,9 +8,6 @@ import { evaluateDailyHealthStatus, frontDoorFetcher, loadFrontDoor, - renderDailyHealthCheckHtml, - renderDailyHealthCheckText, - sendDailyHealthCheck, } from './daily-health-check' function summary(overrides: Partial = {}): DailyHealthCheckSummary { @@ -98,19 +95,6 @@ function summary(overrides: Partial = {}): DailyHealthC function createDb() { const sqlite = new Database(':memory:') - sqlite.exec(` - CREATE TABLE daily_health_checks ( - report_date TEXT PRIMARY KEY, - health_status TEXT NOT NULL, - delivery_status TEXT NOT NULL, - recipient TEXT NOT NULL, - claimed_at INTEGER NOT NULL, - sent_at INTEGER, - message_id TEXT, - error TEXT, - summary_json TEXT NOT NULL - ) - `) const db = { prepare(sql: string) { @@ -573,136 +557,6 @@ describe('cronPeriodSeconds', () => { }) }) -describe('daily health rendering', () => { - it('renders text and escapes untrusted HTML', () => { - const input = summary({ - reasons: ['Pipeline '], - pipeline: { - ...summary().pipeline, - failedJobDetails: [{ queue: 'sync', jobType: '