From 05dd2ddd49871b13b1e43cb4324261ab8e1e06b8 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Sat, 23 May 2026 10:31:23 +0200 Subject: [PATCH] feat: per-provider availability + slack alerts on offline/recovered Provider lifecycle When prom returns null for a provider's p50/p90/p99 queries this cycle, spec.ts now tags the augmented entry with availability='unavailable' instead of leaving it as a row of zeros that ranks #1 on lower-is-better benches (the 0slot / cardano UX bug). LedgerTable renders the row at the bottom with a soft amber pill 'Currently unavailable' + 'Awaiting next successful scrape'. Stats helpers (computeFieldStats, rankProviders, benchmark-body, chain-headings-summary, bench-template, citation) all exclude unavailable rows so Best/Median/Spread no longer collapse to 0. Slack alerting New /api/cron/health-check route. For each live bench it asks prom two questions per provider: 'did you receive samples in the last 10 min?' and the same query offset 6 min back. The edge (was-live xor is-live) triggers a slack post via SLACK_WEBHOOK_URL env. No external state - we use prom's own series history as the source of truth. Operator setup (one-time, in Vercel project settings) - SLACK_WEBHOOK_URL : the incoming-webhook url (channel of choice) - CRON_SECRET : random string, used to gate the cron route vercel.json schedules the route every 5 minutes. Vercel injects the Authorization: Bearer $CRON_SECRET header automatically. --- src/app/api/cron/health-check/route.ts | 153 ++++++++++++++++++++++ src/components/benchmark-body.tsx | 2 +- src/components/chain-headings-summary.tsx | 2 +- src/components/ledger-table.tsx | 138 +++++++++++-------- src/lib/bench-template.ts | 2 +- src/lib/citation.ts | 4 +- src/lib/providers.ts | 4 +- src/lib/spec.ts | 11 +- src/lib/stats.ts | 9 +- src/types/benchmark.ts | 17 +++ vercel.json | 9 ++ 11 files changed, 284 insertions(+), 67 deletions(-) create mode 100644 src/app/api/cron/health-check/route.ts create mode 100644 vercel.json diff --git a/src/app/api/cron/health-check/route.ts b/src/app/api/cron/health-check/route.ts new file mode 100644 index 00000000..07594422 --- /dev/null +++ b/src/app/api/cron/health-check/route.ts @@ -0,0 +1,153 @@ +import { NextResponse, type NextRequest } from "next/server"; +import { getBenchmarkSlugs } from "@/data/benchmarks"; +import { getSpecs } from "@/lib/spec"; +import { Prometheus } from "@/lib/prometheus"; + +export const runtime = "nodejs"; +// Always read live state from prom. ISR cache here would defeat the +// alert. Vercel calls this from the cron entry in vercel.json every 5 min. +export const dynamic = "force-dynamic"; + +/** + * Cron health-check + slack alerter. + * + * Walks every live bench, asks prometheus two questions per provider: + * 1. Was the provider returning data 6 minutes ago? (prom range query) + * 2. Is it returning data now? + * + * Transitions trigger a slack message. Without state storage we use prom + * itself as the source-of-truth: comparing "now" vs "6 min ago" gives us + * the same up/down/recovered edges Alertmanager would compute, with one + * fewer service to run. + * + * Wiring (operator): + * 1. Set `SLACK_WEBHOOK_URL` in vercel env (production scope only). + * 2. Set `CRON_SECRET` in vercel env, used to gate this route. + * 3. The vercel.json cron config posts here every 5 minutes with + * `Authorization: Bearer ${CRON_SECRET}`. + * + * If `SLACK_WEBHOOK_URL` is unset the route runs in dry-mode: it still + * computes the transitions and returns them as JSON, useful for testing + * without spamming the channel. + */ + +const LOOKBACK_SECONDS = 360; // 6 minutes. covers the 5-minute cron + slack delivery slack. +const PROVIDER_FRESH_THRESHOLD_SECONDS = 600; // 10 minutes. window during which we still treat the provider as live. + +type ProviderState = { + benchSlug: string; + benchTitle: string; + providerSlug: string; + providerName: string; + wasLive: boolean; + isLive: boolean; +}; + +function isAuthorized(req: NextRequest): boolean { + const secret = process.env.CRON_SECRET; + if (!secret) return true; // dev mode, no auth required + const auth = req.headers.get("authorization") ?? ""; + return auth === `Bearer ${secret}`; +} + +export async function GET(req: NextRequest) { + if (!isAuthorized(req)) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + + const promUrl = process.env.PROMETHEUS_URL?.trim(); + if (!promUrl) { + return NextResponse.json({ error: "PROMETHEUS_URL not set" }, { status: 500 }); + } + + const prom = new Prometheus(promUrl); + const slugs = await getBenchmarkSlugs(); + const specs = await getSpecs(); + const liveSpecs = specs.filter( + (s) => slugs.includes(s.slug) && s.status === "live", + ); + + const transitions: ProviderState[] = []; + + for (const spec of liveSpecs) { + for (const provider of spec.providers) { + const q = provider.queries?.p50; + if (!q) continue; + + // For the freshness probe we just need a metric name to ask prom + // "do you have any sample within X seconds for this label set". + // The trick: `count_over_time(metric{...}[Xs])` returns >0 when + // prom received at least one sample in the window, 0 otherwise. + const metricName = extractFirstMetric(q); + const labelSelector = extractLabelSelector(q); + if (!metricName) continue; + + const recentQ = `count_over_time(${metricName}${labelSelector}[${PROVIDER_FRESH_THRESHOLD_SECONDS}s])`; + const pastQ = `count_over_time(${metricName}${labelSelector}[${PROVIDER_FRESH_THRESHOLD_SECONDS}s] offset ${LOOKBACK_SECONDS}s)`; + + const [now, past] = await Promise.all([ + prom.scalar(recentQ).catch(() => null), + prom.scalar(pastQ).catch(() => null), + ]); + + const isLive = (now ?? 0) > 0; + const wasLive = (past ?? 0) > 0; + + if (isLive !== wasLive) { + transitions.push({ + benchSlug: spec.slug, + benchTitle: spec.title, + providerSlug: provider.slug, + providerName: provider.name, + wasLive, + isLive, + }); + } + } + } + + const webhook = process.env.SLACK_WEBHOOK_URL?.trim(); + const sent: { provider: string; text: string }[] = []; + if (webhook && transitions.length > 0) { + for (const t of transitions) { + const emoji = t.isLive ? "✅" : "🔴"; + const verb = t.isLive ? "back online" : "offline"; + const text = `${emoji} *${t.providerName}* ${verb} on \`${t.benchSlug}\`\nbench: `; + try { + await fetch(webhook, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text }), + }); + sent.push({ provider: `${t.benchSlug}/${t.providerSlug}`, text }); + } catch { + // best effort - don't break the cron on a single slack failure + } + } + } + + return NextResponse.json({ + checked: liveSpecs.length, + transitions: transitions.length, + sent: sent.length, + dryRun: !webhook, + transitionsList: transitions, + }); +} + +// Pull the first raw metric identifier out of a promql query. Mirrors the +// helper in lib/prometheus.ts but specialised to also capture the label +// selector that follows so we can reuse it in the freshness probe. +function extractFirstMetric(q: string): string | null { + const m = /\b([a-zA-Z_:][a-zA-Z0-9_:]*)\s*\{/.exec(q); + if (!m) { + const bare = /\b([a-zA-Z_:][a-zA-Z0-9_:]*)\b/.exec(q); + return bare ? bare[1] : null; + } + return m[1]; +} + +function extractLabelSelector(q: string): string { + const m = /\{[^{}]*\}/.exec(q); + return m ? m[0] : ""; +} diff --git a/src/components/benchmark-body.tsx b/src/components/benchmark-body.tsx index c1e25dc6..16df0a1a 100644 --- a/src/components/benchmark-body.tsx +++ b/src/components/benchmark-body.tsx @@ -65,7 +65,7 @@ function syncParam( function summarize(b: Benchmark | undefined): ChainMeta | null { if (!b) return null; - const live = b.results.filter((r) => r.ms.p50 > 0); + const live = b.results.filter((r) => r.availability !== "unavailable" && r.ms.p50 > 0); if (live.length === 0) return { providers: 0, metric: b.metric }; const sorted = [...live].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 diff --git a/src/components/chain-headings-summary.tsx b/src/components/chain-headings-summary.tsx index f6d6aaa4..8ae86ad7 100644 --- a/src/components/chain-headings-summary.tsx +++ b/src/components/chain-headings-summary.tsx @@ -18,7 +18,7 @@ import { fmtUnit } from "@/lib/format"; * No-op when the bench has no live data yet. */ export function ChainHeadingsSummary({ benchmark }: { benchmark: Benchmark }) { - const liveResults = benchmark.results.filter((r) => r.ms.p50 > 0); + const liveResults = benchmark.results.filter((r) => r.availability !== "unavailable" && r.ms.p50 > 0); if (liveResults.length === 0) return null; // Render for benches whose providers ARE chains - currently the diff --git a/src/components/ledger-table.tsx b/src/components/ledger-table.tsx index 146daf73..72ada369 100644 --- a/src/components/ledger-table.tsx +++ b/src/components/ledger-table.tsx @@ -23,11 +23,16 @@ type Props = { export function LedgerTable({ benchmark }: Props) { const { results, unit, extras } = benchmark; const secondary = results[0]?.secondary?.label; - const sorted = [...results].sort( - benchmark.higherIsBetter - ? (a, b) => b.ms.p50 - a.ms.p50 - : (a, b) => a.ms.p50 - b.ms.p50, - ); + // Sort by p50 then push unavailable providers to the bottom. Without + // the secondary sort they'd land at rank #1 on lower-is-better benches + // because their placeholder p50 is 0 - which is what made 0slot, then + // cardano, show up as "fastest" in the recent SERP screenshots. + const sorted = [...results].sort((a, b) => { + const aOff = a.availability === "unavailable" ? 1 : 0; + const bOff = b.availability === "unavailable" ? 1 : 0; + if (aOff !== bOff) return aOff - bOff; + return benchmark.higherIsBetter ? b.ms.p50 - a.ms.p50 : a.ms.p50 - b.ms.p50; + }); const colors = buildProviderColors(results); const allSeries = Object.values(extras.series24h).flat(); @@ -127,13 +132,14 @@ function Row({ sparkMax: number; color: string; }) { + const isOffline = r.availability === "unavailable"; const deltaPct = fieldP50 > 0 ? ((r.ms.p50 - fieldP50) / fieldP50) * 100 : 0; const deltaSign = deltaPct > 0 ? "+" : deltaPct < 0 ? "−" : "±"; // Inline p50 bar width relative to the field max const barPct = Math.max(2, (r.ms.p50 / maxP50) * 100); return ( - + {/* Color accent. left edge of row */} @@ -154,7 +160,7 @@ function Row({ {isRegion(r.slug) ? ( {r.name} @@ -162,69 +168,89 @@ function Row({ {r.name} )} - {r.tag && ( + {r.tag && !isOffline && ( {r.tag} )} - {r.type && ( + {isOffline && ( + + + Currently unavailable + + )} + {r.type && !isOffline && ( )} - {/* p50 with inline data bar */} - - - - - {fmtUnit(r.ms.p50, unit)} - - - - - {fmtUnit(r.ms.p90, unit)} - - - {fmtUnit(r.ms.p99, unit)} - - - {fmtUnit(r.ms.mean, unit)} - - - {fieldP50 > 0 ? `${deltaSign}${Math.abs(deltaPct).toFixed(0)}%` : "-"} - - - {r.successRate.toFixed(2)}% - - - - - - - {hasSecondary && ( - - {r.secondary?.value ?? "-"} + {isOffline ? ( + + Awaiting next successful scrape + ) : ( + <> + {/* p50 with inline data bar */} + + + + + {fmtUnit(r.ms.p50, unit)} + + + + + {fmtUnit(r.ms.p90, unit)} + + + {fmtUnit(r.ms.p99, unit)} + + + {fmtUnit(r.ms.mean, unit)} + + + {fieldP50 > 0 ? `${deltaSign}${Math.abs(deltaPct).toFixed(0)}%` : "-"} + + + {r.successRate.toFixed(2)}% + + + + + + + {hasSecondary && ( + + {r.secondary?.value ?? "-"} + + )} + )} ); diff --git a/src/lib/bench-template.ts b/src/lib/bench-template.ts index 4a1cd08a..e3c2d2d3 100644 --- a/src/lib/bench-template.ts +++ b/src/lib/bench-template.ts @@ -38,7 +38,7 @@ const TEMPLATE_RE = /\{\{\s*([a-z][a-z0-9_]*)(?::([a-z0-9-]+))?\s*\}\}/gi; export function renderTemplate(text: string, benchmark: Benchmark): string { if (!text || text.indexOf("{{") === -1) return text; - const liveResults = benchmark.results.filter((r) => r.ms.p50 > 0); + const liveResults = benchmark.results.filter((r) => r.availability !== "unavailable" && r.ms.p50 > 0); const sorted = [...liveResults].sort((a, b) => benchmark.higherIsBetter ? b.ms.p50 - a.ms.p50 : a.ms.p50 - b.ms.p50 ); diff --git a/src/lib/citation.ts b/src/lib/citation.ts index d0cc56c5..f1918f6d 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -10,7 +10,7 @@ import { fmtUnit } from "@/lib/format"; /** Median value of the benchmark (the field shown in the headline). */ export function fieldValue(b: Benchmark): number | null { if (b.status !== "live") return null; - const live = b.results.filter((r) => r.ms.p50 > 0); + const live = b.results.filter((r) => r.availability !== "unavailable" && r.ms.p50 > 0); if (live.length === 0) return null; const sorted = [...live].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 @@ -21,7 +21,7 @@ export function fieldValue(b: Benchmark): number | null { /** Who is currently #1 on this benchmark, if any. */ export function leader(b: Benchmark): { name: string; slug: string; value: number } | null { if (b.status !== "live") return null; - const live = b.results.filter((r) => r.ms.p50 > 0); + const live = b.results.filter((r) => r.availability !== "unavailable" && r.ms.p50 > 0); if (live.length === 0) return null; const sorted = [...live].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 diff --git a/src/lib/providers.ts b/src/lib/providers.ts index f2827f1d..02a7d484 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -31,7 +31,9 @@ export type ProviderProfile = { }; function rankProviders(b: Benchmark): ProviderResult[] { - const live = b.results.filter((r) => r.ms.p50 > 0); + const live = b.results.filter( + (r) => r.availability !== "unavailable" && r.ms.p50 > 0, + ); return [...live].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, ); diff --git a/src/lib/spec.ts b/src/lib/spec.ts index 41c3ea14..26e1c5c9 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -189,10 +189,14 @@ async function specToBenchmark( // return data this cycle. Without this, providers with transiently // missing Prom data fall out of `getProviders()` entirely → their // /products/ page 404s and they disappear from the sitemap. - // We append them as zero-valued entries; `rankProviders` already - // filters p50<=0 out of the leaderboard, and the product page shows - // them as "awaiting samples" — same UX as a brand-new draft bench. + // + // These entries are tagged `availability: "unavailable"` so the + // leaderboard renders a soft offline pill ("Currently unavailable") + // instead of a row of 0 ms / 0% that misleads readers into thinking + // the provider is genuinely the fastest. Mark live entries explicitly + // too so a missing `availability` field always reads as "unknown". const liveSlugs = new Set(live.results.map((r) => r.slug.toLowerCase())); + for (const r of live.results) r.availability = "live"; for (const p of spec.providers) { if (liveSlugs.has(p.slug.toLowerCase())) continue; live.results.push({ @@ -203,6 +207,7 @@ async function specToBenchmark( ms: { p50: 0, p90: 0, p99: 0, mean: 0 }, successRate: 0, secondary: p.secondary, + availability: "unavailable", }); } // Resolve {{p50:slug}} / {{best_name}} / {{count}} etc. placeholders diff --git a/src/lib/stats.ts b/src/lib/stats.ts index 9d4d5771..5bbf2bfe 100644 --- a/src/lib/stats.ts +++ b/src/lib/stats.ts @@ -14,8 +14,13 @@ export function computeFieldStats(results: ProviderResult[]): { tailMax: number; tailSpread: number; } { - const p50s = results.map((r) => r.ms.p50); - const p99s = results.map((r) => r.ms.p99); + // Drop "currently unavailable" providers from every aggregate. Their + // placeholder values are 0 and would otherwise drag the Best stat to + // 0 ms, the Spread tailMin to 0 (which kills the ratio), and the + // Median toward the lower half of the field. + const live = results.filter((r) => r.availability !== "unavailable"); + const p50s = live.map((r) => r.ms.p50); + const p99s = live.map((r) => r.ms.p99); const fieldMin = p50s.length ? Math.min(...p50s) : 0; const fieldMax = p50s.length ? Math.max(...p50s) : 0; diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index e8f92f85..009ae943 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -6,6 +6,18 @@ export type ProviderType = "protocol" | "aggregator" | "intent" | "relay"; +/** + * Per-provider data availability. Drives whether the leaderboard renders + * the numbers or a "currently unavailable" stub. + * + * - `live` : fresh data, all the latency aggregates are real. + * - `unavailable` : the underlying source (provider API, the harness + * that scrapes it, the prom job) is not delivering + * samples right now. Numbers are zero placeholders - + * show a soft offline pill, not 0 ms. + */ +export type ProviderAvailability = "live" | "unavailable"; + export type ProviderResult = { name: string; slug: string; @@ -18,6 +30,11 @@ export type ProviderResult = { /** Per-provider sample count over the run window. */ sampleSize?: number; secondary?: { label: string; value: string }; + /** Defaults to "live" when the provider returns numbers; the spec + * loader sets "unavailable" when prom has no data for the p50 / p90 / + * p99 queries so the UI can render a soft offline state instead of + * zero values. */ + availability?: ProviderAvailability; }; export type RegionPoint = { diff --git a/vercel.json b/vercel.json new file mode 100644 index 00000000..6165ffa5 --- /dev/null +++ b/vercel.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "crons": [ + { + "path": "/api/cron/health-check", + "schedule": "*/5 * * * *" + } + ] +}