diff --git a/src/app/api/citable/route.ts b/src/app/api/citable/route.ts index 39b16f8b..3340cb98 100644 --- a/src/app/api/citable/route.ts +++ b/src/app/api/citable/route.ts @@ -1,15 +1,33 @@ import { NextResponse } from "next/server"; import { getBenchmarks } from "@/data/benchmarks"; import { SITE } from "@/data/site"; +import { AllBenchmarksDraftError } from "@/lib/spec"; import { citeBundle, fieldValue, leader, headlineSentence } from "@/lib/citation"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; export const runtime = "nodejs"; export const revalidate = 60; +/** Short 503 with a Retry-After hint, served when the aggregator has + * no live snapshot to surface (Prom blackout + cold KV). Beats serving + * an all-draft index that downstream LLM agents would treat as truth. */ +function unavailable(): NextResponse { + return NextResponse.json( + { error: "benchmarks_unavailable", retryAfterSec: 60 }, + { + status: 503, + headers: { + "cache-control": "no-store", + "retry-after": "60", + "access-control-allow-origin": "*", + }, + }, + ); +} + /** * Flat machine-readable index of every citable benchmark. Designed to be - * the **first** endpoint an AI agent or journalist crawls - gives them + * the **first** endpoint an AI agent or journalist crawls. Gives them * everything they need to decide whether to deep-link to a specific bench. * * License is intentionally surfaced per-row so downstream agents can @@ -19,7 +37,15 @@ export async function GET(req: Request) { const r = rateLimit(clientKey(req, "citable"), 60, 60); if (!r.ok) return tooManyRequests(r.retryAfterSec); - const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); + let benches; + try { + benches = (await getBenchmarks()).filter( + (b) => b.editorialStatus === "live", + ); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) return unavailable(); + throw err; + } const data = benches.map((b) => { const top = leader(b); return { diff --git a/src/app/api/llm-context/route.ts b/src/app/api/llm-context/route.ts index d4414fb1..5ba13994 100644 --- a/src/app/api/llm-context/route.ts +++ b/src/app/api/llm-context/route.ts @@ -1,5 +1,6 @@ import { getBenchmarks } from "@/data/benchmarks"; import { SITE } from "@/data/site"; +import { AllBenchmarksDraftError } from "@/lib/spec"; import { fmtUnit } from "@/lib/format"; import { fieldValue, headlineSentence, leader } from "@/lib/citation"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; @@ -26,7 +27,25 @@ export async function GET(req: Request) { return new Response(await tooMany.text(), { status: tooMany.status, headers: tooMany.headers }); } - const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); + let benches; + try { + benches = (await getBenchmarks()).filter( + (b) => b.editorialStatus === "live", + ); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) { + return new Response("benchmarks_unavailable\n", { + status: 503, + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + "retry-after": "60", + "access-control-allow-origin": "*", + }, + }); + } + throw err; + } const now = new Date().toISOString(); const lines: string[] = []; diff --git a/src/app/benchmarks/[slug]/page.tsx b/src/app/benchmarks/[slug]/page.tsx index cf34a093..059ff60b 100644 --- a/src/app/benchmarks/[slug]/page.tsx +++ b/src/app/benchmarks/[slug]/page.tsx @@ -5,7 +5,7 @@ import Link from "next/link"; import { ArrowUpRight, ChevronDown } from "lucide-react"; import { BackLink } from "@/components/back-link"; import { nonAllValues } from "@/lib/dimensions"; -import { getBenchmark, getBenchmarks } from "@/data/benchmarks"; +import { getBenchmark, getBenchmarksSafe } from "@/data/benchmarks"; import { Pill } from "@/components/pill"; import { BenchmarkBody } from "@/components/benchmark-body"; import { BenchmarkBodySkeleton } from "@/components/benchmark-body-skeleton"; @@ -150,7 +150,7 @@ export default async function BenchmarkPage({ // /api/bench/[slug]/variant when a tab is flipped (per-variant // unstable_cache keeps that at one cheap Prom roundtrip per 60 s // across all users), and renders the aggregate while it loads. - const all = await getBenchmarks(); + const all = await getBenchmarksSafe(); // Seed ONLY the unfiltered key. Seeding the initially-selected // chain/region/kind combo with the aggregate made the client believe // it already had that variant, so it never fetched the real one: the diff --git a/src/app/benchmarks/category/[cat]/page.tsx b/src/app/benchmarks/category/[cat]/page.tsx index 39b98bd7..ac7f977f 100644 --- a/src/app/benchmarks/category/[cat]/page.tsx +++ b/src/app/benchmarks/category/[cat]/page.tsx @@ -1,6 +1,6 @@ import type { Metadata } from "next"; import { notFound } from "next/navigation"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { BenchmarkGrid } from "@/components/benchmark-grid"; import { Breadcrumb } from "@/components/breadcrumb"; import { safeJsonLd, buildItemListJsonLd } from "@/lib/jsonld"; @@ -57,7 +57,7 @@ export default async function BenchmarkCategoryPage({ const entry = CATEGORY_BY_SLUG.get(cat); if (!entry) notFound(); - const all = await getBenchmarks(); + const all = await getBenchmarksSafe(); const benchmarks = all.filter((b) => b.category === entry.label); // Empty-category guard: a category in the enum that has no live bench // yet returns 404 so the crawler doesn't land on a thin page. The diff --git a/src/app/benchmarks/page.tsx b/src/app/benchmarks/page.tsx index 7cb7faae..a92d6428 100644 --- a/src/app/benchmarks/page.tsx +++ b/src/app/benchmarks/page.tsx @@ -1,5 +1,5 @@ import type { Metadata } from "next"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { BenchmarkGrid } from "@/components/benchmark-grid"; import { safeJsonLd, buildItemListJsonLd } from "@/lib/jsonld"; import { SITE } from "@/data/site"; @@ -17,7 +17,7 @@ export const metadata: Metadata = pageMetadata({ }); export default async function BenchmarksPage() { - const benchmarks = await getBenchmarks(); + const benchmarks = await getBenchmarksSafe(); // ItemList + BreadcrumbList JSON-LD so search engines and LLMs see the // page as a structured registry (the data is already in the DOM but diff --git a/src/app/llms.txt/route.ts b/src/app/llms.txt/route.ts index 4723e2e2..05d85aa4 100644 --- a/src/app/llms.txt/route.ts +++ b/src/app/llms.txt/route.ts @@ -1,5 +1,6 @@ import { getBenchmarks } from "@/data/benchmarks"; import { SITE } from "@/data/site"; +import { AllBenchmarksDraftError } from "@/lib/spec"; import { headlineSentence } from "@/lib/citation"; export const runtime = "nodejs"; @@ -13,7 +14,23 @@ export const revalidate = 300; * https://llmstxt.org for the spec. */ export async function GET() { - const benches = await getBenchmarks(); + let benches; + try { + benches = await getBenchmarks(); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) { + return new Response("benchmarks_unavailable\n", { + status: 503, + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + "retry-after": "60", + "access-control-allow-origin": "*", + }, + }); + } + throw err; + } const lines: string[] = []; lines.push(`# OpenChainBench`); diff --git a/src/app/mcp/page.tsx b/src/app/mcp/page.tsx index 29d8938d..a06b4836 100644 --- a/src/app/mcp/page.tsx +++ b/src/app/mcp/page.tsx @@ -1,7 +1,7 @@ import type { Metadata } from "next"; import Link from "next/link"; import { CopyButton } from "@/components/copy-button"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { mcpPageLd } from "@/lib/hub-jsonld"; import { safeJsonLd } from "@/lib/jsonld"; import { pageMetadata } from "@/lib/page-metadata"; @@ -48,7 +48,7 @@ export const metadata: Metadata = pageMetadata({ export const revalidate = 300; export default async function McpPage() { - const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); + const benches = (await getBenchmarksSafe()).filter((b) => b.editorialStatus === "live"); return (
diff --git a/src/app/page.tsx b/src/app/page.tsx index 111ea70e..554021a7 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,7 +1,7 @@ import type { Metadata } from "next"; import Link from "next/link"; import { ArrowRight } from "lucide-react"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { HeroRadar } from "@/components/hero-radar"; import { HomeBenchTable } from "@/components/home-bench-table"; import { LiveDashboard } from "@/components/live/dashboard"; @@ -32,7 +32,7 @@ export const metadata: Metadata = { }; export default async function HomePage() { - const benchmarks = await getBenchmarks(); + const benchmarks = await getBenchmarksSafe(); return (
diff --git a/src/app/rss.xml/route.ts b/src/app/rss.xml/route.ts index 1af7fd10..8d871f81 100644 --- a/src/app/rss.xml/route.ts +++ b/src/app/rss.xml/route.ts @@ -24,7 +24,7 @@ */ import { NextResponse } from "next/server"; -import { loadAllBenchmarks } from "@/lib/spec"; +import { AllBenchmarksDraftError, loadAllBenchmarks } from "@/lib/spec"; import { getBenchCreatedAt } from "@/lib/seo/bench-dates"; import { headlineSentence } from "@/lib/citation"; import { SITE } from "@/data/site"; @@ -59,7 +59,22 @@ function itemDescription(b: Benchmark): string { } export async function GET() { - const all = await loadAllBenchmarks(); + let all; + try { + all = await loadAllBenchmarks(); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) { + return new Response("benchmarks_unavailable\n", { + status: 503, + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + "retry-after": "60", + }, + }); + } + throw err; + } const live = all.filter((b) => b.editorialStatus === "live"); const items = live diff --git a/src/data/benchmarks.ts b/src/data/benchmarks.ts index 01cf5ce5..c02f4f92 100644 --- a/src/data/benchmarks.ts +++ b/src/data/benchmarks.ts @@ -10,10 +10,28 @@ import path from "node:path"; import yaml from "js-yaml"; import { cache } from "react"; import type { Benchmark } from "@/types/benchmark"; -import { loadAllBenchmarks, loadBenchmark } from "@/lib/spec"; +import { + loadAllBenchmarks, + loadAllBenchmarksSafe, + loadBenchmark, +} from "@/lib/spec"; +/** + * Strict loader. Throws AllBenchmarksDraftError when every bench has + * collapsed to draft (Prom blackout, cold start with no KV snapshot). + * Use this in API endpoints, feeds, and crons that should return 503 + * rather than poison downstream consumers with an all-draft snapshot. + */ export const getBenchmarks = cache(loadAllBenchmarks); +/** + * Build-and-render safe loader. Catches the all-draft sentinel and + * returns the draft-placeholder list so `next build` and hub pages + * still render. Use this in pages enumerated by generateStaticParams + * or any UI surface that must always produce HTML. + */ +export const getBenchmarksSafe = cache(loadAllBenchmarksSafe); + export async function getBenchmark( slug: string, options: { chain?: string; region?: string } = {} diff --git a/src/lib/chains.ts b/src/lib/chains.ts index 61e0d620..9f0f4826 100644 --- a/src/lib/chains.ts +++ b/src/lib/chains.ts @@ -12,7 +12,7 @@ */ import { cache } from "react"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import type { Benchmark } from "@/types/benchmark"; type ChainCategory = "L1" | "L2"; @@ -225,7 +225,7 @@ export const CHAIN_BY_SLUG = new Map(CHAINS.map((c) => [c.slug, c])); export const getBenchmarksForChain = cache(async function getBenchmarksForChain( chainSlug: string, ): Promise { - const benches = await getBenchmarks(); + const benches = await getBenchmarksSafe(); return benches.filter((b) => { if (b.results.some((r) => r.slug === chainSlug)) return true; if (b.dimensions?.chain?.some((c) => c.value === chainSlug)) return true; diff --git a/src/lib/providers.ts b/src/lib/providers.ts index b977edcf..f74c04a1 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -10,7 +10,7 @@ import { cache } from "react"; import { unstable_cache } from "next/cache"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { isAll } from "@/lib/dimensions"; import { liveResults } from "@/lib/provider-filters"; import { readBestPerChain } from "@/lib/per-chain-contract"; @@ -224,7 +224,7 @@ function rankPerChainForBench( } async function buildProviders(): Promise { - const benches = await getBenchmarks(); + const benches = await getBenchmarksSafe(); const byKey = new Map(); for (const b of benches) { diff --git a/src/lib/spec.test.ts b/src/lib/spec.test.ts index 108c8c6c..de704f6b 100644 --- a/src/lib/spec.test.ts +++ b/src/lib/spec.test.ts @@ -1,5 +1,11 @@ import { describe, expect, test } from "bun:test"; -import { injectLabels } from "./spec"; +import { + AllBenchmarksDraftError, + aggregateBenchmarks, + injectLabels, +} from "./spec"; +import type { Benchmark } from "@/types/benchmark"; +import type { Spec } from "@/lib/spec-schema"; describe("injectLabels", () => { test("injects into an empty selector", () => { @@ -45,3 +51,107 @@ describe("injectLabels", () => { expect(injectLabels("time()", { job: "api" })).toBe("time()"); }); }); + +// Regression coverage for the /api/citable poisoning bug (see +// AllBenchmarksDraftError in spec.ts). The aggregator must NEVER return +// a stable all-draft list that gets cached as truth. It either returns +// a list with at least one live bench, or throws AllBenchmarksDraftError +// so unstable_cache preserves the previous good value and downstream +// callers can decide whether to 503 (APIs) or render placeholders (pages). +function fakeSpec(slug: string, status: "live" | "draft" = "live"): Spec { + return { + slug, + number: "000", + title: `Bench ${slug}`, + subtitle: "", + abstract: "", + metric: "Latency", + unit: "ms", + higher_is_better: false, + category: "RPCs", + status, + findings: [], + methodology: [], + source: "", + providers: [], + queries: { p50: "", p90: "", p99: "", sample_size: "" }, + } as unknown as Spec; +} + +function fakeLiveBench(slug: string): Benchmark { + return { + slug, + number: "000", + title: `Bench ${slug}`, + subtitle: "", + lastRunAt: new Date().toISOString(), + status: "live", + editorialStatus: "live", + sampleSize: 1, + abstract: "", + metric: "Latency", + unit: "ms", + higherIsBetter: false, + category: "RPCs", + results: [], + findings: [], + methodology: [], + source: "", + extras: { series24h: {}, regions: {} }, + } as Benchmark; +} + +describe("aggregateBenchmarks (all-draft poisoning regression)", () => { + test("throws AllBenchmarksDraftError when every per-bench loader rejects", async () => { + const specs = [fakeSpec("a"), fakeSpec("b"), fakeSpec("c")]; + const loader = async () => { + throw new Error("prom blackout"); + }; + let caught: unknown = null; + try { + await aggregateBenchmarks(specs, loader); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(AllBenchmarksDraftError); + expect((caught as AllBenchmarksDraftError).slugCount).toBe(3); + }); + + test("throws AllBenchmarksDraftError when every loader returns undefined", async () => { + const specs = [fakeSpec("a"), fakeSpec("b")]; + const loader = async () => undefined; + await expect(aggregateBenchmarks(specs, loader)).rejects.toBeInstanceOf( + AllBenchmarksDraftError, + ); + }); + + test("returns the list when at least one bench is live", async () => { + const specs = [fakeSpec("a"), fakeSpec("b"), fakeSpec("c")]; + const loader = async (slug: string) => { + if (slug === "b") return fakeLiveBench(slug); + throw new Error("prom timeout"); + }; + const out = await aggregateBenchmarks(specs, loader); + expect(out).toHaveLength(3); + const live = out.filter((b) => b.status === "live"); + expect(live).toHaveLength(1); + expect(live[0].slug).toBe("b"); + }); + + test("never silently returns an all-draft stable list", async () => { + // The bug being regressed: pre-fix, aggregateBenchmarks returned an + // all-draft list when Prom was blacked out at cold start, and + // unstable_cache then served that list as truth for 60s. Any future + // refactor that drops the throw must trip this test. + const specs = [fakeSpec("x"), fakeSpec("y")]; + const loader = async () => undefined; + let threw = false; + try { + await aggregateBenchmarks(specs, loader); + } catch (err) { + threw = true; + expect(err).toBeInstanceOf(AllBenchmarksDraftError); + } + expect(threw).toBe(true); + }); +}); diff --git a/src/lib/spec.ts b/src/lib/spec.ts index 911d94f5..f5f078a9 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -193,57 +193,94 @@ const loadBenchmarkUnfilteredCached = unstable_cache( { revalidate: 60, tags: ["benchmarks"] }, ); +// Sentinel thrown by the aggregator when EVERY bench collapses to +// draft. Catchable by name at call sites that want to fall back to +// placeholders (page rendering) vs. propagate as 503 (APIs and feeds). +// +// Why an explicit error type: returning the draft set from the cached +// path poisons unstable_cache (Upstash KV) with an all-draft snapshot +// that then serves as "truth" to every downstream consumer for the +// rest of the revalidate window. /api/citable was the visible symptom. +// Throwing instead makes unstable_cache keep the previous good value +// and skip writing the bad one. +export class AllBenchmarksDraftError extends Error { + readonly slugCount: number; + constructor(slugCount: number) { + super( + `loadAllBenchmarks: every bench (${slugCount}) collapsed to draft. ` + + "Refusing to cache the all-draft set. Likely Prom blackout or " + + "cold start with no KV snapshot.", + ); + this.name = "AllBenchmarksDraftError"; + this.slugCount = slugCount; + } +} + +/** + * Pure aggregation step. Exported for tests. Given a spec list and a + * per-bench loader (real or mocked), fans out in parallel, substitutes + * a draft placeholder for any per-bench throw, and throws + * AllBenchmarksDraftError when literally every bench resolves to draft. + * + * Kept separate from the unstable_cache wrapper below so the test + * suite can exercise the error path without standing up Prometheus or + * the Next cache backend. + */ +export async function aggregateBenchmarks( + specs: Spec[], + loadOne: (slug: string) => Promise, +): Promise { + const settled = await Promise.allSettled(specs.map((s) => loadOne(s.slug))); + const benchmarks: Benchmark[] = []; + for (let i = 0; i < specs.length; i++) { + const spec = specs[i]; + const r = settled[i]; + if (r.status === "fulfilled" && r.value) { + benchmarks.push(r.value); + } else { + // Per-bench throw fired with no previous cache to fall back to. + // Surface a placeholder so the page still renders rather than + // dropping the bench from the list (which would break the sitemap, + // the products pages, and the "More benchmarks" rail). + // [DRAFT-TRACE] this path produces a visible "draft" render. Log + // so we can correlate with KV / Prom state. + const reason = + r.status === "rejected" + ? r.reason instanceof Error + ? r.reason.message + : String(r.reason) + : "no_value"; + console.warn( + `[DRAFT-TRACE] placeholder_used slug=${spec.slug} reason=${reason}`, + ); + benchmarks.push(draftPlaceholderForSpec(spec)); + } + } + const live = benchmarks.filter((b) => b.status === "live"); + if (benchmarks.length > 0 && live.length === 0) { + // Throw so unstable_cache keeps the previous good value during a + // Prom blackout. Previously we returned the all-draft set with a + // warning, which got cached for 60s and made /api/citable, sitemap, + // and hub pages all report every bench as draft while the per-slug + // /api/stat/ path returned live data. Build-time callers wrap + // in try/catch (see loadAllBenchmarksSafe). + console.warn( + `[DRAFT-TRACE] all_draft slug_count=${benchmarks.length} throwing to preserve previous cache value`, + ); + throw new AllBenchmarksDraftError(benchmarks.length); + } + return benchmarks.sort((a, b) => a.number.localeCompare(b.number)); +} + // Aggregator: fan out to N per-bench caches in parallel via // Promise.allSettled so a single bench's transient throw doesn't bring -// down the whole list. For specs whose per-bench cache is empty AND the -// fresh fetch threw (cold-start blackout case), we fall back to a draft -// placeholder so the page renders. The OG "all benches draft" safety -// throw is preserved: if literally every bench resolves to draft we -// throw to keep the previous all-benches cache intact. +// down the whole list. See aggregateBenchmarks for the actual logic. +// This wrapper layers unstable_cache on top so the result is shared +// across requests with 60s revalidate. const loadAllBenchmarksCached = unstable_cache( async (): Promise => { const specs = await loadSpecs(); - const settled = await Promise.allSettled( - specs.map((s) => loadBenchmarkUnfilteredCached(s.slug)), - ); - const benchmarks: Benchmark[] = []; - for (let i = 0; i < specs.length; i++) { - const spec = specs[i]; - const r = settled[i]; - if (r.status === "fulfilled" && r.value) { - benchmarks.push(r.value); - } else { - // The per-bench throw fired with no previous cache to fall back - // to. Surface a placeholder so the page still renders rather - // than dropping the bench from the list (which would break the - // sitemap, the products pages, and the "More benchmarks" rail). - // [DRAFT-TRACE] this is the path that produces a visible "draft" - // render to the user — log so we can correlate with KV / prom state. - const reason = - r.status === "rejected" ? (r.reason instanceof Error ? r.reason.message : String(r.reason)) : "no_value"; - console.warn( - `[DRAFT-TRACE] placeholder_used slug=${spec.slug} reason=${reason}`, - ); - benchmarks.push(draftPlaceholderForSpec(spec)); - } - } - const live = benchmarks.filter((b) => b.status === "live"); - if (benchmarks.length > 0 && live.length === 0) { - // Previously this threw to make unstable_cache keep the previous - // value during a Prom blackout. The throw, however, propagates - // unhandled into the BUILD-time page-generation path (no previous - // cache there) and crashes `next build` with "Error: every bench - // draft". With the snapshot/KV fallback + 10s Prom timeout + cron - // pre-warm now in place, this safety net mostly fires during the - // build window of a cold deploy. Warn loudly and return the - // draft set — the page renders with placeholders and recovers on - // the next revalidate cycle. Avoids deploy crashes. - console.warn( - "[DRAFT-TRACE] all_draft slug_count=" + benchmarks.length + - " — Prom blackout during build/cold-start, returning placeholders", - ); - } - return benchmarks.sort((a, b) => a.number.localeCompare(b.number)); + return aggregateBenchmarks(specs, loadBenchmarkUnfilteredCached); }, // v6: aggregate cache. Bumped together with bench-unfiltered-v4 so // the per-bench providersPerChain field actually propagates into the @@ -255,11 +292,48 @@ const loadAllBenchmarksCached = unstable_cache( // v10: bumped with bench-unfiltered-v8 (sec unit). // v11: bumped with bench-unfiltered-v9 (bp unit). // v12: bumped with bench-unfiltered-v10 (dimensions overlay). - ["all-benchmarks-v12"], + // v13: bumped to flush any poisoned all-draft snapshot written by the + // pre-fix code path during a Prom blackout. The fix throws on + // all-draft so unstable_cache no longer caches the bad set, but any + // already-stored v12 snapshot in Upstash KV would still serve for up + // to 60s after deploy. Bumping the key sidesteps that window. + ["all-benchmarks-v13"], { revalidate: 60, tags: ["benchmarks"] }, ); export const loadAllBenchmarks = cache(loadAllBenchmarksCached); +/** + * Safe wrapper for build-time and page-render callers that must produce + * SOMETHING even during a full Prom blackout. Catches the all-draft + * sentinel and substitutes a draft-placeholder list built directly from + * the specs. Does not cache the fallback (so the next call hits the + * cached path again and recovers as soon as Prom is back). + * + * Use this for: pages rendered at build time (`next build` enumerates + * generateStaticParams over benches; a throw there crashes the build), + * and for hub pages that should degrade gracefully to placeholders. + * + * Do NOT use this for: /api/citable, /api/llm-context, /llms.txt, + * /rss.xml, /api/cron/*. Those callers should let the throw surface and + * return 503 so downstream consumers (LLM agents, RSS readers, crons) + * don't silently treat the placeholder set as ground truth. + */ +export const loadAllBenchmarksSafe = cache( + async (): Promise => { + try { + return await loadAllBenchmarksCached(); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) { + const specs = await loadSpecs(); + return specs + .map((s) => draftPlaceholderForSpec(s)) + .sort((a, b) => a.number.localeCompare(b.number)); + } + throw err; + } + }, +); + /** * Cross-request server cache for filtered loads. Each (slug, filters) combo