From 9380d847eb0fbabce21fd246c35d2184f128efd3 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Sun, 21 Jun 2026 14:31:21 +0200 Subject: [PATCH] fix(citable): stop poisoning aggregator cache with all-draft set Fixes /api/citable returning all benchmarks as draft when /api/stat/ for the same bench returns live data. Root cause: loadAllBenchmarksCached returned an all-draft set whenever per-bench fetches threw at cold start, and the cache then served that poisoned set for the rest of the 60s revalidate window. Restores the throw-on-all-draft behaviour at the cache boundary so unstable_cache keeps the previous good value during a Prom blackout. Adds an AllBenchmarksDraftError sentinel and a getBenchmarksSafe wrapper. Call sites split: - API endpoints and feeds (citable, llm-context, llms.txt, rss.xml) catch the sentinel and return 503 with retry-after, so downstream consumers do not treat the placeholder set as ground truth. - Pages (home, benchmarks index, bench slug, category, mcp page) and shared loaders (chains, providers) use getBenchmarksSafe which catches the sentinel and renders draft placeholders so the build and the page render still succeed. - Sitemap already uses safeLoad which catches and falls back. Cache key bumped from v12 to v13 to flush any poisoned snapshot already stored in Upstash KV. Adds a regression test that asserts the aggregator never returns a stable all-draft list. Extracted aggregateBenchmarks as a pure function so the test does not need Prometheus or unstable_cache. --- src/app/api/citable/route.ts | 30 +++- src/app/api/llm-context/route.ts | 21 ++- src/app/benchmarks/[slug]/page.tsx | 4 +- src/app/benchmarks/category/[cat]/page.tsx | 4 +- src/app/benchmarks/page.tsx | 4 +- src/app/llms.txt/route.ts | 19 ++- src/app/mcp/page.tsx | 4 +- src/app/page.tsx | 4 +- src/app/rss.xml/route.ts | 19 ++- src/data/benchmarks.ts | 20 ++- src/lib/chains.ts | 4 +- src/lib/providers.ts | 4 +- src/lib/spec.test.ts | 112 +++++++++++++- src/lib/spec.ts | 168 +++++++++++++++------ 14 files changed, 348 insertions(+), 69 deletions(-) diff --git a/src/app/api/citable/route.ts b/src/app/api/citable/route.ts index 39b16f8b..3340cb98 100644 --- a/src/app/api/citable/route.ts +++ b/src/app/api/citable/route.ts @@ -1,15 +1,33 @@ import { NextResponse } from "next/server"; import { getBenchmarks } from "@/data/benchmarks"; import { SITE } from "@/data/site"; +import { AllBenchmarksDraftError } from "@/lib/spec"; import { citeBundle, fieldValue, leader, headlineSentence } from "@/lib/citation"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; export const runtime = "nodejs"; export const revalidate = 60; +/** Short 503 with a Retry-After hint, served when the aggregator has + * no live snapshot to surface (Prom blackout + cold KV). Beats serving + * an all-draft index that downstream LLM agents would treat as truth. */ +function unavailable(): NextResponse { + return NextResponse.json( + { error: "benchmarks_unavailable", retryAfterSec: 60 }, + { + status: 503, + headers: { + "cache-control": "no-store", + "retry-after": "60", + "access-control-allow-origin": "*", + }, + }, + ); +} + /** * Flat machine-readable index of every citable benchmark. Designed to be - * the **first** endpoint an AI agent or journalist crawls - gives them + * the **first** endpoint an AI agent or journalist crawls. Gives them * everything they need to decide whether to deep-link to a specific bench. * * License is intentionally surfaced per-row so downstream agents can @@ -19,7 +37,15 @@ export async function GET(req: Request) { const r = rateLimit(clientKey(req, "citable"), 60, 60); if (!r.ok) return tooManyRequests(r.retryAfterSec); - const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); + let benches; + try { + benches = (await getBenchmarks()).filter( + (b) => b.editorialStatus === "live", + ); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) return unavailable(); + throw err; + } const data = benches.map((b) => { const top = leader(b); return { diff --git a/src/app/api/llm-context/route.ts b/src/app/api/llm-context/route.ts index d4414fb1..5ba13994 100644 --- a/src/app/api/llm-context/route.ts +++ b/src/app/api/llm-context/route.ts @@ -1,5 +1,6 @@ import { getBenchmarks } from "@/data/benchmarks"; import { SITE } from "@/data/site"; +import { AllBenchmarksDraftError } from "@/lib/spec"; import { fmtUnit } from "@/lib/format"; import { fieldValue, headlineSentence, leader } from "@/lib/citation"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; @@ -26,7 +27,25 @@ export async function GET(req: Request) { return new Response(await tooMany.text(), { status: tooMany.status, headers: tooMany.headers }); } - const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); + let benches; + try { + benches = (await getBenchmarks()).filter( + (b) => b.editorialStatus === "live", + ); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) { + return new Response("benchmarks_unavailable\n", { + status: 503, + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + "retry-after": "60", + "access-control-allow-origin": "*", + }, + }); + } + throw err; + } const now = new Date().toISOString(); const lines: string[] = []; diff --git a/src/app/benchmarks/[slug]/page.tsx b/src/app/benchmarks/[slug]/page.tsx index cf34a093..059ff60b 100644 --- a/src/app/benchmarks/[slug]/page.tsx +++ b/src/app/benchmarks/[slug]/page.tsx @@ -5,7 +5,7 @@ import Link from "next/link"; import { ArrowUpRight, ChevronDown } from "lucide-react"; import { BackLink } from "@/components/back-link"; import { nonAllValues } from "@/lib/dimensions"; -import { getBenchmark, getBenchmarks } from "@/data/benchmarks"; +import { getBenchmark, getBenchmarksSafe } from "@/data/benchmarks"; import { Pill } from "@/components/pill"; import { BenchmarkBody } from "@/components/benchmark-body"; import { BenchmarkBodySkeleton } from "@/components/benchmark-body-skeleton"; @@ -150,7 +150,7 @@ export default async function BenchmarkPage({ // /api/bench/[slug]/variant when a tab is flipped (per-variant // unstable_cache keeps that at one cheap Prom roundtrip per 60 s // across all users), and renders the aggregate while it loads. - const all = await getBenchmarks(); + const all = await getBenchmarksSafe(); // Seed ONLY the unfiltered key. Seeding the initially-selected // chain/region/kind combo with the aggregate made the client believe // it already had that variant, so it never fetched the real one: the diff --git a/src/app/benchmarks/category/[cat]/page.tsx b/src/app/benchmarks/category/[cat]/page.tsx index 39b98bd7..ac7f977f 100644 --- a/src/app/benchmarks/category/[cat]/page.tsx +++ b/src/app/benchmarks/category/[cat]/page.tsx @@ -1,6 +1,6 @@ import type { Metadata } from "next"; import { notFound } from "next/navigation"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { BenchmarkGrid } from "@/components/benchmark-grid"; import { Breadcrumb } from "@/components/breadcrumb"; import { safeJsonLd, buildItemListJsonLd } from "@/lib/jsonld"; @@ -57,7 +57,7 @@ export default async function BenchmarkCategoryPage({ const entry = CATEGORY_BY_SLUG.get(cat); if (!entry) notFound(); - const all = await getBenchmarks(); + const all = await getBenchmarksSafe(); const benchmarks = all.filter((b) => b.category === entry.label); // Empty-category guard: a category in the enum that has no live bench // yet returns 404 so the crawler doesn't land on a thin page. The diff --git a/src/app/benchmarks/page.tsx b/src/app/benchmarks/page.tsx index 7cb7faae..a92d6428 100644 --- a/src/app/benchmarks/page.tsx +++ b/src/app/benchmarks/page.tsx @@ -1,5 +1,5 @@ import type { Metadata } from "next"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { BenchmarkGrid } from "@/components/benchmark-grid"; import { safeJsonLd, buildItemListJsonLd } from "@/lib/jsonld"; import { SITE } from "@/data/site"; @@ -17,7 +17,7 @@ export const metadata: Metadata = pageMetadata({ }); export default async function BenchmarksPage() { - const benchmarks = await getBenchmarks(); + const benchmarks = await getBenchmarksSafe(); // ItemList + BreadcrumbList JSON-LD so search engines and LLMs see the // page as a structured registry (the data is already in the DOM but diff --git a/src/app/llms.txt/route.ts b/src/app/llms.txt/route.ts index 4723e2e2..05d85aa4 100644 --- a/src/app/llms.txt/route.ts +++ b/src/app/llms.txt/route.ts @@ -1,5 +1,6 @@ import { getBenchmarks } from "@/data/benchmarks"; import { SITE } from "@/data/site"; +import { AllBenchmarksDraftError } from "@/lib/spec"; import { headlineSentence } from "@/lib/citation"; export const runtime = "nodejs"; @@ -13,7 +14,23 @@ export const revalidate = 300; * https://llmstxt.org for the spec. */ export async function GET() { - const benches = await getBenchmarks(); + let benches; + try { + benches = await getBenchmarks(); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) { + return new Response("benchmarks_unavailable\n", { + status: 503, + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + "retry-after": "60", + "access-control-allow-origin": "*", + }, + }); + } + throw err; + } const lines: string[] = []; lines.push(`# OpenChainBench`); diff --git a/src/app/mcp/page.tsx b/src/app/mcp/page.tsx index 29d8938d..a06b4836 100644 --- a/src/app/mcp/page.tsx +++ b/src/app/mcp/page.tsx @@ -1,7 +1,7 @@ import type { Metadata } from "next"; import Link from "next/link"; import { CopyButton } from "@/components/copy-button"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { mcpPageLd } from "@/lib/hub-jsonld"; import { safeJsonLd } from "@/lib/jsonld"; import { pageMetadata } from "@/lib/page-metadata"; @@ -48,7 +48,7 @@ export const metadata: Metadata = pageMetadata({ export const revalidate = 300; export default async function McpPage() { - const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); + const benches = (await getBenchmarksSafe()).filter((b) => b.editorialStatus === "live"); return (
diff --git a/src/app/page.tsx b/src/app/page.tsx index 111ea70e..554021a7 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,7 +1,7 @@ import type { Metadata } from "next"; import Link from "next/link"; import { ArrowRight } from "lucide-react"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { HeroRadar } from "@/components/hero-radar"; import { HomeBenchTable } from "@/components/home-bench-table"; import { LiveDashboard } from "@/components/live/dashboard"; @@ -32,7 +32,7 @@ export const metadata: Metadata = { }; export default async function HomePage() { - const benchmarks = await getBenchmarks(); + const benchmarks = await getBenchmarksSafe(); return (
diff --git a/src/app/rss.xml/route.ts b/src/app/rss.xml/route.ts index 1af7fd10..8d871f81 100644 --- a/src/app/rss.xml/route.ts +++ b/src/app/rss.xml/route.ts @@ -24,7 +24,7 @@ */ import { NextResponse } from "next/server"; -import { loadAllBenchmarks } from "@/lib/spec"; +import { AllBenchmarksDraftError, loadAllBenchmarks } from "@/lib/spec"; import { getBenchCreatedAt } from "@/lib/seo/bench-dates"; import { headlineSentence } from "@/lib/citation"; import { SITE } from "@/data/site"; @@ -59,7 +59,22 @@ function itemDescription(b: Benchmark): string { } export async function GET() { - const all = await loadAllBenchmarks(); + let all; + try { + all = await loadAllBenchmarks(); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) { + return new Response("benchmarks_unavailable\n", { + status: 503, + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + "retry-after": "60", + }, + }); + } + throw err; + } const live = all.filter((b) => b.editorialStatus === "live"); const items = live diff --git a/src/data/benchmarks.ts b/src/data/benchmarks.ts index 01cf5ce5..c02f4f92 100644 --- a/src/data/benchmarks.ts +++ b/src/data/benchmarks.ts @@ -10,10 +10,28 @@ import path from "node:path"; import yaml from "js-yaml"; import { cache } from "react"; import type { Benchmark } from "@/types/benchmark"; -import { loadAllBenchmarks, loadBenchmark } from "@/lib/spec"; +import { + loadAllBenchmarks, + loadAllBenchmarksSafe, + loadBenchmark, +} from "@/lib/spec"; +/** + * Strict loader. Throws AllBenchmarksDraftError when every bench has + * collapsed to draft (Prom blackout, cold start with no KV snapshot). + * Use this in API endpoints, feeds, and crons that should return 503 + * rather than poison downstream consumers with an all-draft snapshot. + */ export const getBenchmarks = cache(loadAllBenchmarks); +/** + * Build-and-render safe loader. Catches the all-draft sentinel and + * returns the draft-placeholder list so `next build` and hub pages + * still render. Use this in pages enumerated by generateStaticParams + * or any UI surface that must always produce HTML. + */ +export const getBenchmarksSafe = cache(loadAllBenchmarksSafe); + export async function getBenchmark( slug: string, options: { chain?: string; region?: string } = {} diff --git a/src/lib/chains.ts b/src/lib/chains.ts index 61e0d620..9f0f4826 100644 --- a/src/lib/chains.ts +++ b/src/lib/chains.ts @@ -12,7 +12,7 @@ */ import { cache } from "react"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import type { Benchmark } from "@/types/benchmark"; type ChainCategory = "L1" | "L2"; @@ -225,7 +225,7 @@ export const CHAIN_BY_SLUG = new Map(CHAINS.map((c) => [c.slug, c])); export const getBenchmarksForChain = cache(async function getBenchmarksForChain( chainSlug: string, ): Promise { - const benches = await getBenchmarks(); + const benches = await getBenchmarksSafe(); return benches.filter((b) => { if (b.results.some((r) => r.slug === chainSlug)) return true; if (b.dimensions?.chain?.some((c) => c.value === chainSlug)) return true; diff --git a/src/lib/providers.ts b/src/lib/providers.ts index b977edcf..f74c04a1 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -10,7 +10,7 @@ import { cache } from "react"; import { unstable_cache } from "next/cache"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe } from "@/data/benchmarks"; import { isAll } from "@/lib/dimensions"; import { liveResults } from "@/lib/provider-filters"; import { readBestPerChain } from "@/lib/per-chain-contract"; @@ -224,7 +224,7 @@ function rankPerChainForBench( } async function buildProviders(): Promise { - const benches = await getBenchmarks(); + const benches = await getBenchmarksSafe(); const byKey = new Map(); for (const b of benches) { diff --git a/src/lib/spec.test.ts b/src/lib/spec.test.ts index 108c8c6c..de704f6b 100644 --- a/src/lib/spec.test.ts +++ b/src/lib/spec.test.ts @@ -1,5 +1,11 @@ import { describe, expect, test } from "bun:test"; -import { injectLabels } from "./spec"; +import { + AllBenchmarksDraftError, + aggregateBenchmarks, + injectLabels, +} from "./spec"; +import type { Benchmark } from "@/types/benchmark"; +import type { Spec } from "@/lib/spec-schema"; describe("injectLabels", () => { test("injects into an empty selector", () => { @@ -45,3 +51,107 @@ describe("injectLabels", () => { expect(injectLabels("time()", { job: "api" })).toBe("time()"); }); }); + +// Regression coverage for the /api/citable poisoning bug (see +// AllBenchmarksDraftError in spec.ts). The aggregator must NEVER return +// a stable all-draft list that gets cached as truth. It either returns +// a list with at least one live bench, or throws AllBenchmarksDraftError +// so unstable_cache preserves the previous good value and downstream +// callers can decide whether to 503 (APIs) or render placeholders (pages). +function fakeSpec(slug: string, status: "live" | "draft" = "live"): Spec { + return { + slug, + number: "000", + title: `Bench ${slug}`, + subtitle: "", + abstract: "", + metric: "Latency", + unit: "ms", + higher_is_better: false, + category: "RPCs", + status, + findings: [], + methodology: [], + source: "", + providers: [], + queries: { p50: "", p90: "", p99: "", sample_size: "" }, + } as unknown as Spec; +} + +function fakeLiveBench(slug: string): Benchmark { + return { + slug, + number: "000", + title: `Bench ${slug}`, + subtitle: "", + lastRunAt: new Date().toISOString(), + status: "live", + editorialStatus: "live", + sampleSize: 1, + abstract: "", + metric: "Latency", + unit: "ms", + higherIsBetter: false, + category: "RPCs", + results: [], + findings: [], + methodology: [], + source: "", + extras: { series24h: {}, regions: {} }, + } as Benchmark; +} + +describe("aggregateBenchmarks (all-draft poisoning regression)", () => { + test("throws AllBenchmarksDraftError when every per-bench loader rejects", async () => { + const specs = [fakeSpec("a"), fakeSpec("b"), fakeSpec("c")]; + const loader = async () => { + throw new Error("prom blackout"); + }; + let caught: unknown = null; + try { + await aggregateBenchmarks(specs, loader); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(AllBenchmarksDraftError); + expect((caught as AllBenchmarksDraftError).slugCount).toBe(3); + }); + + test("throws AllBenchmarksDraftError when every loader returns undefined", async () => { + const specs = [fakeSpec("a"), fakeSpec("b")]; + const loader = async () => undefined; + await expect(aggregateBenchmarks(specs, loader)).rejects.toBeInstanceOf( + AllBenchmarksDraftError, + ); + }); + + test("returns the list when at least one bench is live", async () => { + const specs = [fakeSpec("a"), fakeSpec("b"), fakeSpec("c")]; + const loader = async (slug: string) => { + if (slug === "b") return fakeLiveBench(slug); + throw new Error("prom timeout"); + }; + const out = await aggregateBenchmarks(specs, loader); + expect(out).toHaveLength(3); + const live = out.filter((b) => b.status === "live"); + expect(live).toHaveLength(1); + expect(live[0].slug).toBe("b"); + }); + + test("never silently returns an all-draft stable list", async () => { + // The bug being regressed: pre-fix, aggregateBenchmarks returned an + // all-draft list when Prom was blacked out at cold start, and + // unstable_cache then served that list as truth for 60s. Any future + // refactor that drops the throw must trip this test. + const specs = [fakeSpec("x"), fakeSpec("y")]; + const loader = async () => undefined; + let threw = false; + try { + await aggregateBenchmarks(specs, loader); + } catch (err) { + threw = true; + expect(err).toBeInstanceOf(AllBenchmarksDraftError); + } + expect(threw).toBe(true); + }); +}); diff --git a/src/lib/spec.ts b/src/lib/spec.ts index 911d94f5..f5f078a9 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -193,57 +193,94 @@ const loadBenchmarkUnfilteredCached = unstable_cache( { revalidate: 60, tags: ["benchmarks"] }, ); +// Sentinel thrown by the aggregator when EVERY bench collapses to +// draft. Catchable by name at call sites that want to fall back to +// placeholders (page rendering) vs. propagate as 503 (APIs and feeds). +// +// Why an explicit error type: returning the draft set from the cached +// path poisons unstable_cache (Upstash KV) with an all-draft snapshot +// that then serves as "truth" to every downstream consumer for the +// rest of the revalidate window. /api/citable was the visible symptom. +// Throwing instead makes unstable_cache keep the previous good value +// and skip writing the bad one. +export class AllBenchmarksDraftError extends Error { + readonly slugCount: number; + constructor(slugCount: number) { + super( + `loadAllBenchmarks: every bench (${slugCount}) collapsed to draft. ` + + "Refusing to cache the all-draft set. Likely Prom blackout or " + + "cold start with no KV snapshot.", + ); + this.name = "AllBenchmarksDraftError"; + this.slugCount = slugCount; + } +} + +/** + * Pure aggregation step. Exported for tests. Given a spec list and a + * per-bench loader (real or mocked), fans out in parallel, substitutes + * a draft placeholder for any per-bench throw, and throws + * AllBenchmarksDraftError when literally every bench resolves to draft. + * + * Kept separate from the unstable_cache wrapper below so the test + * suite can exercise the error path without standing up Prometheus or + * the Next cache backend. + */ +export async function aggregateBenchmarks( + specs: Spec[], + loadOne: (slug: string) => Promise, +): Promise { + const settled = await Promise.allSettled(specs.map((s) => loadOne(s.slug))); + const benchmarks: Benchmark[] = []; + for (let i = 0; i < specs.length; i++) { + const spec = specs[i]; + const r = settled[i]; + if (r.status === "fulfilled" && r.value) { + benchmarks.push(r.value); + } else { + // Per-bench throw fired with no previous cache to fall back to. + // Surface a placeholder so the page still renders rather than + // dropping the bench from the list (which would break the sitemap, + // the products pages, and the "More benchmarks" rail). + // [DRAFT-TRACE] this path produces a visible "draft" render. Log + // so we can correlate with KV / Prom state. + const reason = + r.status === "rejected" + ? r.reason instanceof Error + ? r.reason.message + : String(r.reason) + : "no_value"; + console.warn( + `[DRAFT-TRACE] placeholder_used slug=${spec.slug} reason=${reason}`, + ); + benchmarks.push(draftPlaceholderForSpec(spec)); + } + } + const live = benchmarks.filter((b) => b.status === "live"); + if (benchmarks.length > 0 && live.length === 0) { + // Throw so unstable_cache keeps the previous good value during a + // Prom blackout. Previously we returned the all-draft set with a + // warning, which got cached for 60s and made /api/citable, sitemap, + // and hub pages all report every bench as draft while the per-slug + // /api/stat/ path returned live data. Build-time callers wrap + // in try/catch (see loadAllBenchmarksSafe). + console.warn( + `[DRAFT-TRACE] all_draft slug_count=${benchmarks.length} throwing to preserve previous cache value`, + ); + throw new AllBenchmarksDraftError(benchmarks.length); + } + return benchmarks.sort((a, b) => a.number.localeCompare(b.number)); +} + // Aggregator: fan out to N per-bench caches in parallel via // Promise.allSettled so a single bench's transient throw doesn't bring -// down the whole list. For specs whose per-bench cache is empty AND the -// fresh fetch threw (cold-start blackout case), we fall back to a draft -// placeholder so the page renders. The OG "all benches draft" safety -// throw is preserved: if literally every bench resolves to draft we -// throw to keep the previous all-benches cache intact. +// down the whole list. See aggregateBenchmarks for the actual logic. +// This wrapper layers unstable_cache on top so the result is shared +// across requests with 60s revalidate. const loadAllBenchmarksCached = unstable_cache( async (): Promise => { const specs = await loadSpecs(); - const settled = await Promise.allSettled( - specs.map((s) => loadBenchmarkUnfilteredCached(s.slug)), - ); - const benchmarks: Benchmark[] = []; - for (let i = 0; i < specs.length; i++) { - const spec = specs[i]; - const r = settled[i]; - if (r.status === "fulfilled" && r.value) { - benchmarks.push(r.value); - } else { - // The per-bench throw fired with no previous cache to fall back - // to. Surface a placeholder so the page still renders rather - // than dropping the bench from the list (which would break the - // sitemap, the products pages, and the "More benchmarks" rail). - // [DRAFT-TRACE] this is the path that produces a visible "draft" - // render to the user — log so we can correlate with KV / prom state. - const reason = - r.status === "rejected" ? (r.reason instanceof Error ? r.reason.message : String(r.reason)) : "no_value"; - console.warn( - `[DRAFT-TRACE] placeholder_used slug=${spec.slug} reason=${reason}`, - ); - benchmarks.push(draftPlaceholderForSpec(spec)); - } - } - const live = benchmarks.filter((b) => b.status === "live"); - if (benchmarks.length > 0 && live.length === 0) { - // Previously this threw to make unstable_cache keep the previous - // value during a Prom blackout. The throw, however, propagates - // unhandled into the BUILD-time page-generation path (no previous - // cache there) and crashes `next build` with "Error: every bench - // draft". With the snapshot/KV fallback + 10s Prom timeout + cron - // pre-warm now in place, this safety net mostly fires during the - // build window of a cold deploy. Warn loudly and return the - // draft set — the page renders with placeholders and recovers on - // the next revalidate cycle. Avoids deploy crashes. - console.warn( - "[DRAFT-TRACE] all_draft slug_count=" + benchmarks.length + - " — Prom blackout during build/cold-start, returning placeholders", - ); - } - return benchmarks.sort((a, b) => a.number.localeCompare(b.number)); + return aggregateBenchmarks(specs, loadBenchmarkUnfilteredCached); }, // v6: aggregate cache. Bumped together with bench-unfiltered-v4 so // the per-bench providersPerChain field actually propagates into the @@ -255,11 +292,48 @@ const loadAllBenchmarksCached = unstable_cache( // v10: bumped with bench-unfiltered-v8 (sec unit). // v11: bumped with bench-unfiltered-v9 (bp unit). // v12: bumped with bench-unfiltered-v10 (dimensions overlay). - ["all-benchmarks-v12"], + // v13: bumped to flush any poisoned all-draft snapshot written by the + // pre-fix code path during a Prom blackout. The fix throws on + // all-draft so unstable_cache no longer caches the bad set, but any + // already-stored v12 snapshot in Upstash KV would still serve for up + // to 60s after deploy. Bumping the key sidesteps that window. + ["all-benchmarks-v13"], { revalidate: 60, tags: ["benchmarks"] }, ); export const loadAllBenchmarks = cache(loadAllBenchmarksCached); +/** + * Safe wrapper for build-time and page-render callers that must produce + * SOMETHING even during a full Prom blackout. Catches the all-draft + * sentinel and substitutes a draft-placeholder list built directly from + * the specs. Does not cache the fallback (so the next call hits the + * cached path again and recovers as soon as Prom is back). + * + * Use this for: pages rendered at build time (`next build` enumerates + * generateStaticParams over benches; a throw there crashes the build), + * and for hub pages that should degrade gracefully to placeholders. + * + * Do NOT use this for: /api/citable, /api/llm-context, /llms.txt, + * /rss.xml, /api/cron/*. Those callers should let the throw surface and + * return 503 so downstream consumers (LLM agents, RSS readers, crons) + * don't silently treat the placeholder set as ground truth. + */ +export const loadAllBenchmarksSafe = cache( + async (): Promise => { + try { + return await loadAllBenchmarksCached(); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) { + const specs = await loadSpecs(); + return specs + .map((s) => draftPlaceholderForSpec(s)) + .sort((a, b) => a.number.localeCompare(b.number)); + } + throw err; + } + }, +); + /** * Cross-request server cache for filtered loads. Each (slug, filters) combo