From 644e96626308dac51ed0d5867e0df57395b103d1 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:44:26 +0200 Subject: [PATCH] fix(bench): sample-size PromQL + leader reliability floor (#1175) * fix(bench): sample-size PromQL + leader reliability floor Two bench-correctness fixes surfaced by an audit of the oracle and gas surfaces we plan to cite in outreach. Bench 025 (oracle-deviation): - sample_size was 'count(count by (source) (ocb_oracle_price))' which returns the count of distinct sources (3 or 4), not the count of samples over the window. The citable API therefore reported sampleSize=3-4 on every pair, at odds with the spec's 'polled every 30s' claim. Rewrites the query to 'sum(count_over_time(ocb_oracle_price[24h]))' across all 10 pair entries so the number reflects actual poll volume. Leader ranking: - citationCandidates() now excludes providers with a measured success rate below 50 percent from the leader pool. Before this fix, gas-estimation showed Owlracle as #1 with successRate=6.48 and a p50 gap of 0.001 gwei: technically accurate on the rare successful poll but useless as a leader claim. Falls back to the full live pool when every provider is under the floor so a totally degraded bench still surfaces a best-of-bad-options leader rather than vanishing. Freshness benches without a success query default to 100 percent in the loader so the guard is inert there. Adds src/lib/citation.test.ts with 5 tests covering the Owlracle scenario, higher-is-better ranking, and the degraded-pool fallback. * fix(bench): propagate reliability floor to rankings + template + hub Follow-up on the previous commit. Review flagged that filtering the leader without filtering the ranking arrays leaves self-contradictory JSON: /api/stat can report leader=Etherscan while rankings[0]=Owlracle on the same document. Exports citationCandidates and adds rankedCandidates helper that returns the same eligible pool sorted by the bench's higher_is_better direction. leader() and fieldValue() now share it. Wires the shared helper into every surface that emits a rankings-like list or a best-of claim: - src/app/api/stat/[slug]: rankings array now sorted from rankedCandidates so rankings[0].slug always equals leader.slug on the same JSON. - src/app/api/llm-context: the Rankings numbered list under each bench matches the Headline sentence above it. - src/app/api/mcp/[transport]: get_benchmark tool response and the openchainbench://benchmark/{slug} resource both share the pool. - src/lib/bench-template: {{best_name}} / {{best_p50}} / {{worst_name}} tokens in bench copy resolve against the same pool so a bench body never crowns a provider its own headline excludes. Per-slug tokens {{p50:some-slug}} still resolve against the raw live pool so unknown tokens still fall through. - src/lib/providers: rankProviders (drives /products hub wins count and per-chain leadership chips) now runs on the reliable pool. Adds a consistency test locking that rankedCandidates[0].slug always matches leader().slug on the same bench. Total: 6 tests in citation.test.ts, 95/95 pass across src/. Uses (r.successRate ?? 100) for the numeric guard to future-proof against ProviderResult shapes that skip the field. Load path still defaults to 100 (materialize/load.ts:748) so this is inert today, but the extra ?? removes the fragility. /api/compare/[a]/[b] is intentionally left unfiltered: the user picks the two providers explicitly, so the head-to-head is a requested comparison rather than a leader claim. Applying the floor there would drop rows the user explicitly asked to see. * fix(hub-card): honor citation leader in the hub grid headline Follow-up review flagged that benchmark-card.tsx re-sorted b.results locally and picked results[0], so a bench where citation demotes the raw-p50 winner (e.g. Owlracle on gas-estimation) would show the demoted provider on the hub grid while the bench page names the correct leader. Two documents contradicting each other in one click. Adds leaderSlug: string | null to BenchmarkCardData and populates it at the projection boundary via leader(b), which already applies the reliability + insufficient-sample filter. The card picks the leader by matching that slug in the projected results, and falls back to the raw best when the projection returned no leader (draft, insufficient, or all providers filtered out) so a card still renders a headline value in every state. Kept the local sort so the fallback path and any downstream chip using sorted[0].name keeps working. Test suite unchanged: 95/95 pass across src/. No test needed since this is a projection-boundary wire-up. --------- Co-authored-by: Florent Tapponnier --- benchmarks/oracle-deviation.yml | 20 ++--- src/app/api/llm-context/route.ts | 11 +-- src/app/api/mcp/[transport]/route.ts | 31 +++----- src/app/api/stat/[slug]/route.ts | 30 ++++--- src/components/benchmark-card.tsx | 12 ++- src/data/benchmarks.ts | 10 +++ src/lib/bench-template.ts | 13 ++- src/lib/citation.test.ts | 113 +++++++++++++++++++++++++++ src/lib/citation.ts | 66 ++++++++++++---- src/lib/providers.ts | 11 ++- 10 files changed, 247 insertions(+), 70 deletions(-) create mode 100644 src/lib/citation.test.ts diff --git a/benchmarks/oracle-deviation.yml b/benchmarks/oracle-deviation.yml index 79e38f0c..6f3ed8dd 100644 --- a/benchmarks/oracle-deviation.yml +++ b/benchmarks/oracle-deviation.yml @@ -135,7 +135,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="BTC/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="BTC/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="BTC/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="BTC/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="BTC/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="BTC/USD"} * 100 - slug: eth-usd @@ -148,7 +148,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="ETH/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="ETH/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="ETH/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="ETH/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="ETH/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="ETH/USD"} * 100 - slug: sol-usd @@ -161,7 +161,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="SOL/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="SOL/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="SOL/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="SOL/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="SOL/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="SOL/USD"} * 100 - slug: bnb-usd @@ -174,7 +174,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="BNB/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="BNB/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="BNB/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="BNB/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="BNB/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="BNB/USD"} * 100 - slug: avax-usd @@ -187,7 +187,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="AVAX/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="AVAX/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="AVAX/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="AVAX/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="AVAX/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="AVAX/USD"} * 100 - slug: link-usd @@ -200,7 +200,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="LINK/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="LINK/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="LINK/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="LINK/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="LINK/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="LINK/USD"} * 100 - slug: matic-usd @@ -213,7 +213,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="MATIC/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="MATIC/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="MATIC/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="MATIC/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="MATIC/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="MATIC/USD"} * 100 - slug: xrp-usd @@ -226,7 +226,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="XRP/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="XRP/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="XRP/USD"}[24h])) / (3 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="XRP/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="XRP/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="XRP/USD"} * 100 - slug: ada-usd @@ -239,7 +239,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="ADA/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="ADA/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="ADA/USD"}[24h])) / (3 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="ADA/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="ADA/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="ADA/USD"} * 100 - slug: doge-usd @@ -252,5 +252,5 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="DOGE/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="DOGE/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="DOGE/USD"}[24h])) / (3 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="DOGE/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="DOGE/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="DOGE/USD"} * 100 diff --git a/src/app/api/llm-context/route.ts b/src/app/api/llm-context/route.ts index c6617f49..27aada6f 100644 --- a/src/app/api/llm-context/route.ts +++ b/src/app/api/llm-context/route.ts @@ -7,6 +7,7 @@ import { headlineSentence, isInsufficient, leader, + rankedCandidates, } from "@/lib/citation"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; @@ -84,11 +85,11 @@ export async function GET(req: Request) { lines.push(`- Headline: ${headlineSentence(b)}`); lines.push(""); lines.push(`**Rankings (p50, 24h):**`); - const ranked = [...b.results] - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ); + // Shares `rankedCandidates` with `leader()` so the numbered list + // below matches the Headline sentence above. Without the shared + // filter, an LLM pasting this block would see e.g. "Etherscan + // leads" then a rankings list with Owlracle at #1. + const ranked = rankedCandidates(b); for (let i = 0; i < ranked.length; i++) { const r = ranked[i]; lines.push( diff --git a/src/app/api/mcp/[transport]/route.ts b/src/app/api/mcp/[transport]/route.ts index 4371f45a..e9461cb6 100644 --- a/src/app/api/mcp/[transport]/route.ts +++ b/src/app/api/mcp/[transport]/route.ts @@ -10,6 +10,7 @@ import { headlineSentence, isInsufficient, leader, + rankedCandidates, sparklineFor, } from "@/lib/citation"; import { fmtUnit } from "@/lib/format"; @@ -297,17 +298,12 @@ const mcpHandler = createMcpHandler( ms: { p50: null, p90: null, p99: null, mean: null }, successRate: r.successRate, })) - : b.results - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ) - .map((r) => ({ - name: r.name, - slug: r.slug, - ms: r.ms, - successRate: r.successRate, - })); + : rankedCandidates(b).map((r) => ({ + name: r.name, + slug: r.slug, + ms: r.ms, + successRate: r.successRate, + })); const payload = { slug: b.slug, title: b.title, @@ -480,13 +476,12 @@ const mcpHandler = createMcpHandler( } const insufficient = isInsufficient(b); const top = insufficient ? null : leader(b); - const ranked = insufficient - ? [] - : b.results - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ); + // Shares `rankedCandidates` with `leader()` so the Markdown + // Rankings list matches the Headline sentence above and the + // `rankings` field on the JSON tool response below. Without + // this, an agent reading this resource would see e.g. + // "Etherscan leads" then a numbered list with Owlracle at #1. + const ranked = insufficient ? [] : rankedCandidates(b); const md: string[] = []; md.push(`# ${b.title}`); diff --git a/src/app/api/stat/[slug]/route.ts b/src/app/api/stat/[slug]/route.ts index a75f11d4..49efc2da 100644 --- a/src/app/api/stat/[slug]/route.ts +++ b/src/app/api/stat/[slug]/route.ts @@ -8,6 +8,7 @@ import { fieldValue, headlineSentence, leader, + rankedCandidates, sparklineFor, } from "@/lib/citation"; import { valueInDeclaredUnit } from "@/lib/format"; @@ -98,22 +99,19 @@ export async function GET( insufficient || !top ? null : { ...top, value: valueInDeclaredUnit(top.value, b.unit) }, - rankings: b.results - .filter((r) => r.ms.p50 > 0) - // Drop "insufficient" rows from the machine-readable ranking too: - // a row that the page hides from the leaderboard must not surface - // here either. - .filter((r) => r.dataConfidence !== "insufficient") - .sort((a, c) => (b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50)) - .map((r) => ({ - name: r.name, - slug: r.slug, - ms: r.ms, - successRate: r.successRate, - sampleSize: r.sampleSize, - sampleHealth: r.sampleHealth, - dataConfidence: r.dataConfidence, - })), + // Shares `rankedCandidates` with `leader()` so `rankings[0]` + // stays consistent with the `leader` field on the same JSON blob: + // a document that names Etherscan as leader must not also list + // Owlracle first here. + rankings: rankedCandidates(b).map((r) => ({ + name: r.name, + slug: r.slug, + ms: r.ms, + successRate: r.successRate, + sampleSize: r.sampleSize, + sampleHealth: r.sampleHealth, + dataConfidence: r.dataConfidence, + })), sparkline: sparklineFor(b, top?.slug), sampleSize: b.sampleSize, expectedN: b.expectedN, diff --git a/src/components/benchmark-card.tsx b/src/components/benchmark-card.tsx index 6a03581f..8ec09662 100644 --- a/src/components/benchmark-card.tsx +++ b/src/components/benchmark-card.tsx @@ -27,12 +27,18 @@ export function BenchmarkCard({ benchmark }: { benchmark: BenchmarkCardData }) { const insufficient = !isDraft && isInsufficient(b); const catColor = CATEGORY_COLOR[b.category] ?? "var(--color-ink-muted)"; - // Field composite p50: best-of-class (or worst if higher-is-better is - // false). We pick the leader, same heuristic as the table. + // Prefer the canonical leader from citation.ts (reliability floor + + // insufficient-sample filter applied at projection). Falls back to + // the raw best of the projected results when the projection never + // returned a leader (draft, insufficient, or every provider filtered + // out) so the card still surfaces a headline value instead of "n/a" + // whenever any result is present. const sorted = [...b.results].sort( b.higherIsBetter ? (a, x) => x.ms.p50 - a.ms.p50 : (a, x) => a.ms.p50 - x.ms.p50, ); - const leader = sorted[0]; + const leader = b.leaderSlug + ? (sorted.find((r) => r.slug === b.leaderSlug) ?? sorted[0]) + : sorted[0]; const headlineValue = !isDraft && !insufficient && leader ? fmtValue(leader.ms.p50, b.unit) : "n/a"; // Pass the leader's p50 so unitSuffix mirrors fmtUnit's auto-conversion diff --git a/src/data/benchmarks.ts b/src/data/benchmarks.ts index 96cbb6e5..7aa14f07 100644 --- a/src/data/benchmarks.ts +++ b/src/data/benchmarks.ts @@ -9,6 +9,7 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import yaml from "js-yaml"; import { cache } from "react"; +import { leader } from "@/lib/citation"; import { downsample, MINI_CHART_POINTS } from "@/lib/downsample"; import type { Benchmark } from "@/types/benchmark"; import { @@ -51,6 +52,14 @@ export type BenchmarkCardData = { sampleSize: number; lastRunAt: string; metric: string; + /** Slug of the citation-eligible leader for this bench (result of + * `leader(b)` in citation.ts). Passed at the projection boundary so + * the hub card can render the same headline the bench page names + * without duplicating the reliability + insufficient-sample filter + * logic. Null when the bench has no defensible leader (draft, + * insufficient, or every provider below the reliability floor with + * an empty live pool). */ + leaderSlug: string | null; results: { slug: string; name: string; @@ -76,6 +85,7 @@ export function toBenchmarkCardData(b: Benchmark): BenchmarkCardData { sampleSize: b.sampleSize, lastRunAt: b.lastRunAt, metric: b.metric, + leaderSlug: leader(b)?.slug ?? null, results: b.results.map((r) => ({ slug: r.slug, name: r.name, diff --git a/src/lib/bench-template.ts b/src/lib/bench-template.ts index f36e4300..f1addc5f 100644 --- a/src/lib/bench-template.ts +++ b/src/lib/bench-template.ts @@ -38,6 +38,7 @@ import type { Benchmark, ProviderResult } from "@/types/benchmark"; import { liveResults } from "@/lib/provider-filters"; +import { citationCandidates } from "@/lib/citation"; import { rankResults } from "@/lib/ranking"; import { fmtUnit } from "@/lib/format"; @@ -87,8 +88,18 @@ function worstForChain(b: Benchmark, chain: string): ProviderResult | undefined export function renderTemplate(text: string, benchmark: Benchmark): string { if (!text || text.indexOf("{{") === -1) return text; + // `live` still drives per-slug lookups so callers of {{p50:some-slug}} + // can still address unreliable providers by name (the token is + // explicit). `bestPool` applies the same reliability floor as + // `leader()` so {{best_name}} and {{best_p50}} tokens in bench copy + // never elevate a provider that would be filtered out of the + // citation headline. const live = liveResults(benchmark.results); - const sorted = rankResults(live, benchmark.higherIsBetter); + const bestPool = citationCandidates(benchmark); + const sorted = rankResults( + bestPool.length > 0 ? bestPool : live, + benchmark.higherIsBetter, + ); const best = sorted[0]; const worst = sorted[sorted.length - 1]; diff --git a/src/lib/citation.test.ts b/src/lib/citation.test.ts new file mode 100644 index 00000000..376995f8 --- /dev/null +++ b/src/lib/citation.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, test } from "bun:test"; +import { leader, fieldValue, rankedCandidates } from "./citation"; +import type { Benchmark, ProviderResult } from "@/types/benchmark"; + +function r( + slug: string, + name: string, + p50: number, + successRate = 100, +): ProviderResult { + return { + slug, + name, + ms: { p50, p90: p50, p99: p50, mean: p50 }, + successRate, + availability: "live", + }; +} + +function bench(results: ProviderResult[]): Benchmark { + return { + slug: "test", + number: "001", + title: "Test bench", + subtitle: "", + lastRunAt: "2026-07-14T00:00:00.000Z", + status: "live", + editorialStatus: "live", + sampleSize: 100, + abstract: "", + metric: "Latency", + unit: "ms", + higherIsBetter: false, + category: "RPCs", + results, + findings: [], + methodology: [], + source: "", + extras: { series24h: {}, regions: {} }, + }; +} + +describe("citation reliability threshold", () => { + test("leader excludes providers with success rate below 50 percent", () => { + // Owlracle scenario: technically most accurate (lowest p50 gap) but + // fails 93 percent of calls. Etherscan should win despite a higher p50. + const b = bench([ + r("owlracle", "Owlracle", 0.001, 6.48), + r("etherscan", "Etherscan", 1.0, 95.7), + r("publicnode", "PublicNode", 1.5, 99.99), + ]); + const top = leader(b); + expect(top?.slug).toBe("etherscan"); + expect(top?.value).toBe(1.0); + }); + + test("fieldValue reflects the reliability-filtered leader", () => { + const b = bench([ + r("owlracle", "Owlracle", 0.001, 6.48), + r("etherscan", "Etherscan", 1.0, 95.7), + ]); + expect(fieldValue(b)).toBe(1.0); + }); + + test("falls back to the full live pool when every provider is unreliable", () => { + // Bench in a totally degraded state: rather than vanishing from + // downstream surfaces, surface the least-bad provider so readers + // still see a live number with the caveats their spec already + // documents. + const b = bench([ + r("a", "A", 5, 20), + r("b", "B", 10, 30), + ]); + const top = leader(b); + expect(top?.slug).toBe("a"); + }); + + test("providers with the default 100 percent success rate pass through", () => { + // Freshness / gauge-only benches never emit a `success` query; + // the loader defaults to 100 percent, so the guard is inert. + const b = bench([r("a", "A", 100), r("b", "B", 200)]); + expect(leader(b)?.slug).toBe("a"); + }); + + test("higher-is-better ranks the correct reliable leader", () => { + const b = bench([ + r("high-but-flaky", "Flaky", 999, 10), + r("modest-reliable", "Reliable", 50, 100), + r("mid-reliable", "Mid", 80, 100), + ]); + b.higherIsBetter = true; + // higherIsBetter=true reverses sort so the biggest p50 wins. + // Flaky wins on raw value but is filtered out; Mid (80) wins the + // reliable pool. + expect(leader(b)?.slug).toBe("mid-reliable"); + }); + + test("rankedCandidates[0] matches leader for the same bench", () => { + // Locks the invariant that downstream surfaces (/api/stat rankings, + // llm-context, MCP resource) can share `rankedCandidates` with + // `leader()` and never emit a document where the "leader" field + // contradicts the first entry of the "rankings" list. + const b = bench([ + r("owlracle", "Owlracle", 0.001, 6.48), + r("etherscan", "Etherscan", 1.0, 95.7), + r("publicnode", "PublicNode", 1.5, 99.99), + ]); + const top = leader(b); + const ranks = rankedCandidates(b); + expect(top?.slug).toBe(ranks[0].slug); + expect(top?.value).toBe(ranks[0].ms.p50); + }); +}); diff --git a/src/lib/citation.ts b/src/lib/citation.ts index ab2ed58c..09062a88 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -8,16 +8,56 @@ import type { Benchmark, ProviderResult } from "@/types/benchmark"; import { liveResults } from "@/lib/provider-filters"; import { fmtUnit } from "@/lib/format"; +/** Minimum measured success rate (in percent, 0-100) for a provider to + * contribute to the headline leader claim. Providers with a real + * success measurement below this floor are excluded from citation + * candidates because an "unreliable but accurate when it works" + * outlier should not top the leaderboard: it misleads AI agents citing + * the bench and any human reader glancing at the headline. Benches + * whose harness does not emit a `success` query default to 100 in the + * load path (see materialize/load.ts), so this guard is a no-op for + * freshness / gauge-only benches and only bites where the harness + * actually measures polling reliability (gas-estimation, RPC + * benches). */ +const LEADER_MIN_SUCCESS_PCT = 50; + /** Provider set used to derive the headline figures. Drops rows whose * per-provider sample-health is "insufficient" (set on the load path * when the bench declares expected_n and the row falls below the 10 - * percent of expected floor). Those rows can still render in some - * surfaces with a soft tag, but they must not contribute to the leader - * claim shipped to AI agents and journalists via the citable APIs. */ -function citationCandidates(b: Benchmark): ProviderResult[] { + * percent of expected floor) and rows whose measured success rate + * sits below the reliability floor. Those rows can still render in + * some surfaces with a soft tag, but they must not contribute to the + * leader claim shipped to AI agents and journalists via the citable + * APIs. Falls back to the full live pool when every provider is + * below the reliability floor so a totally-degraded bench still + * reports a best-of-bad-options leader instead of vanishing. + * + * Exported so downstream machine-readable surfaces (`/api/stat` + * rankings, `/api/llm-context`, MCP `get_benchmark`, `/api/compare`) + * can share the same eligibility rule as `leader()` and stay + * internally consistent: a JSON that names Etherscan as leader + * should not simultaneously rank Owlracle first in its `rankings` + * array. */ +export function citationCandidates(b: Benchmark): ProviderResult[] { const live = liveResults(b.results); - if (!b.expectedN) return live; - return live.filter((r) => r.dataConfidence !== "insufficient"); + const reliable = live.filter( + (r) => (r.successRate ?? 100) >= LEADER_MIN_SUCCESS_PCT, + ); + const pool = reliable.length > 0 ? reliable : live; + if (!b.expectedN) return pool; + return pool.filter((r) => r.dataConfidence !== "insufficient"); +} + +/** Sorted candidate pool for the machine-readable `rankings` array on + * `/api/stat`, MCP, llm-context and any downstream that ranks the + * full field. Applies the same reliability + insufficient-sample + * filters as `leader()` so a document that names X as leader ranks X + * first in its own list. Sort direction honors the bench's + * `higherIsBetter` flag. */ +export function rankedCandidates(b: Benchmark): ProviderResult[] { + return [...citationCandidates(b)].sort((a, c) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, + ); } /** Timestamp of the last real measurement, or null when the bench has @@ -42,11 +82,8 @@ export function fieldValue(b: Benchmark): number | null { // (downstream LLM tools and SERP snippets would otherwise quote a // number drawn from a wildly undersized field). if (b.dataConfidence === "insufficient") return null; - const candidates = citationCandidates(b); - if (candidates.length === 0) return null; - const sorted = [...candidates].sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 - ); + const sorted = rankedCandidates(b); + if (sorted.length === 0) return null; return sorted[0].ms.p50; } @@ -54,11 +91,8 @@ export function fieldValue(b: Benchmark): number | null { export function leader(b: Benchmark): { name: string; slug: string; value: number } | null { if (b.status !== "live") return null; if (b.dataConfidence === "insufficient") return null; - const candidates = citationCandidates(b); - if (candidates.length === 0) return null; - const sorted = [...candidates].sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 - ); + const sorted = rankedCandidates(b); + if (sorted.length === 0) return null; return { name: sorted[0].name, slug: sorted[0].slug, value: sorted[0].ms.p50 }; } diff --git a/src/lib/providers.ts b/src/lib/providers.ts index acd98747..01043029 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -12,6 +12,7 @@ import { cache } from "react"; import { unstable_cache } from "next/cache"; import { getBenchmarksSafe } from "@/data/benchmarks"; import { liveResults } from "@/lib/provider-filters"; +import { citationCandidates } from "@/lib/citation"; import { readBestPerChain } from "@/lib/per-chain-contract"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; @@ -221,7 +222,15 @@ export type ProviderProfile = { }; function rankProviders(b: Benchmark): ProviderResult[] { - const live = liveResults(b.results); + // Drives the /products hub `wins` count and per-chain leadership + // chips. Shares the reliability + insufficient-sample filters with + // `leader()` (citation.ts) so a provider that is filtered from the + // bench headline never gets credited as a bench winner on its + // product page. Falls back to the raw live pool when every + // candidate is filtered so a totally degraded bench still surfaces + // a best-of-bad-options ranking. + const pool = citationCandidates(b); + const live = pool.length > 0 ? pool : liveResults(b.results); return [...live].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, );