diff --git a/benchmarks/oracle-deviation.yml b/benchmarks/oracle-deviation.yml index 79e38f0c..6f3ed8dd 100644 --- a/benchmarks/oracle-deviation.yml +++ b/benchmarks/oracle-deviation.yml @@ -135,7 +135,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="BTC/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="BTC/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="BTC/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="BTC/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="BTC/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="BTC/USD"} * 100 - slug: eth-usd @@ -148,7 +148,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="ETH/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="ETH/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="ETH/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="ETH/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="ETH/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="ETH/USD"} * 100 - slug: sol-usd @@ -161,7 +161,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="SOL/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="SOL/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="SOL/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="SOL/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="SOL/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="SOL/USD"} * 100 - slug: bnb-usd @@ -174,7 +174,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="BNB/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="BNB/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="BNB/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="BNB/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="BNB/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="BNB/USD"} * 100 - slug: avax-usd @@ -187,7 +187,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="AVAX/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="AVAX/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="AVAX/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="AVAX/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="AVAX/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="AVAX/USD"} * 100 - slug: link-usd @@ -200,7 +200,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="LINK/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="LINK/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="LINK/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="LINK/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="LINK/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="LINK/USD"} * 100 - slug: matic-usd @@ -213,7 +213,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="MATIC/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="MATIC/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="MATIC/USD"}[24h])) / (4 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="MATIC/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="MATIC/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="MATIC/USD"} * 100 - slug: xrp-usd @@ -226,7 +226,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="XRP/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="XRP/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="XRP/USD"}[24h])) / (3 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="XRP/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="XRP/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="XRP/USD"} * 100 - slug: ada-usd @@ -239,7 +239,7 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="ADA/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="ADA/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="ADA/USD"}[24h])) / (3 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="ADA/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="ADA/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="ADA/USD"} * 100 - slug: doge-usd @@ -252,5 +252,5 @@ providers: p99: quantile_over_time(0.99, ocb_oracle_max_deviation_pct{pair="DOGE/USD"}[24h]) * 100 mean: avg_over_time(ocb_oracle_max_deviation_pct{pair="DOGE/USD"}[24h]) * 100 success: clamp_min(clamp_max(1 - sum(rate(ocb_oracle_scrape_errors_total{pair="DOGE/USD"}[24h])) / (3 * (1/30)), 1), 0) - sample_size: count(count by (source) (ocb_oracle_price{pair="DOGE/USD"})) + sample_size: sum(count_over_time(ocb_oracle_price{pair="DOGE/USD"}[24h])) series: ocb_oracle_max_deviation_pct{pair="DOGE/USD"} * 100 diff --git a/src/app/api/llm-context/route.ts b/src/app/api/llm-context/route.ts index c6617f49..27aada6f 100644 --- a/src/app/api/llm-context/route.ts +++ b/src/app/api/llm-context/route.ts @@ -7,6 +7,7 @@ import { headlineSentence, isInsufficient, leader, + rankedCandidates, } from "@/lib/citation"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; @@ -84,11 +85,11 @@ export async function GET(req: Request) { lines.push(`- Headline: ${headlineSentence(b)}`); lines.push(""); lines.push(`**Rankings (p50, 24h):**`); - const ranked = [...b.results] - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ); + // Shares `rankedCandidates` with `leader()` so the numbered list + // below matches the Headline sentence above. Without the shared + // filter, an LLM pasting this block would see e.g. "Etherscan + // leads" then a rankings list with Owlracle at #1. + const ranked = rankedCandidates(b); for (let i = 0; i < ranked.length; i++) { const r = ranked[i]; lines.push( diff --git a/src/app/api/mcp/[transport]/route.ts b/src/app/api/mcp/[transport]/route.ts index 4371f45a..e9461cb6 100644 --- a/src/app/api/mcp/[transport]/route.ts +++ b/src/app/api/mcp/[transport]/route.ts @@ -10,6 +10,7 @@ import { headlineSentence, isInsufficient, leader, + rankedCandidates, sparklineFor, } from "@/lib/citation"; import { fmtUnit } from "@/lib/format"; @@ -297,17 +298,12 @@ const mcpHandler = createMcpHandler( ms: { p50: null, p90: null, p99: null, mean: null }, successRate: r.successRate, })) - : b.results - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ) - .map((r) => ({ - name: r.name, - slug: r.slug, - ms: r.ms, - successRate: r.successRate, - })); + : rankedCandidates(b).map((r) => ({ + name: r.name, + slug: r.slug, + ms: r.ms, + successRate: r.successRate, + })); const payload = { slug: b.slug, title: b.title, @@ -480,13 +476,12 @@ const mcpHandler = createMcpHandler( } const insufficient = isInsufficient(b); const top = insufficient ? null : leader(b); - const ranked = insufficient - ? [] - : b.results - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ); + // Shares `rankedCandidates` with `leader()` so the Markdown + // Rankings list matches the Headline sentence above and the + // `rankings` field on the JSON tool response below. Without + // this, an agent reading this resource would see e.g. + // "Etherscan leads" then a numbered list with Owlracle at #1. + const ranked = insufficient ? [] : rankedCandidates(b); const md: string[] = []; md.push(`# ${b.title}`); diff --git a/src/app/api/stat/[slug]/route.ts b/src/app/api/stat/[slug]/route.ts index a75f11d4..49efc2da 100644 --- a/src/app/api/stat/[slug]/route.ts +++ b/src/app/api/stat/[slug]/route.ts @@ -8,6 +8,7 @@ import { fieldValue, headlineSentence, leader, + rankedCandidates, sparklineFor, } from "@/lib/citation"; import { valueInDeclaredUnit } from "@/lib/format"; @@ -98,22 +99,19 @@ export async function GET( insufficient || !top ? null : { ...top, value: valueInDeclaredUnit(top.value, b.unit) }, - rankings: b.results - .filter((r) => r.ms.p50 > 0) - // Drop "insufficient" rows from the machine-readable ranking too: - // a row that the page hides from the leaderboard must not surface - // here either. - .filter((r) => r.dataConfidence !== "insufficient") - .sort((a, c) => (b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50)) - .map((r) => ({ - name: r.name, - slug: r.slug, - ms: r.ms, - successRate: r.successRate, - sampleSize: r.sampleSize, - sampleHealth: r.sampleHealth, - dataConfidence: r.dataConfidence, - })), + // Shares `rankedCandidates` with `leader()` so `rankings[0]` + // stays consistent with the `leader` field on the same JSON blob: + // a document that names Etherscan as leader must not also list + // Owlracle first here. + rankings: rankedCandidates(b).map((r) => ({ + name: r.name, + slug: r.slug, + ms: r.ms, + successRate: r.successRate, + sampleSize: r.sampleSize, + sampleHealth: r.sampleHealth, + dataConfidence: r.dataConfidence, + })), sparkline: sparklineFor(b, top?.slug), sampleSize: b.sampleSize, expectedN: b.expectedN, diff --git a/src/components/benchmark-card.tsx b/src/components/benchmark-card.tsx index 6a03581f..8ec09662 100644 --- a/src/components/benchmark-card.tsx +++ b/src/components/benchmark-card.tsx @@ -27,12 +27,18 @@ export function BenchmarkCard({ benchmark }: { benchmark: BenchmarkCardData }) { const insufficient = !isDraft && isInsufficient(b); const catColor = CATEGORY_COLOR[b.category] ?? "var(--color-ink-muted)"; - // Field composite p50: best-of-class (or worst if higher-is-better is - // false). We pick the leader, same heuristic as the table. + // Prefer the canonical leader from citation.ts (reliability floor + + // insufficient-sample filter applied at projection). Falls back to + // the raw best of the projected results when the projection never + // returned a leader (draft, insufficient, or every provider filtered + // out) so the card still surfaces a headline value instead of "n/a" + // whenever any result is present. const sorted = [...b.results].sort( b.higherIsBetter ? (a, x) => x.ms.p50 - a.ms.p50 : (a, x) => a.ms.p50 - x.ms.p50, ); - const leader = sorted[0]; + const leader = b.leaderSlug + ? (sorted.find((r) => r.slug === b.leaderSlug) ?? sorted[0]) + : sorted[0]; const headlineValue = !isDraft && !insufficient && leader ? fmtValue(leader.ms.p50, b.unit) : "n/a"; // Pass the leader's p50 so unitSuffix mirrors fmtUnit's auto-conversion diff --git a/src/data/benchmarks.ts b/src/data/benchmarks.ts index 96cbb6e5..7aa14f07 100644 --- a/src/data/benchmarks.ts +++ b/src/data/benchmarks.ts @@ -9,6 +9,7 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import yaml from "js-yaml"; import { cache } from "react"; +import { leader } from "@/lib/citation"; import { downsample, MINI_CHART_POINTS } from "@/lib/downsample"; import type { Benchmark } from "@/types/benchmark"; import { @@ -51,6 +52,14 @@ export type BenchmarkCardData = { sampleSize: number; lastRunAt: string; metric: string; + /** Slug of the citation-eligible leader for this bench (result of + * `leader(b)` in citation.ts). Passed at the projection boundary so + * the hub card can render the same headline the bench page names + * without duplicating the reliability + insufficient-sample filter + * logic. Null when the bench has no defensible leader (draft, + * insufficient, or every provider below the reliability floor with + * an empty live pool). */ + leaderSlug: string | null; results: { slug: string; name: string; @@ -76,6 +85,7 @@ export function toBenchmarkCardData(b: Benchmark): BenchmarkCardData { sampleSize: b.sampleSize, lastRunAt: b.lastRunAt, metric: b.metric, + leaderSlug: leader(b)?.slug ?? null, results: b.results.map((r) => ({ slug: r.slug, name: r.name, diff --git a/src/lib/bench-template.ts b/src/lib/bench-template.ts index f36e4300..f1addc5f 100644 --- a/src/lib/bench-template.ts +++ b/src/lib/bench-template.ts @@ -38,6 +38,7 @@ import type { Benchmark, ProviderResult } from "@/types/benchmark"; import { liveResults } from "@/lib/provider-filters"; +import { citationCandidates } from "@/lib/citation"; import { rankResults } from "@/lib/ranking"; import { fmtUnit } from "@/lib/format"; @@ -87,8 +88,18 @@ function worstForChain(b: Benchmark, chain: string): ProviderResult | undefined export function renderTemplate(text: string, benchmark: Benchmark): string { if (!text || text.indexOf("{{") === -1) return text; + // `live` still drives per-slug lookups so callers of {{p50:some-slug}} + // can still address unreliable providers by name (the token is + // explicit). `bestPool` applies the same reliability floor as + // `leader()` so {{best_name}} and {{best_p50}} tokens in bench copy + // never elevate a provider that would be filtered out of the + // citation headline. const live = liveResults(benchmark.results); - const sorted = rankResults(live, benchmark.higherIsBetter); + const bestPool = citationCandidates(benchmark); + const sorted = rankResults( + bestPool.length > 0 ? bestPool : live, + benchmark.higherIsBetter, + ); const best = sorted[0]; const worst = sorted[sorted.length - 1]; diff --git a/src/lib/citation.test.ts b/src/lib/citation.test.ts new file mode 100644 index 00000000..376995f8 --- /dev/null +++ b/src/lib/citation.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, test } from "bun:test"; +import { leader, fieldValue, rankedCandidates } from "./citation"; +import type { Benchmark, ProviderResult } from "@/types/benchmark"; + +function r( + slug: string, + name: string, + p50: number, + successRate = 100, +): ProviderResult { + return { + slug, + name, + ms: { p50, p90: p50, p99: p50, mean: p50 }, + successRate, + availability: "live", + }; +} + +function bench(results: ProviderResult[]): Benchmark { + return { + slug: "test", + number: "001", + title: "Test bench", + subtitle: "", + lastRunAt: "2026-07-14T00:00:00.000Z", + status: "live", + editorialStatus: "live", + sampleSize: 100, + abstract: "", + metric: "Latency", + unit: "ms", + higherIsBetter: false, + category: "RPCs", + results, + findings: [], + methodology: [], + source: "", + extras: { series24h: {}, regions: {} }, + }; +} + +describe("citation reliability threshold", () => { + test("leader excludes providers with success rate below 50 percent", () => { + // Owlracle scenario: technically most accurate (lowest p50 gap) but + // fails 93 percent of calls. Etherscan should win despite a higher p50. + const b = bench([ + r("owlracle", "Owlracle", 0.001, 6.48), + r("etherscan", "Etherscan", 1.0, 95.7), + r("publicnode", "PublicNode", 1.5, 99.99), + ]); + const top = leader(b); + expect(top?.slug).toBe("etherscan"); + expect(top?.value).toBe(1.0); + }); + + test("fieldValue reflects the reliability-filtered leader", () => { + const b = bench([ + r("owlracle", "Owlracle", 0.001, 6.48), + r("etherscan", "Etherscan", 1.0, 95.7), + ]); + expect(fieldValue(b)).toBe(1.0); + }); + + test("falls back to the full live pool when every provider is unreliable", () => { + // Bench in a totally degraded state: rather than vanishing from + // downstream surfaces, surface the least-bad provider so readers + // still see a live number with the caveats their spec already + // documents. + const b = bench([ + r("a", "A", 5, 20), + r("b", "B", 10, 30), + ]); + const top = leader(b); + expect(top?.slug).toBe("a"); + }); + + test("providers with the default 100 percent success rate pass through", () => { + // Freshness / gauge-only benches never emit a `success` query; + // the loader defaults to 100 percent, so the guard is inert. + const b = bench([r("a", "A", 100), r("b", "B", 200)]); + expect(leader(b)?.slug).toBe("a"); + }); + + test("higher-is-better ranks the correct reliable leader", () => { + const b = bench([ + r("high-but-flaky", "Flaky", 999, 10), + r("modest-reliable", "Reliable", 50, 100), + r("mid-reliable", "Mid", 80, 100), + ]); + b.higherIsBetter = true; + // higherIsBetter=true reverses sort so the biggest p50 wins. + // Flaky wins on raw value but is filtered out; Mid (80) wins the + // reliable pool. + expect(leader(b)?.slug).toBe("mid-reliable"); + }); + + test("rankedCandidates[0] matches leader for the same bench", () => { + // Locks the invariant that downstream surfaces (/api/stat rankings, + // llm-context, MCP resource) can share `rankedCandidates` with + // `leader()` and never emit a document where the "leader" field + // contradicts the first entry of the "rankings" list. + const b = bench([ + r("owlracle", "Owlracle", 0.001, 6.48), + r("etherscan", "Etherscan", 1.0, 95.7), + r("publicnode", "PublicNode", 1.5, 99.99), + ]); + const top = leader(b); + const ranks = rankedCandidates(b); + expect(top?.slug).toBe(ranks[0].slug); + expect(top?.value).toBe(ranks[0].ms.p50); + }); +}); diff --git a/src/lib/citation.ts b/src/lib/citation.ts index ab2ed58c..09062a88 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -8,16 +8,56 @@ import type { Benchmark, ProviderResult } from "@/types/benchmark"; import { liveResults } from "@/lib/provider-filters"; import { fmtUnit } from "@/lib/format"; +/** Minimum measured success rate (in percent, 0-100) for a provider to + * contribute to the headline leader claim. Providers with a real + * success measurement below this floor are excluded from citation + * candidates because an "unreliable but accurate when it works" + * outlier should not top the leaderboard: it misleads AI agents citing + * the bench and any human reader glancing at the headline. Benches + * whose harness does not emit a `success` query default to 100 in the + * load path (see materialize/load.ts), so this guard is a no-op for + * freshness / gauge-only benches and only bites where the harness + * actually measures polling reliability (gas-estimation, RPC + * benches). */ +const LEADER_MIN_SUCCESS_PCT = 50; + /** Provider set used to derive the headline figures. Drops rows whose * per-provider sample-health is "insufficient" (set on the load path * when the bench declares expected_n and the row falls below the 10 - * percent of expected floor). Those rows can still render in some - * surfaces with a soft tag, but they must not contribute to the leader - * claim shipped to AI agents and journalists via the citable APIs. */ -function citationCandidates(b: Benchmark): ProviderResult[] { + * percent of expected floor) and rows whose measured success rate + * sits below the reliability floor. Those rows can still render in + * some surfaces with a soft tag, but they must not contribute to the + * leader claim shipped to AI agents and journalists via the citable + * APIs. Falls back to the full live pool when every provider is + * below the reliability floor so a totally-degraded bench still + * reports a best-of-bad-options leader instead of vanishing. + * + * Exported so downstream machine-readable surfaces (`/api/stat` + * rankings, `/api/llm-context`, MCP `get_benchmark`, `/api/compare`) + * can share the same eligibility rule as `leader()` and stay + * internally consistent: a JSON that names Etherscan as leader + * should not simultaneously rank Owlracle first in its `rankings` + * array. */ +export function citationCandidates(b: Benchmark): ProviderResult[] { const live = liveResults(b.results); - if (!b.expectedN) return live; - return live.filter((r) => r.dataConfidence !== "insufficient"); + const reliable = live.filter( + (r) => (r.successRate ?? 100) >= LEADER_MIN_SUCCESS_PCT, + ); + const pool = reliable.length > 0 ? reliable : live; + if (!b.expectedN) return pool; + return pool.filter((r) => r.dataConfidence !== "insufficient"); +} + +/** Sorted candidate pool for the machine-readable `rankings` array on + * `/api/stat`, MCP, llm-context and any downstream that ranks the + * full field. Applies the same reliability + insufficient-sample + * filters as `leader()` so a document that names X as leader ranks X + * first in its own list. Sort direction honors the bench's + * `higherIsBetter` flag. */ +export function rankedCandidates(b: Benchmark): ProviderResult[] { + return [...citationCandidates(b)].sort((a, c) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, + ); } /** Timestamp of the last real measurement, or null when the bench has @@ -42,11 +82,8 @@ export function fieldValue(b: Benchmark): number | null { // (downstream LLM tools and SERP snippets would otherwise quote a // number drawn from a wildly undersized field). if (b.dataConfidence === "insufficient") return null; - const candidates = citationCandidates(b); - if (candidates.length === 0) return null; - const sorted = [...candidates].sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 - ); + const sorted = rankedCandidates(b); + if (sorted.length === 0) return null; return sorted[0].ms.p50; } @@ -54,11 +91,8 @@ export function fieldValue(b: Benchmark): number | null { export function leader(b: Benchmark): { name: string; slug: string; value: number } | null { if (b.status !== "live") return null; if (b.dataConfidence === "insufficient") return null; - const candidates = citationCandidates(b); - if (candidates.length === 0) return null; - const sorted = [...candidates].sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 - ); + const sorted = rankedCandidates(b); + if (sorted.length === 0) return null; return { name: sorted[0].name, slug: sorted[0].slug, value: sorted[0].ms.p50 }; } diff --git a/src/lib/providers.ts b/src/lib/providers.ts index acd98747..01043029 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -12,6 +12,7 @@ import { cache } from "react"; import { unstable_cache } from "next/cache"; import { getBenchmarksSafe } from "@/data/benchmarks"; import { liveResults } from "@/lib/provider-filters"; +import { citationCandidates } from "@/lib/citation"; import { readBestPerChain } from "@/lib/per-chain-contract"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; @@ -221,7 +222,15 @@ export type ProviderProfile = { }; function rankProviders(b: Benchmark): ProviderResult[] { - const live = liveResults(b.results); + // Drives the /products hub `wins` count and per-chain leadership + // chips. Shares the reliability + insufficient-sample filters with + // `leader()` (citation.ts) so a provider that is filtered from the + // bench headline never gets credited as a bench winner on its + // product page. Falls back to the raw live pool when every + // candidate is filtered so a totally degraded bench still surfaces + // a best-of-bad-options ranking. + const pool = citationCandidates(b); + const live = pool.length > 0 ? pool : liveResults(b.results); return [...live].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, );