diff --git a/benchmarks/pm-api-latency.yml b/benchmarks/pm-api-latency.yml index b318746a..b022741a 100644 --- a/benchmarks/pm-api-latency.yml +++ b/benchmarks/pm-api-latency.yml @@ -41,11 +41,9 @@ abstract: | this API up" is answered by direct measurement from three regions rather than by user reports. Samples that fail because our pinned market expired are classified probe_invalid and never count against the venue. - Third party data aggregators that resell venue prices (Codex) - are tracked separately in the pm-data-freshness bench and on - the data feeds tab of the prediction markets hub. This page ranks the - seven venues' own APIs only, so the leaderboard answers one clean - question: which prediction market venue API is fastest right now. + This page ranks the venues' own APIs only, so the leaderboard + answers one clean question: which prediction market venue API is + fastest right now. methodology: - "Price endpoint per venue. Polymarket: CLOB `/midpoint`. Kalshi: `/markets/{ticker}`. Limitless: `/markets/{slug}`. Manifold: `/v0/market/{id}`. Myriad: `/markets/{slug}`. This is the hot path of real integrations: one market, one quote, polled in a loop." @@ -53,10 +51,10 @@ methodology: - "Every sample carries a cache flag read from cf-cache-status, x-cache and age headers. Latency aggregates exclude cache hits: a CDN hit measures the nearest edge, not the venue's API, and including Manifold's residual cache hits would flatter it by design." - "Uptime is the average of a per venue, per region health gauge (pmapi_health) computed by the harness from probe outcomes. 100 percent means every probe cycle in the window succeeded from that region. The 24h and 7d panels below are the live answer to whether each API is up." - "Outcome classification: ok, timeout (8s), http_4xx, http_5xx, throttled, probe_invalid, net_error. probe_invalid covers failures that are our pin's fault, such as a market that resolved intraday; those are excluded from both latency and uptime. The pinned market is re selected daily at 00:00 UTC and immediately on probe_invalid." - - "Division of labor with the sibling benches: how each venue behaves as request rates climb, including throttle onset and 429 handling, is measured in pm-rate-limits (bench 037). How fresh third party data providers relay Polymarket data is measured in pm-data-freshness (bench 032). This bench ranks the venues' own APIs on latency and uptime at a polite request rate." + - "Division of labor with the sibling benches: how each venue behaves as request rates climb, including throttle onset and 429 handling, is measured in pm-rate-limits (bench 037). WebSocket connect-to-snapshot and trade publication lag are measured in pm-ws-latency (bench 114). This bench ranks the venues' own APIs on latency and uptime at a polite request rate." - "Myriad's origin is a single region US East deployment behind Heroku, so its latency from eu-west and sgp is dominated by geography. Reported as measured, called out per region." - "All five venues are probed by the same open source harness as bench 037, from the same processes, so the two benches share one probe budget and one identifying User-Agent: `OpenChainBench/1.0 (+https://openchainbench.com/methodology; contact@openchainbench.com)`. We publish latency and uptime measurements only, never market data." - - "Aggregators that resell venue data (Codex) are deliberately excluded from this leaderboard. They are not venue APIs, they are relays on top of venue APIs, and ranking them next to the venues they relay would compare two different products on one axis. Their freshness lag against the Polymarket CLOB T0 stream is the relevant metric, measured in the pm-data-freshness bench. The prediction markets hub data feeds tab lists each aggregator with the venues it covers." + - "Data aggregators that resell venue prices are deliberately excluded from this leaderboard. They are relays on top of venue APIs, and ranking them next to the venues they relay would compare two different products on one axis. The prediction markets hub data feeds tab lists each aggregator with the venues it covers." - "Regions: us-east, eu-west, sgp (Railway). Histogram buckets 25ms to 10s." findings: @@ -65,7 +63,7 @@ findings: - "{{name:manifold}} would look faster than it is if cache hits counted: its whole API sits behind a 5 second CDN cache. With cache hits excluded its origin answers at {{p50:manifold}} p50, an honest number a trading bot polling fresh quotes will actually see." - "{{name:myriad}} serves every region from a single US East origin, so its cohort worst p50 of {{p50:myriad}} is mostly geography. Switch the region dimension to us-east to see the API itself rather than the speed of light." - "{{name:polymarket}} answers its midpoint endpoint at {{p50:polymarket}} p50. Because we probe it every 5 seconds from three regions, the uptime panel doubles as a Polymarket API status check that updates continuously instead of waiting for user reports." - - "For builders who consume venue data through a managed relay, the relevant question is freshness lag, not request latency. Codex is measured in the pm-data-freshness bench (T0 against the Polymarket CLOB stream). The prediction markets hub data feeds tab lists each aggregator with the venues it covers." + - "For builders who consume venue data through a managed relay, the relevant question is freshness lag and WebSocket latency, not REST request latency. The pm-ws-latency bench measures trade publication lag on the Polymarket CLOB WebSocket. The prediction markets hub data feeds tab lists each aggregator with the venues it covers." disclaimer: "Uptime here means our probes succeeded from three specific regions at a polite request rate. A venue can be up for us and degraded for you, especially during regional network incidents, and a brief blip between probe cycles can go unrecorded. Treat the panel as a measured signal, not a guarantee." @@ -83,7 +81,7 @@ faq: - q: "Why do latency numbers exclude CDN cache hits?" a: "Because a cache hit measures the distance to the nearest edge node, not the venue's API. Manifold serves its entire API behind a 5 second cache and Kalshi's market list sits on CloudFront, so a naive average would mostly rank CDN proximity. Every sample carries a cache flag read from the response headers, and the headline aggregates only non cached, successful requests. The number you see is what a client polling for fresh quotes actually experiences at the origin." - q: "Does this benchmark cover throttling or data providers?" - a: "No, deliberately. What happens to each venue's API as request rates climb, including 429 onset and queueing behaviour, is its own benchmark, pm-rate-limits, which runs a careful daily ramp. And if you consume venue data through a provider such as Codex rather than calling the venue directly, the relevant number is relay freshness, measured in pm-data-freshness. This page answers two narrower questions: which venue API is fastest at a polite request rate, and is it up right now." + a: "No, deliberately. What happens to each venue's API as request rates climb, including 429 onset and queueing behaviour, is its own benchmark, pm-rate-limits, which runs a careful daily ramp. WebSocket latency and trade publication lag are covered in pm-ws-latency. This page answers two narrower questions: which venue API is fastest at a polite request rate, and is it up right now." per_chain_explainer: - slug: polymarket diff --git a/benchmarks/pm-rate-limits.yml b/benchmarks/pm-rate-limits.yml index 27e41b86..12c50271 100644 --- a/benchmarks/pm-rate-limits.yml +++ b/benchmarks/pm-rate-limits.yml @@ -79,7 +79,7 @@ faq: - q: "What is the Kalshi API rate limit?" a: "Kalshi documents tiered rate limits per access level, with the basic read tier around 20 requests per second at the time of writing. It is the only venue in this cohort with a documented token bucket. Our ramp stops at the first 429 on Kalshi out of respect for that documented contract, and the tier where it happens (or does not) is recorded in the ramp panel." - q: "Which prediction market has the fastest API?" - a: "{{best_name}} currently has the lowest warm book endpoint latency at {{best_p50}} (p50 over the last 24h). Note what this does and does not measure: it is the latency of the venue's own public API over a warm connection at a polite request rate, per region. It is not data freshness via third party providers, which is a separate benchmark (pm-data-freshness)." + a: "{{best_name}} currently has the lowest warm book endpoint latency at {{best_p50}} (p50 over the last 24h). Note what this does and does not measure: it is the latency of the venue's own public API over a warm connection at a polite request rate, per region. For data freshness and WebSocket latency, see the pm-api-latency and pm-ws-latency benches." - q: "Does Manifold have an API rate limit?" a: "Yes, 500 requests per minute per IP, documented, and Manifold explicitly welcomes bots as long as they stay on a single IP. Be aware that the whole API is served behind a cache with max-age=5 and stale-while-revalidate=10, so polling faster than every 6 seconds mostly returns cached responses. Our probes space out to 7 seconds per URL and label every sample that still comes back from cache." - q: "What are the Limitless API rate limits?" @@ -87,7 +87,7 @@ faq: - q: "How do you test rate limits without abusing the APIs?" a: "Four guardrails. Tiers stay well inside documented budgets where they exist (at most 7 percent of Polymarket's book allowance). Every run aborts as soon as throttled plus server errors exceed 1 percent of a 10 second window, and Kalshi stops at the first 429. Only one region ramps a venue at a time, on disjoint UTC hours. And every request carries an identifying User-Agent with a contact address, so a venue can reach us or filter us selectively. The measurement is throttle onset, not stress to failure." - q: "Why measure the venue APIs directly instead of a data provider?" - a: "They answer different questions. If you build directly on a venue, its native API latency and throttle behaviour set your floor, and that is what this benchmark measures. If you consume the venue through a data provider such as Codex or Predexon, what matters is how fresh the provider's relay is, which is measured separately in the pm-data-freshness benchmark. Both pages link each other so you can compare the paths." + a: "They answer different questions. If you build directly on a venue, its native API latency and throttle behaviour set your floor, and that is what this benchmark measures. For API latency at a polite rate, see pm-api-latency. For WebSocket connect and trade publication lag, see pm-ws-latency." per_chain_explainer: - slug: polymarket diff --git a/src/app/benchmarks/[slug]/page.tsx b/src/app/benchmarks/[slug]/page.tsx index fd197b5d..de1a044d 100644 --- a/src/app/benchmarks/[slug]/page.tsx +++ b/src/app/benchmarks/[slug]/page.tsx @@ -78,7 +78,7 @@ type Params = { slug: string }; // Rendered ON DEMAND (first request, then ISR-cached). Prerendering the // 25+ bench pages at build pushed the full multi-bench Prom load through // the CI runner, whose DNS resolver throttles under hundreds of lookups; -// observed 2026-06-11: /benchmarks/pm-data-freshness failing 3×60s +// observed 2026-06-11: a bench page failing 3×60s // export attempts and killing the deploy. Without the embedded variant // matrix an on-demand first render is a few seconds once per deploy per // slug, then the CDN serves it. diff --git a/src/app/prediction-markets/page.tsx b/src/app/prediction-markets/page.tsx index 79f966c1..3e8953db 100644 --- a/src/app/prediction-markets/page.tsx +++ b/src/app/prediction-markets/page.tsx @@ -8,7 +8,7 @@ import { SITE } from "@/data/site"; /** * Hub landing page for the prediction markets cohort. SSR'd against * the `pm-cohort-stats` harness gauges plus the existing PM bench - * gauges (pm-api-latency, pm-resolution-delay, pm-data-freshness). One + * gauges (pm-api-latency, pm-resolution-delay, pm-ws-latency). One * server fetch, one client tab swap between Venues and Data feeds. * * The page positions OCB as the neutral cross venue measurement layer: @@ -138,7 +138,7 @@ export default async function PredictionMarketsHubPage() { pm-resolution-delay Bench - pm-data-freshness + pm-ws-latency ,{" "} - - /benchmarks/pm-data-freshness + + /benchmarks/pm-ws-latency {" "} and{" "} diff --git a/src/app/products/[slug]/page.tsx b/src/app/products/[slug]/page.tsx index 9486f262..f1abb55d 100644 --- a/src/app/products/[slug]/page.tsx +++ b/src/app/products/[slug]/page.tsx @@ -575,23 +575,12 @@ export default async function ProviderPage({ jump from a venue/data provider product page to the PM coverage hub. Hard-coded slug list, same shape as the HL companion treatment elsewhere. */} - {/* Prediction-market pill: 6 native venues + Codex. - Codex is the aggregator measured on pm-data-freshness - alongside Polymarket + Kalshi as the third data - provider indexing PM feeds; retaining the pill lets - readers jump from /products/codex to the PM hub - where its freshness ranking surfaces. Mobula was - previously in this list but does not currently - appear on any PM bench and the pill read as a - routing bug on its data-API profile. */} {( p.slug === "polymarket" || p.slug === "kalshi" || p.slug === "limitless" || - p.slug === "manifold" || p.slug === "myriad" || - p.slug === "predexon" || - p.slug === "codex" + p.slug === "predexon" ) && ( <> · diff --git a/src/components/pm-data-feeds-leaderboard.tsx b/src/components/pm-data-feeds-leaderboard.tsx index 8a8b6630..d1e73b31 100644 --- a/src/components/pm-data-feeds-leaderboard.tsx +++ b/src/components/pm-data-feeds-leaderboard.tsx @@ -8,11 +8,10 @@ import { ProviderLogo } from "@/components/provider-logo"; import type { PmDataFeedRow } from "@/lib/pm-stats"; /** - * Data feeds leaderboard for /prediction-markets. Ranks every relay - * tracked by the pm-data-freshness bench by p50 freshness lag against - * the T0 reference (the venue itself). The reference row is rendered - * with a "T0 reference" badge so it never claims a 0 ms freshness - * lead over itself. + * Data feeds leaderboard for /prediction-markets. Lists every tracked + * relay with p50 freshness lag when available. The reference row is + * rendered with a "T0 reference" badge so it never claims a 0 ms + * freshness lead over itself. */ type SortKey = "freshnessP50Ms" | "freshnessP99Ms" | "uptime24h"; diff --git a/src/data/bench-published.json b/src/data/bench-published.json index e4506e1e..1780b3b4 100644 --- a/src/data/bench-published.json +++ b/src/data/bench-published.json @@ -42,8 +42,8 @@ "perp-open-interest": "2026-06-24T13:14:37.000Z", "perp-volume-share": "2026-06-24T13:14:37.000Z", "pm-api-latency": "2026-06-16T11:48:27.000Z", - "pm-data-freshness": "2026-06-04T17:50:08.000Z", "pm-rate-limits": "2026-06-16T11:48:27.000Z", + "pm-ws-latency": "2026-07-29T00:00:00.000Z", "polygon-rpc": "2026-07-03T14:33:15.000Z", "polymarket-resolution-delay": "2026-06-16T11:48:27.000Z", "portfolio-chain-coverage": "2026-07-06T13:47:16.000Z", diff --git a/src/lib/format.ts b/src/lib/format.ts index 77705c6a..7879d82c 100644 --- a/src/lib/format.ts +++ b/src/lib/format.ts @@ -117,9 +117,8 @@ export function fmtUnit(value: number, unit: string) { // fmtUnit produced but unitSuffix independently returned " min". if (value >= 60000) return `${(value / 60000).toFixed(1)} min`; if (value >= 1000) return `${(value / 1000).toFixed(2)} s`; - // Sub-millisecond values keep one decimal: pm-data-freshness's 0.5 ms - // anchor rendered "1 ms", contradicting the 0.5 published by the - // citable API (2x apart, flagged by the coherence audit). + // Sub-millisecond values keep one decimal: 0.5 ms would render as + // "1 ms" without this, contradicting the published citable value. if (value > 0 && value < 1) return `${value.toFixed(1)} ms`; return `${value.toFixed(0)} ms`; } diff --git a/src/lib/pm-stats.ts b/src/lib/pm-stats.ts index 8ee00198..c59f4a8c 100644 --- a/src/lib/pm-stats.ts +++ b/src/lib/pm-stats.ts @@ -1,16 +1,9 @@ /** * Server-side helper for the /prediction-markets hub page. Reads the - * `pm_venue_*` gauges exposed by the `pm-cohort-stats` harness, plus a - * handful of cross-bench histograms already in prod: + * `pm_venue_*` gauges exposed by the `pm-cohort-stats` harness, plus + * cross-bench histograms: * - pmapi_request_duration_seconds_bucket (bench pm-api-latency) - * - pm_health (bench pm-data-freshness, uptime) * - pmres_resolution_delay_seconds_bucket (bench pm-resolution-delay) - * - pm_freshness_delta_ms_bucket (bench pm-data-freshness) - * - * The freshness bench publishes a histogram keyed by - * `{provider, venue, kind}` (NOT a pre-aggregated p50/p99 gauge), so we - * compute the quantile client-side via `histogram_quantile()` and key - * the data-feed rows by the `provider` label. * * Shape mirrors `hl-builder-stats.ts` exactly so the page composes the * same way as /hyperliquid: one server fetch, one client tab swap. @@ -88,10 +81,6 @@ const PM_VENUES: VenueSeed[] = [ ]; const PM_DATA_FEEDS: DataFeedSeed[] = [ - // Mobula dropped 2026-07-19: they stopped serving the PM WebSocket - // relay, so the hub cohort and pm-data-freshness bench no longer - // rank a dead endpoint. - { slug: "codex", name: "Codex", coverage: ["polymarket", "kalshi"], isReference: false }, { slug: "predexon", name: "Predexon", coverage: ["polymarket", "kalshi", "limitless"], isReference: false }, ]; @@ -136,9 +125,6 @@ export async function fetchPmCohortFresh(): Promise { marketsAbove1m, apiLatencyP50, resolutionDelayP50, - freshnessP50, - freshnessP99, - uptime24h, ] = await Promise.all([ queryVector(prom, `pm_venue_volume_30d_usd`), queryVector(prom, `pm_venue_volume_24h_usd`), @@ -160,18 +146,6 @@ export async function fetchPmCohortFresh(): Promise { prom, `histogram_quantile(0.5, sum by (le) (rate(pmres_resolution_delay_seconds_bucket[30d])))`, ), - queryVector( - prom, - `histogram_quantile(0.5, sum by (provider, le) (rate(pm_freshness_delta_ms_bucket{venue="polymarket",provider!="polymarket"}[1h])))`, - ), - queryVector( - prom, - `histogram_quantile(0.99, sum by (provider, le) (rate(pm_freshness_delta_ms_bucket{venue="polymarket",provider!="polymarket"}[1h])))`, - ), - queryVector( - prom, - `avg_over_time(pm_health{venue="polymarket",provider!="polymarket"}[24h])`, - ), ]); // Map raw vectors onto the venue rows. Order of the seed list is kept @@ -228,31 +202,15 @@ export async function fetchPmCohortFresh(): Promise { row.medianResolutionDelayMin = Number.isFinite(s.value) ? s.value / 60 : null; } - // Data feeds. Coverage stays static (taken from the seed); the live - // values come from the freshness bench, which keys by `provider`. The - // T0 reference (Polymarket CLOB) is excluded from PM_DATA_FEEDS since - // measuring it against itself would always yield zero lag. - const freshnessP50By = new Map(); - const freshnessP99By = new Map(); - const uptimeBy = new Map(); - for (const s of freshnessP50 ?? []) { - const r = s.labels.provider; - if (r) freshnessP50By.set(r, s.value); - } - for (const s of freshnessP99 ?? []) { - const r = s.labels.provider; - if (r) freshnessP99By.set(r, s.value); - } - for (const s of uptime24h ?? []) { - const r = s.labels.provider; - if (r) uptimeBy.set(r, s.value); - } + // Data feeds. Coverage stays static (taken from the seed). The + // freshness bench was retired in 2026-07, so all numeric fields are null + // until a replacement harness ships. const dataFeeds: PmDataFeedRow[] = PM_DATA_FEEDS.map((f) => ({ slug: f.slug, name: f.name, - freshnessP50Ms: f.isReference ? null : (freshnessP50By.get(f.slug) ?? null), - freshnessP99Ms: f.isReference ? null : (freshnessP99By.get(f.slug) ?? null), - uptime24h: f.isReference ? null : (uptimeBy.get(f.slug) ?? null), + freshnessP50Ms: null, + freshnessP99Ms: null, + uptime24h: null, coverage: f.coverage, isReference: f.isReference, })); diff --git a/src/lib/pm-venue-context.ts b/src/lib/pm-venue-context.ts index bf9c10fd..a3d061c5 100644 --- a/src/lib/pm-venue-context.ts +++ b/src/lib/pm-venue-context.ts @@ -22,8 +22,7 @@ export const PM_VENUE_META: Record polymarket: { url: "https://polymarket.com", chainLabel: "Polygon" }, kalshi: { url: "https://kalshi.com", chainLabel: "Offchain US" }, limitless: { url: "https://limitless.exchange", chainLabel: "Base" }, - manifold: { url: "https://manifold.markets", chainLabel: "Offchain play money" }, - myriad: { url: "https://myriad.markets", chainLabel: "Offchain multi chain" }, + myriad: { url: "https://myriad.markets", chainLabel: "Abstract L2" }, }; // Logo paths come from the central manifest (`src/lib/logo-manifest.ts`) @@ -34,7 +33,6 @@ export const PM_VENUE_META: Record // mobula, and predexon. export const PM_FEED_META: Record = { "polymarket-clob": { url: "https://polymarket.com" }, - codex: { url: "https://codex.io" }, predexon: { url: "https://predexon.com" }, }; @@ -128,9 +126,9 @@ export function benchRowsForVenue( tone: "cyan", }, { - benchSlug: "pm-data-freshness", - label: "Data freshness", - blurb: "Third party relays vs the venue's canonical T0 stream.", + benchSlug: "pm-ws-latency", + label: "WS latency", + blurb: "Connect-to-snapshot and trade publication lag on the venue WebSocket.", rank: null, cohortSize, value: null, @@ -153,9 +151,9 @@ export function benchRowsForVenue( export function benchRowsForDataFeed(feed: PmDataFeedRow): PmVenueBenchRow[] { return [ { - benchSlug: "pm-data-freshness", - label: "Freshness vs T0", - blurb: "Lag between this relay and Polymarket CLOB T0.", + benchSlug: "pm-ws-latency", + label: "WS latency", + blurb: "Venue WebSocket connect-to-snapshot and trade publication lag.", rank: null, cohortSize: 0, value: feed.isReference ? null : fmtMs(feed.freshnessP50Ms), diff --git a/src/lib/search-featured.ts b/src/lib/search-featured.ts index 86403404..73253daf 100644 --- a/src/lib/search-featured.ts +++ b/src/lib/search-featured.ts @@ -19,7 +19,7 @@ import { loadSnapshotFromBlob } from "@/lib/bench-blob"; import { leader, fieldValue } from "@/lib/citation"; const FEATURED_BENCH_SLUGS = [ - "pm-data-freshness", + "pm-ws-latency", "aggregator-head-lag", "l1-finality", "rpc-capabilities",