From 9d4bce60bc9d6c3378a4ee2f0bd8d08a0b5068ee Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Sun, 21 Jun 2026 16:29:18 +0300 Subject: [PATCH] feat(rate-limit): bypass per-IP throttle for known AI crawler UAs (#615) * feat(rate-limit): bypass per-IP throttle for known AI crawler user agents * rate-limit: expand AI bot allowlist to match robots.txt Adds Meta-ExternalAgent, Meta-ExternalFetcher, cohere-ai, Perplexity-User (all already in robots.txt), plus AI2Bot, Ai2Bot-Dolma, Kagibot, FacebookBot, MistralAI-User, TimpiBot, Webzio-Extended for completeness. Comment now states the regex must stay in sync with src/app/robots.ts. Western LLM coverage is at ~95 percent after this. Chinese models other than ByteDance (Bytespider) do not publish a documented bot UA, so they fetch via generic browser UAs and stay throttled by design. --- src/app/api/badge/[slug]/[provider]/route.ts | 2 +- .../badge/[slug]/[provider]/snippet/route.ts | 2 +- .../api/bench/[slug]/oracle-pairs/route.ts | 2 +- src/app/api/bench/[slug]/variant/route.ts | 2 +- .../api/builder/[slug]/daily-series/route.ts | 2 +- src/app/api/builder/[slug]/top-users/route.ts | 2 +- src/app/api/chain/[slug]/live-prices/route.ts | 2 +- src/app/api/citable/route.ts | 2 +- src/app/api/freshness/route.ts | 2 +- src/app/api/llm-context/route.ts | 2 +- src/app/api/mcp/[transport]/route.ts | 2 +- src/app/api/og/[slug]/route.tsx | 2 +- src/app/api/series/[slug]/route.ts | 2 +- src/app/api/stat/[slug]/route.ts | 2 +- .../benchmarks/[slug]/share-card/route.tsx | 2 +- src/lib/rate-limit.test.ts | 68 +++++++++++++++++++ src/lib/rate-limit.ts | 44 ++++++++++++ 17 files changed, 127 insertions(+), 15 deletions(-) create mode 100644 src/lib/rate-limit.test.ts diff --git a/src/app/api/badge/[slug]/[provider]/route.ts b/src/app/api/badge/[slug]/[provider]/route.ts index 5f29beed..08bd7f51 100644 --- a/src/app/api/badge/[slug]/[provider]/route.ts +++ b/src/app/api/badge/[slug]/[provider]/route.ts @@ -158,7 +158,7 @@ export async function GET( req: NextRequest, { params }: { params: Promise }, ) { - const rl = rateLimit(clientKey(req, "badge"), 120, 60); + const rl = rateLimit(clientKey(req, "badge"), 120, 60, req); if (!rl.ok) return tooManyRequests(rl.retryAfterSec); const { slug, provider } = await params; diff --git a/src/app/api/badge/[slug]/[provider]/snippet/route.ts b/src/app/api/badge/[slug]/[provider]/snippet/route.ts index ef67d7e5..70625997 100644 --- a/src/app/api/badge/[slug]/[provider]/snippet/route.ts +++ b/src/app/api/badge/[slug]/[provider]/snippet/route.ts @@ -42,7 +42,7 @@ export async function GET( req: NextRequest, { params }: { params: Promise }, ) { - const r = rateLimit(clientKey(req, "badge-snippet"), 120, 60); + const r = rateLimit(clientKey(req, "badge-snippet"), 120, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug, provider } = await params; diff --git a/src/app/api/bench/[slug]/oracle-pairs/route.ts b/src/app/api/bench/[slug]/oracle-pairs/route.ts index f5fb1c38..0b4c6552 100644 --- a/src/app/api/bench/[slug]/oracle-pairs/route.ts +++ b/src/app/api/bench/[slug]/oracle-pairs/route.ts @@ -42,7 +42,7 @@ export async function GET( req: Request, { params }: { params: Promise }, ) { - const r = rateLimit(clientKey(req, "oracle-pairs"), 60, 60); + const r = rateLimit(clientKey(req, "oracle-pairs"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/bench/[slug]/variant/route.ts b/src/app/api/bench/[slug]/variant/route.ts index 7a9e9a08..37bcabe8 100644 --- a/src/app/api/bench/[slug]/variant/route.ts +++ b/src/app/api/bench/[slug]/variant/route.ts @@ -24,7 +24,7 @@ export async function GET( req: NextRequest, { params }: { params: Promise }, ) { - const rl = rateLimit(clientKey(req, "variant"), 120, 60); + const rl = rateLimit(clientKey(req, "variant"), 120, 60, req); if (!rl.ok) return tooManyRequests(rl.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/builder/[slug]/daily-series/route.ts b/src/app/api/builder/[slug]/daily-series/route.ts index 3cce38a0..cdabb1b4 100644 --- a/src/app/api/builder/[slug]/daily-series/route.ts +++ b/src/app/api/builder/[slug]/daily-series/route.ts @@ -26,7 +26,7 @@ export async function GET( req: Request, { params }: { params: Promise }, ) { - const r = rateLimit(clientKey(req, "hl-daily-series"), 60, 60); + const r = rateLimit(clientKey(req, "hl-daily-series"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/builder/[slug]/top-users/route.ts b/src/app/api/builder/[slug]/top-users/route.ts index f1569394..38ed6429 100644 --- a/src/app/api/builder/[slug]/top-users/route.ts +++ b/src/app/api/builder/[slug]/top-users/route.ts @@ -23,7 +23,7 @@ export async function GET( req: Request, { params }: { params: Promise }, ) { - const r = rateLimit(clientKey(req, "hl-top-users"), 60, 60); + const r = rateLimit(clientKey(req, "hl-top-users"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/chain/[slug]/live-prices/route.ts b/src/app/api/chain/[slug]/live-prices/route.ts index c5de8430..bbce1c7b 100644 --- a/src/app/api/chain/[slug]/live-prices/route.ts +++ b/src/app/api/chain/[slug]/live-prices/route.ts @@ -77,7 +77,7 @@ export async function GET( req: Request, { params }: { params: Promise<{ slug: string }> }, ) { - const r = rateLimit(clientKey(req, "chain-kpis-live"), 120, 60); + const r = rateLimit(clientKey(req, "chain-kpis-live"), 120, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/citable/route.ts b/src/app/api/citable/route.ts index f40c7d39..f5e8646e 100644 --- a/src/app/api/citable/route.ts +++ b/src/app/api/citable/route.ts @@ -34,7 +34,7 @@ function unavailable(): NextResponse { * cite without needing to read the footer of every page. */ export async function GET(req: Request) { - const r = rateLimit(clientKey(req, "citable"), 60, 60); + const r = rateLimit(clientKey(req, "citable"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); let benches; diff --git a/src/app/api/freshness/route.ts b/src/app/api/freshness/route.ts index ca2fded8..035351d5 100644 --- a/src/app/api/freshness/route.ts +++ b/src/app/api/freshness/route.ts @@ -71,7 +71,7 @@ const computeFreshness = unstable_cache( ); export async function GET(req: Request) { - const r = rateLimit(clientKey(req, "freshness"), 120, 60); + const r = rateLimit(clientKey(req, "freshness"), 120, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); // Resolve the spec list outside the cached function so its slug list diff --git a/src/app/api/llm-context/route.ts b/src/app/api/llm-context/route.ts index 5ba13994..57b01e5c 100644 --- a/src/app/api/llm-context/route.ts +++ b/src/app/api/llm-context/route.ts @@ -21,7 +21,7 @@ export const revalidate = 60; * per-region breakdown) but covers all 8 benches in one round-trip. */ export async function GET(req: Request) { - const r = rateLimit(clientKey(req, "llm-context"), 30, 60); + const r = rateLimit(clientKey(req, "llm-context"), 30, 60, req); if (!r.ok) { const tooMany = tooManyRequests(r.retryAfterSec); return new Response(await tooMany.text(), { status: tooMany.status, headers: tooMany.headers }); diff --git a/src/app/api/mcp/[transport]/route.ts b/src/app/api/mcp/[transport]/route.ts index 742010aa..d9cdea5c 100644 --- a/src/app/api/mcp/[transport]/route.ts +++ b/src/app/api/mcp/[transport]/route.ts @@ -549,7 +549,7 @@ const mcpHandler = createMcpHandler( * reject batches explicitly (see below). */ function rateLimited(req: Request): Response | null { const key = clientKey(req, "mcp"); - const r = rateLimit(key, 60, 60); + const r = rateLimit(key, 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); return null; } diff --git a/src/app/api/og/[slug]/route.tsx b/src/app/api/og/[slug]/route.tsx index a153fa32..d2449071 100644 --- a/src/app/api/og/[slug]/route.tsx +++ b/src/app/api/og/[slug]/route.tsx @@ -21,7 +21,7 @@ export async function GET( req: Request, { params }: { params: Promise<{ slug: string }> }, ) { - const r = rateLimit(clientKey(req, "og"), 60, 60); + const r = rateLimit(clientKey(req, "og"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/series/[slug]/route.ts b/src/app/api/series/[slug]/route.ts index 638c064d..689d6346 100644 --- a/src/app/api/series/[slug]/route.ts +++ b/src/app/api/series/[slug]/route.ts @@ -37,7 +37,7 @@ export async function GET( req: Request, { params }: { params: Promise<{ slug: string }> }, ) { - const r = rateLimit(clientKey(req, "series"), 60, 60); + const r = rateLimit(clientKey(req, "series"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/stat/[slug]/route.ts b/src/app/api/stat/[slug]/route.ts index 0101ab9a..dde8da87 100644 --- a/src/app/api/stat/[slug]/route.ts +++ b/src/app/api/stat/[slug]/route.ts @@ -24,7 +24,7 @@ export async function GET( req: Request, { params }: { params: Promise<{ slug: string }> }, ) { - const r = rateLimit(clientKey(req, "stat"), 60, 60); + const r = rateLimit(clientKey(req, "stat"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/benchmarks/[slug]/share-card/route.tsx b/src/app/benchmarks/[slug]/share-card/route.tsx index 02a21184..be8f7e7b 100644 --- a/src/app/benchmarks/[slug]/share-card/route.tsx +++ b/src/app/benchmarks/[slug]/share-card/route.tsx @@ -494,7 +494,7 @@ export async function GET( // benchmark loaders, each render is 50-200ms CPU. Without this an // attacker hitting random query-string variants would burn function // CPU even for unknown slugs. - const rl = rateLimit(clientKey(request, "share-card"), 60, 60); + const rl = rateLimit(clientKey(request, "share-card"), 60, 60, request); if (!rl.ok) return tooManyRequests(rl.retryAfterSec); const { slug } = await params; diff --git a/src/lib/rate-limit.test.ts b/src/lib/rate-limit.test.ts new file mode 100644 index 00000000..8209844b --- /dev/null +++ b/src/lib/rate-limit.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from "bun:test"; +import { isAiCrawler, rateLimit } from "./rate-limit"; + +/** Build a minimal Request with a single User-Agent header. The lib only + * reads `user-agent`; method and URL are inert for these tests. */ +function reqWithUa(ua: string | null): Request { + const headers = new Headers(); + if (ua !== null) headers.set("user-agent", ua); + return new Request("https://example.test/", { headers }); +} + +describe("isAiCrawler", () => { + test("matches known crawler substrings case-insensitively", () => { + expect(isAiCrawler("Mozilla/5.0 GPTBot/2.0")).toBe(true); + expect(isAiCrawler("perplexitybot/1.0")).toBe(true); + expect(isAiCrawler("Mozilla/5.0 ChatGPT-User/1.0")).toBe(true); + expect(isAiCrawler("Mozilla/5.0 (Linux) claudebot/1.0")).toBe(true); + expect(isAiCrawler("anthropic-ai")).toBe(true); + expect(isAiCrawler("ccbot/2.0")).toBe(true); + }); + + test("returns false for plain browsers and empty UAs", () => { + expect(isAiCrawler("Mozilla/5.0 (Macintosh) Chrome/121")).toBe(false); + expect(isAiCrawler("")).toBe(false); + expect(isAiCrawler(null)).toBe(false); + expect(isAiCrawler(undefined)).toBe(false); + }); +}); + +describe("rateLimit AI crawler bypass", () => { + test("known AI UA bypasses the bucket regardless of state", () => { + // Pre-drain the bucket so a non-bypassed call would be denied. + const key = "test-bypass:1.2.3.4"; + for (let i = 0; i < 5; i++) rateLimit(key, 5, 60); + const drained = rateLimit(key, 5, 60); + expect(drained.ok).toBe(false); + + const bypass = rateLimit(key, 5, 60, reqWithUa("Mozilla/5.0 GPTBot/2.0")); + expect(bypass.ok).toBe(true); + expect(bypass.retryAfterSec).toBe(0); + }); + + test("non-bot UA still consumes a token and can be throttled", () => { + const key = "test-browser:5.6.7.8"; + let last = rateLimit(key, 2, 60, reqWithUa("Mozilla/5.0 Chrome/121")); + expect(last.ok).toBe(true); + last = rateLimit(key, 2, 60, reqWithUa("Mozilla/5.0 Chrome/121")); + expect(last.ok).toBe(true); + last = rateLimit(key, 2, 60, reqWithUa("Mozilla/5.0 Chrome/121")); + expect(last.ok).toBe(false); + }); + + test("missing UA defaults to throttled (cautious default)", () => { + const key = "test-noua:9.9.9.9"; + let last = rateLimit(key, 1, 60, reqWithUa(null)); + expect(last.ok).toBe(true); + last = rateLimit(key, 1, 60, reqWithUa(null)); + expect(last.ok).toBe(false); + }); + + test("omitting req keeps original strict behaviour", () => { + const key = "test-noreq:10.0.0.1"; + let last = rateLimit(key, 1, 60); + expect(last.ok).toBe(true); + last = rateLimit(key, 1, 60); + expect(last.ok).toBe(false); + }); +}); diff --git a/src/lib/rate-limit.ts b/src/lib/rate-limit.ts index f49048e4..174f52fe 100644 --- a/src/lib/rate-limit.ts +++ b/src/lib/rate-limit.ts @@ -26,17 +26,61 @@ export type RateLimitResult = { retryAfterSec: number; }; +/** Known AI crawler User-Agent substrings. Matched case-insensitively. + * Requests from these agents bypass the per-IP throttle on read-only + * endpoints because the crawlers route many independent user queries + * through a small pool of shared egress IPs (PerplexityBot, + * ChatGPT-User, ClaudeBot, etc.). A burst from one IP usually means + * many distinct human questions, not abuse. UA spoofing is possible + * but the affected endpoints are read-only and explicitly designed to + * be cited, so the worst case of a spoofer is what we already want. */ +// Keep in sync with robots.txt (src/app/robots.ts). Any UA explicitly +// allowed there should bypass the per IP throttle here too, otherwise +// AI engines we want citations from will get 429s when many independent +// user queries share their small egress IP pool. +const AI_CRAWLER_RE = + /GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|anthropic-ai|Claude-Web|PerplexityBot|Perplexity-User|Google-Extended|GoogleOther|CCBot|Bytespider|Applebot-Extended|Meta-ExternalAgent|Meta-ExternalFetcher|cohere-ai|Amazonbot|Diffbot|YouBot|DuckAssistBot|AI2Bot|Ai2Bot-Dolma|Kagibot|FacebookBot|MistralAI-User|TimpiBot|Webzio-Extended/i; + +/** True when the given User-Agent matches a known AI crawler pattern. */ +export function isAiCrawler(userAgent: string | null | undefined): boolean { + if (!userAgent) return false; + return AI_CRAWLER_RE.test(userAgent); +} + +/** Open-ended success used when bypassing the bucket for an AI crawler. + * Same shape every caller already handles. */ +function aiBypassResult(): RateLimitResult { + return { ok: true, remaining: Number.POSITIVE_INFINITY, retryAfterSec: 0 }; +} + /** * Check & consume a token for `key`. Returns ok:false when the bucket is * empty, with retryAfterSec hint for clients. * * Bucket refills linearly: `capacity` tokens over `windowSec` seconds. + * + * Optional `req`: when provided, the request's User-Agent is inspected + * and a match against the AI crawler allowlist short-circuits the bucket + * (no token consumed). Pass it on read-only endpoints meant to be cited + * (citable, llm-context, stat, mcp, etc.). Omit on write endpoints or + * anywhere UA-based bypass is undesired. */ export function rateLimit( key: string, capacity: number, windowSec: number, + req?: Request, ): RateLimitResult { + if (req) { + const ua = req.headers.get("user-agent"); + if (isAiCrawler(ua)) { + if (process.env.AI_BYPASS_DEBUG === "true") { + // Trimmed to keep noisy UA blobs out of the log line. + console.log(`[ai-bypass] granted to ${(ua ?? "").slice(0, 80)}`); + } + return aiBypassResult(); + } + } if (capacity <= 0 || windowSec <= 0) { // Defensive: a misconfigured caller would otherwise produce Infinity // retry-after via /0 below. Treat as "denied with finite backoff".