hotfix: bypass per-IP rate limit for known AI crawler UAs

src/app/api/badge/[slug]/[provider]/route.ts

@@ -158,7 +158,7 @@ export async function GET(
   req: NextRequest,
   { params }: { params: Promise<Params> },
 ) {
-  const rl = rateLimit(clientKey(req, "badge"), 120, 60);
+  const rl = rateLimit(clientKey(req, "badge"), 120, 60, req);
   if (!rl.ok) return tooManyRequests(rl.retryAfterSec);
 
   const { slug, provider } = await params;

src/app/api/badge/[slug]/[provider]/snippet/route.ts

@@ -42,7 +42,7 @@ export async function GET(
   req: NextRequest,
   { params }: { params: Promise<Params> },
 ) {
-  const r = rateLimit(clientKey(req, "badge-snippet"), 120, 60);
+  const r = rateLimit(clientKey(req, "badge-snippet"), 120, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   const { slug, provider } = await params;

src/app/api/bench/[slug]/oracle-pairs/route.ts

@@ -42,7 +42,7 @@ export async function GET(
   req: Request,
   { params }: { params: Promise<Params> },
 ) {
-  const r = rateLimit(clientKey(req, "oracle-pairs"), 60, 60);
+  const r = rateLimit(clientKey(req, "oracle-pairs"), 60, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   const { slug } = await params;

src/app/api/bench/[slug]/variant/route.ts

@@ -24,7 +24,7 @@ export async function GET(
   req: NextRequest,
   { params }: { params: Promise<Params> },
 ) {
-  const rl = rateLimit(clientKey(req, "variant"), 120, 60);
+  const rl = rateLimit(clientKey(req, "variant"), 120, 60, req);
   if (!rl.ok) return tooManyRequests(rl.retryAfterSec);
 
   const { slug } = await params;

src/app/api/builder/[slug]/daily-series/route.ts

@@ -26,7 +26,7 @@ export async function GET(
   req: Request,
   { params }: { params: Promise<Params> },
 ) {
-  const r = rateLimit(clientKey(req, "hl-daily-series"), 60, 60);
+  const r = rateLimit(clientKey(req, "hl-daily-series"), 60, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   const { slug } = await params;

src/app/api/builder/[slug]/top-users/route.ts

@@ -23,7 +23,7 @@ export async function GET(
   req: Request,
   { params }: { params: Promise<Params> },
 ) {
-  const r = rateLimit(clientKey(req, "hl-top-users"), 60, 60);
+  const r = rateLimit(clientKey(req, "hl-top-users"), 60, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   const { slug } = await params;

src/app/api/chain/[slug]/live-prices/route.ts

@@ -77,7 +77,7 @@ export async function GET(
   req: Request,
   { params }: { params: Promise<{ slug: string }> },
 ) {
-  const r = rateLimit(clientKey(req, "chain-kpis-live"), 120, 60);
+  const r = rateLimit(clientKey(req, "chain-kpis-live"), 120, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   const { slug } = await params;

src/app/api/citable/route.ts

@@ -34,7 +34,7 @@ function unavailable(): NextResponse {
  * cite without needing to read the footer of every page.
  */
 export async function GET(req: Request) {
-  const r = rateLimit(clientKey(req, "citable"), 60, 60);
+  const r = rateLimit(clientKey(req, "citable"), 60, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   let benches;

src/app/api/freshness/route.ts

@@ -71,7 +71,7 @@ const computeFreshness = unstable_cache(
 );
 
 export async function GET(req: Request) {
-  const r = rateLimit(clientKey(req, "freshness"), 120, 60);
+  const r = rateLimit(clientKey(req, "freshness"), 120, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   // Resolve the spec list outside the cached function so its slug list

src/app/api/llm-context/route.ts

@@ -21,7 +21,7 @@ export const revalidate = 60;
  *    per-region breakdown) but covers all 8 benches in one round-trip.
  */
 export async function GET(req: Request) {
-  const r = rateLimit(clientKey(req, "llm-context"), 30, 60);
+  const r = rateLimit(clientKey(req, "llm-context"), 30, 60, req);
   if (!r.ok) {
     const tooMany = tooManyRequests(r.retryAfterSec);
     return new Response(await tooMany.text(), { status: tooMany.status, headers: tooMany.headers });

src/app/api/mcp/[transport]/route.ts

@@ -549,7 +549,7 @@ const mcpHandler = createMcpHandler(
  *  reject batches explicitly (see below). */
 function rateLimited(req: Request): Response | null {
   const key = clientKey(req, "mcp");
-  const r = rateLimit(key, 60, 60);
+  const r = rateLimit(key, 60, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
   return null;
 }

src/app/api/og/[slug]/route.tsx

@@ -21,7 +21,7 @@ export async function GET(
   req: Request,
   { params }: { params: Promise<{ slug: string }> },
 ) {
-  const r = rateLimit(clientKey(req, "og"), 60, 60);
+  const r = rateLimit(clientKey(req, "og"), 60, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   const { slug } = await params;

src/app/api/series/[slug]/route.ts

@@ -37,7 +37,7 @@ export async function GET(
   req: Request,
   { params }: { params: Promise<{ slug: string }> },
 ) {
-  const r = rateLimit(clientKey(req, "series"), 60, 60);
+  const r = rateLimit(clientKey(req, "series"), 60, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   const { slug } = await params;

src/app/api/stat/[slug]/route.ts

@@ -24,7 +24,7 @@ export async function GET(
   req: Request,
   { params }: { params: Promise<{ slug: string }> },
 ) {
-  const r = rateLimit(clientKey(req, "stat"), 60, 60);
+  const r = rateLimit(clientKey(req, "stat"), 60, 60, req);
   if (!r.ok) return tooManyRequests(r.retryAfterSec);
 
   const { slug } = await params;

src/app/benchmarks/[slug]/share-card/route.tsx

@@ -494,7 +494,7 @@ export async function GET(
   // benchmark loaders, each render is 50-200ms CPU. Without this an
   // attacker hitting random query-string variants would burn function
   // CPU even for unknown slugs.
-  const rl = rateLimit(clientKey(request, "share-card"), 60, 60);
+  const rl = rateLimit(clientKey(request, "share-card"), 60, 60, request);
   if (!rl.ok) return tooManyRequests(rl.retryAfterSec);
 
   const { slug } = await params;

src/lib/rate-limit.test.ts

@@ -0,0 +1,68 @@
+import { describe, expect, test } from "bun:test";
+import { isAiCrawler, rateLimit } from "./rate-limit";
+
+/** Build a minimal Request with a single User-Agent header. The lib only
+ *  reads `user-agent`; method and URL are inert for these tests. */
+function reqWithUa(ua: string | null): Request {
+  const headers = new Headers();
+  if (ua !== null) headers.set("user-agent", ua);
+  return new Request("https://example.test/", { headers });
+}
+
+describe("isAiCrawler", () => {
+  test("matches known crawler substrings case-insensitively", () => {
+    expect(isAiCrawler("Mozilla/5.0 GPTBot/2.0")).toBe(true);
+    expect(isAiCrawler("perplexitybot/1.0")).toBe(true);
+    expect(isAiCrawler("Mozilla/5.0 ChatGPT-User/1.0")).toBe(true);
+    expect(isAiCrawler("Mozilla/5.0 (Linux) claudebot/1.0")).toBe(true);
+    expect(isAiCrawler("anthropic-ai")).toBe(true);
+    expect(isAiCrawler("ccbot/2.0")).toBe(true);
+  });
+
+  test("returns false for plain browsers and empty UAs", () => {
+    expect(isAiCrawler("Mozilla/5.0 (Macintosh) Chrome/121")).toBe(false);
+    expect(isAiCrawler("")).toBe(false);
+    expect(isAiCrawler(null)).toBe(false);
+    expect(isAiCrawler(undefined)).toBe(false);
+  });
+});
+
+describe("rateLimit AI crawler bypass", () => {
+  test("known AI UA bypasses the bucket regardless of state", () => {
+    // Pre-drain the bucket so a non-bypassed call would be denied.
+    const key = "test-bypass:1.2.3.4";
+    for (let i = 0; i < 5; i++) rateLimit(key, 5, 60);
+    const drained = rateLimit(key, 5, 60);
+    expect(drained.ok).toBe(false);
+
+    const bypass = rateLimit(key, 5, 60, reqWithUa("Mozilla/5.0 GPTBot/2.0"));
+    expect(bypass.ok).toBe(true);
+    expect(bypass.retryAfterSec).toBe(0);
+  });
+
+  test("non-bot UA still consumes a token and can be throttled", () => {
+    const key = "test-browser:5.6.7.8";
+    let last = rateLimit(key, 2, 60, reqWithUa("Mozilla/5.0 Chrome/121"));
+    expect(last.ok).toBe(true);
+    last = rateLimit(key, 2, 60, reqWithUa("Mozilla/5.0 Chrome/121"));
+    expect(last.ok).toBe(true);
+    last = rateLimit(key, 2, 60, reqWithUa("Mozilla/5.0 Chrome/121"));
+    expect(last.ok).toBe(false);
+  });
+
+  test("missing UA defaults to throttled (cautious default)", () => {
+    const key = "test-noua:9.9.9.9";
+    let last = rateLimit(key, 1, 60, reqWithUa(null));
+    expect(last.ok).toBe(true);
+    last = rateLimit(key, 1, 60, reqWithUa(null));
+    expect(last.ok).toBe(false);
+  });
+
+  test("omitting req keeps original strict behaviour", () => {
+    const key = "test-noreq:10.0.0.1";
+    let last = rateLimit(key, 1, 60);
+    expect(last.ok).toBe(true);
+    last = rateLimit(key, 1, 60);
+    expect(last.ok).toBe(false);
+  });
+});

src/lib/rate-limit.ts

@@ -26,17 +26,61 @@ export type RateLimitResult = {
   retryAfterSec: number;
 };
 
+/** Known AI crawler User-Agent substrings. Matched case-insensitively.
+ *  Requests from these agents bypass the per-IP throttle on read-only
+ *  endpoints because the crawlers route many independent user queries
+ *  through a small pool of shared egress IPs (PerplexityBot,
+ *  ChatGPT-User, ClaudeBot, etc.). A burst from one IP usually means
+ *  many distinct human questions, not abuse. UA spoofing is possible
+ *  but the affected endpoints are read-only and explicitly designed to
+ *  be cited, so the worst case of a spoofer is what we already want. */
+// Keep in sync with robots.txt (src/app/robots.ts). Any UA explicitly
+// allowed there should bypass the per IP throttle here too, otherwise
+// AI engines we want citations from will get 429s when many independent
+// user queries share their small egress IP pool.
+const AI_CRAWLER_RE =
+  /GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|anthropic-ai|Claude-Web|PerplexityBot|Perplexity-User|Google-Extended|GoogleOther|CCBot|Bytespider|Applebot-Extended|Meta-ExternalAgent|Meta-ExternalFetcher|cohere-ai|Amazonbot|Diffbot|YouBot|DuckAssistBot|AI2Bot|Ai2Bot-Dolma|Kagibot|FacebookBot|MistralAI-User|TimpiBot|Webzio-Extended/i;
+
+/** True when the given User-Agent matches a known AI crawler pattern. */
+export function isAiCrawler(userAgent: string | null | undefined): boolean {
+  if (!userAgent) return false;
+  return AI_CRAWLER_RE.test(userAgent);
+}
+
+/** Open-ended success used when bypassing the bucket for an AI crawler.
+ *  Same shape every caller already handles. */
+function aiBypassResult(): RateLimitResult {
+  return { ok: true, remaining: Number.POSITIVE_INFINITY, retryAfterSec: 0 };
+}
+
 /**
  * Check & consume a token for `key`. Returns ok:false when the bucket is
  * empty, with retryAfterSec hint for clients.
  *
  * Bucket refills linearly: `capacity` tokens over `windowSec` seconds.
+ *
+ * Optional `req`: when provided, the request's User-Agent is inspected
+ * and a match against the AI crawler allowlist short-circuits the bucket
+ * (no token consumed). Pass it on read-only endpoints meant to be cited
+ * (citable, llm-context, stat, mcp, etc.). Omit on write endpoints or
+ * anywhere UA-based bypass is undesired.
  */
 export function rateLimit(
   key: string,
   capacity: number,
   windowSec: number,
+  req?: Request,
 ): RateLimitResult {
+  if (req) {
+    const ua = req.headers.get("user-agent");
+    if (isAiCrawler(ua)) {
+      if (process.env.AI_BYPASS_DEBUG === "true") {
+        // Trimmed to keep noisy UA blobs out of the log line.
+        console.log(`[ai-bypass] granted to ${(ua ?? "").slice(0, 80)}`);
+      }
+      return aiBypassResult();
+    }
+  }
   if (capacity <= 0 || windowSec <= 0) {
     // Defensive: a misconfigured caller would otherwise produce Infinity
     // retry-after via /0 below. Treat as "denied with finite backoff".