diff --git a/ccrawl/client.go b/ccrawl/client.go
index a2c3ad8..95ec298 100644
--- a/ccrawl/client.go
+++ b/ccrawl/client.go
@@ -137,19 +137,28 @@ func (h *HTTPClient) DataURL(path string) string { return FileURL(path, h.source
 
 // Get fetches url with retries.
 func (h *HTTPClient) Get(ctx context.Context, url string) (*http.Response, error) {
-	return h.doWith(ctx, h.c, url, "")
+	return h.doWith(ctx, h.c, url, "", false)
+}
+
+// getStatus fetches url and hands back whatever the server finally said, where
+// Get would report a retryable status as an error. Callers that have to tell a
+// 403 from a dead host need the difference: to robots.txt a 403 is "no rules
+// here" and a failed connection is "assume everything is off limits", and one
+// error value cannot say which happened.
+func (h *HTTPClient) getStatus(ctx context.Context, url string) (*http.Response, error) {
+	return h.doWith(ctx, h.c, url, "", true)
 }
 
 // GetRange fetches the [offset, offset+length) byte span of url.
 func (h *HTTPClient) GetRange(ctx context.Context, url string, offset, length int64) (*http.Response, error) {
 	rangeHdr := fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)
-	return h.doWith(ctx, h.c, url, rangeHdr)
+	return h.doWith(ctx, h.c, url, rangeHdr, false)
 }
 
 // GetDownload fetches url with no client timeout (relies on ctx cancellation),
 // for large archive bodies.
 func (h *HTTPClient) GetDownload(ctx context.Context, url string) (*http.Response, error) {
-	return h.doWith(ctx, h.download, url, "")
+	return h.doWith(ctx, h.download, url, "", false)
 }
 
 // ContentLength returns the total size of url. It sends a one-byte range request
@@ -189,7 +198,10 @@ func (h *HTTPClient) FetchBytes(ctx context.Context, url string) ([]byte, error)
 	return io.ReadAll(resp.Body)
 }
 
-func (h *HTTPClient) doWith(ctx context.Context, client *http.Client, url, rangeHdr string) (*http.Response, error) {
+// doWith runs the request with the retry, throttle and backoff policy. When
+// keepStatus is set, a retryable status that survives the last attempt is
+// returned as a response instead of an error.
+func (h *HTTPClient) doWith(ctx context.Context, client *http.Client, url, rangeHdr string, keepStatus bool) (*http.Response, error) {
 	var last error
 	var retryAfter time.Duration // server-advised wait carried from the prior attempt
 	var serverWait bool          // whether the prior attempt sent a Retry-After
@@ -204,12 +216,9 @@ func (h *HTTPClient) doWith(ctx context.Context, client *http.Client, url, range
 			return nil, err
 		}
 		// s3://bucket/key is not something net/http can dial. The bucket is
-		// public, so the REST endpoint for it is a plain anonymous GET and every
+		// public, so the REST endpoint for it is a plain anonymous GET, and every
 		// other line in this loop, the throttle, the retries, the backoff, applies
 		// to it unchanged.
-		// s3://bucket/key is not something net/http can dial. It becomes a GET
-		// against the bucket's REST endpoint, and every other line in this loop,
-		// the throttle, the retries, the backoff, applies to it unchanged.
 		fetch, isS3 := s3Endpoint(url, s3BucketRegion)
 		if !isS3 {
 			fetch = url
@@ -250,6 +259,9 @@ func (h *HTTPClient) doWith(ctx context.Context, client *http.Client, url, range
 			}
 		}
 		if retryableStatus(resp.StatusCode) {
+			if keepStatus && attempt == h.retries {
+				return resp, nil
+			}
 			// A 503/429 commonly carries Retry-After; honor it on the next loop.
 			retryAfter, serverWait = parseRetryAfter(resp.Header.Get("Retry-After"))
 			_ = resp.Body.Close()
diff --git a/ccrawl/crawl.go b/ccrawl/crawl.go
index d744ce9..d6ea59d 100644
--- a/ccrawl/crawl.go
+++ b/ccrawl/crawl.go
@@ -1,7 +1,6 @@
 package ccrawl
 
 import (
-	"bufio"
 	"bytes"
 	"compress/gzip"
 	"context"
@@ -12,7 +11,6 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/andybalholm/brotli"
@@ -20,6 +18,8 @@ import (
 
 // The frontier itself lives in frontier.go, on disk. It used to be a heap and
 // a map right here, which is fine until the seed set is bigger than memory.
+// robots.txt lives in robots.go for the same reason: RFC 9309 is more than the
+// dozen lines of prefix matching that used to sit in this file.
 
 // ContentSHA1 returns the hex SHA-1 of raw content bytes (matches CC's digest
 // field in CDX records).
@@ -29,128 +29,6 @@ func ContentSHA1(content []byte) string {
 	return hex.EncodeToString(h.Sum(nil))
 }
 
-// ── robots.txt cache ──────────────────────────────────────────────────────────
-
-// RobotsRule is one allow/disallow rule from robots.txt.
-type RobotsRule struct {
-	Allow   bool
-	Pattern string
-}
-
-// RobotsEntry is a cached robots.txt for one host.
-type RobotsEntry struct {
-	Rules      []RobotsRule
-	CrawlDelay time.Duration
-	ExpiresAt  int64 // Unix timestamp
-}
-
-// IsAllowed reports whether the given path is allowed. The most specific
-// (longest) matching rule wins, following the standard robots.txt precedence.
-func (e *RobotsEntry) IsAllowed(path string) bool {
-	best := -1
-	bestAllow := true
-	for _, r := range e.Rules {
-		if strings.HasPrefix(path, r.Pattern) && len(r.Pattern) > best {
-			best = len(r.Pattern)
-			bestAllow = r.Allow
-		}
-	}
-	return bestAllow // default allow when no rule matches (best == -1 → true)
-}
-
-// RobotsCache caches parsed robots.txt per host with a TTL.
-type RobotsCache struct {
-	mu      sync.RWMutex
-	entries map[string]*RobotsEntry
-	ttl     time.Duration
-	ua      string // user-agent to match in robots.txt
-}
-
-// NewRobotsCache creates a cache with the given TTL and user-agent string.
-func NewRobotsCache(ttl time.Duration, userAgent string) *RobotsCache {
-	return &RobotsCache{
-		entries: make(map[string]*RobotsEntry),
-		ttl:     ttl,
-		ua:      userAgent,
-	}
-}
-
-// Get returns the cached robots entry for host, or nil if not cached or expired.
-func (rc *RobotsCache) Get(host string) *RobotsEntry {
-	rc.mu.RLock()
-	defer rc.mu.RUnlock()
-	e, ok := rc.entries[host]
-	if !ok || time.Now().Unix() >= e.ExpiresAt {
-		return nil
-	}
-	return e
-}
-
-// Put stores a robots entry for host.
-func (rc *RobotsCache) Put(host string, e *RobotsEntry) {
-	e.ExpiresAt = time.Now().Add(rc.ttl).Unix()
-	rc.mu.Lock()
-	rc.entries[host] = e
-	rc.mu.Unlock()
-}
-
-// FetchRobots fetches and parses robots.txt for the given host. On timeout or
-// 5xx, returns a permissive entry (allow all). On 404, returns allow all
-// permanently. The caller should Put the result into the cache.
-func FetchRobots(ctx context.Context, h *HTTPClient, host, scheme string) *RobotsEntry {
-	robotsURL := scheme + "://" + host + "/robots.txt"
-	resp, err := h.Get(ctx, robotsURL)
-	if err != nil || resp.StatusCode == 404 {
-		return &RobotsEntry{} // allow all
-	}
-	defer func() { _ = resp.Body.Close() }()
-	if resp.StatusCode != 200 {
-		return &RobotsEntry{} // allow all on 4xx/5xx
-	}
-	return parseRobots(resp.Body)
-}
-
-// parseRobots parses a robots.txt body (simplified: only User-agent:*/all,
-// Disallow, Allow, and Crawl-delay).
-func parseRobots(r io.Reader) *RobotsEntry {
-	entry := &RobotsEntry{}
-	var active bool // true when current User-agent block applies to us
-	sc := bufio.NewScanner(r)
-	for sc.Scan() {
-		line := strings.TrimSpace(sc.Text())
-		if strings.HasPrefix(line, "#") || line == "" {
-			continue
-		}
-		key, val, ok := strings.Cut(line, ":")
-		if !ok {
-			continue
-		}
-		key = strings.TrimSpace(strings.ToLower(key))
-		val = strings.TrimSpace(val)
-		switch key {
-		case "user-agent":
-			active = val == "*" || strings.EqualFold(val, "ccrawl")
-		case "disallow":
-			if active && val != "" {
-				entry.Rules = append(entry.Rules, RobotsRule{Allow: false, Pattern: val})
-			}
-		case "allow":
-			if active && val != "" {
-				entry.Rules = append(entry.Rules, RobotsRule{Allow: true, Pattern: val})
-			}
-		case "crawl-delay":
-			if active {
-				var d float64
-				_, _ = fmt.Sscan(val, &d)
-				if d > 0 {
-					entry.CrawlDelay = time.Duration(d * float64(time.Second))
-				}
-			}
-		}
-	}
-	return entry
-}
-
 // ── crawl result ──────────────────────────────────────────────────────────────
 
 // CrawlResult is the output of fetching a single URL.
diff --git a/ccrawl/crawl_test.go b/ccrawl/crawl_test.go
index 941375d..be7c785 100644
--- a/ccrawl/crawl_test.go
+++ b/ccrawl/crawl_test.go
@@ -108,74 +108,6 @@ func TestContentSHA1(t *testing.T) {
 	}
 }
 
-func TestParseRobots(t *testing.T) {
-	robotsTxt := `User-agent: *
-Disallow: /private/
-Allow: /private/public.html
-Crawl-delay: 2
-
-User-agent: googlebot
-Disallow: /admin/`
-	entry := parseRobots(strings.NewReader(robotsTxt))
-	if len(entry.Rules) == 0 {
-		t.Fatal("expected rules to be parsed")
-	}
-	if entry.CrawlDelay != 2*time.Second {
-		t.Errorf("Crawl-delay = %s, want 2s", entry.CrawlDelay)
-	}
-	// /private/ should be disallowed
-	if entry.IsAllowed("/private/secret.html") {
-		t.Error("/private/secret.html should be disallowed")
-	}
-	// /private/public.html should be allowed (Allow rule)
-	if !entry.IsAllowed("/private/public.html") {
-		t.Error("/private/public.html should be allowed")
-	}
-	// /public/ should be allowed (not in rules)
-	if !entry.IsAllowed("/public/page.html") {
-		t.Error("/public/page.html should be allowed")
-	}
-}
-
-func TestRobotsCache(t *testing.T) {
-	rc := NewRobotsCache(24*time.Hour, "ccrawl")
-	if rc.Get("example.com") != nil {
-		t.Error("empty cache should return nil")
-	}
-	e := &RobotsEntry{Rules: []RobotsRule{{Allow: false, Pattern: "/admin/"}}}
-	rc.Put("example.com", e)
-	got := rc.Get("example.com")
-	if got == nil {
-		t.Fatal("Put then Get should return entry")
-	}
-	if len(got.Rules) != 1 {
-		t.Errorf("expected 1 rule, got %d", len(got.Rules))
-	}
-}
-
-func TestFetchRobots(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/robots.txt" {
-			w.Header().Set("Content-Type", "text/plain")
-			_, _ = w.Write([]byte("User-agent: *\nDisallow: /secret/\n"))
-		}
-	}))
-	defer srv.Close()
-
-	h := NewHTTPClient(DefaultConfig())
-	host := strings.TrimPrefix(srv.URL, "http://")
-	entry := FetchRobots(context.Background(), h, host, "http")
-	if entry == nil {
-		t.Fatal("FetchRobots returned nil")
-	}
-	if entry.IsAllowed("/secret/page") {
-		t.Error("/secret/ should be disallowed")
-	}
-	if !entry.IsAllowed("/public/page") {
-		t.Error("/public/ should be allowed")
-	}
-}
-
 func TestCrawlURL(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")
diff --git a/ccrawl/frontier.go b/ccrawl/frontier.go
index 5175db7..1d41612 100644
--- a/ccrawl/frontier.go
+++ b/ccrawl/frontier.go
@@ -706,6 +706,22 @@ func (f *Frontier) Retry(e FrontierEntry, at int64) error {
 	return nil
 }
 
+// Hold keeps a host idle until at least the given time, on top of whatever the
+// frontier's own politeness delay already reserved.
+//
+// This is where a robots.txt Crawl-delay lands. Pop reserves the configured
+// delay when it hands a URL out, because at that point nobody has read the
+// host's robots.txt yet; the caller that fetches it learns the host wants ten
+// seconds and says so here. Nothing is ever brought forward, so the effective
+// delay is the longest anyone asked for.
+func (f *Frontier) Hold(host string, until int64) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if at := f.hostNextAt(host); at < until {
+		f.setHostNextAt(host, until)
+	}
+}
+
 // hostNextAt reads a host's next eligible time, from the LRU when it is hot,
 // from the unflushed writes when it is not, and from the database otherwise. A
 // crawl touches a small set of hosts hard and a long tail once, which is the
diff --git a/ccrawl/robots.go b/ccrawl/robots.go
new file mode 100644
index 0000000..ad991bb
--- /dev/null
+++ b/ccrawl/robots.go
@@ -0,0 +1,405 @@
+package ccrawl
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// robots.txt, to RFC 9309.
+//
+// What was here before matched literal path prefixes, recognised exactly two
+// user agent names, and read a 5xx on /robots.txt as permission to crawl. That
+// last one is the one that matters. A site whose robots endpoint is failing is
+// a site that cannot tell us to stop, and crawling it anyway is how a crawler
+// ends up in a block list and a mailbox full of abuse reports. RFC 9309 section
+// 2.3.1.4 says an unreachable robots.txt means the whole site is disallowed,
+// and that is what this does.
+
+const (
+	// robotsMaxBytes is the parse limit from RFC 9309 section 2.5, which asks
+	// crawlers to handle at least 500 kibibytes. Anything past it is discarded
+	// rather than treated as an error, so a file that is too long still yields
+	// the rules it managed to state.
+	robotsMaxBytes = 500 << 10
+
+	// robotsErrorTTL is how long an unreachable robots.txt is remembered. The
+	// spec allows caching a failure for up to 30 days. Five minutes is the
+	// friendlier reading of the same rule: the host stays disallowed while it is
+	// down and gets crawled again as soon as it recovers.
+	robotsErrorTTL = 5 * time.Minute
+)
+
+// RobotsRule is one allow or disallow rule. The pattern is a path prefix in
+// which * stands for any run of characters and a trailing $ anchors the match
+// to the end of the path.
+type RobotsRule struct {
+	Allow   bool
+	Pattern string
+}
+
+// RobotsEntry is a parsed robots.txt for one host, reduced to the group that
+// applies to us.
+type RobotsEntry struct {
+	Rules      []RobotsRule
+	CrawlDelay time.Duration
+	Sitemaps   []string
+	ExpiresAt  int64 // Unix timestamp, set by RobotsCache.Put
+
+	// TTL, when set, overrides the cache TTL for this entry. It is how a
+	// disallow that came from a failed fetch gets retried in minutes while a
+	// robots.txt we actually read is kept for a day.
+	TTL time.Duration
+}
+
+// IsAllowed reports whether the crawler may fetch a path. The longest matching
+// rule wins and an allow beats a disallow of the same length, per RFC 9309
+// section 2.2.2. A path no rule matches is allowed.
+func (e *RobotsEntry) IsAllowed(path string) bool {
+	if e == nil {
+		return true
+	}
+	if path == "" {
+		path = "/"
+	}
+	path = robotsNormalize(path)
+	best, allow := -1, true
+	for _, r := range e.Rules {
+		if !robotsMatch(r.Pattern, path) {
+			continue
+		}
+		if n := len(r.Pattern); n > best || (n == best && r.Allow) {
+			best, allow = n, r.Allow
+		}
+	}
+	return allow
+}
+
+// robotsMatch reports whether a rule pattern matches a path. The pattern is
+// anchored at the start of the path and free at the end unless it ends in $.
+func robotsMatch(pattern, path string) bool {
+	anchored := strings.HasSuffix(pattern, "$")
+	if anchored {
+		pattern = pattern[:len(pattern)-1]
+	}
+	parts := strings.Split(pattern, "*")
+	if !strings.HasPrefix(path, parts[0]) {
+		return false
+	}
+	pos := len(parts[0])
+	for i, part := range parts[1:] {
+		if anchored && i == len(parts)-2 {
+			// The last literal has to sit at the very end of the path, and there
+			// has to be room for it after everything matched so far.
+			return len(path)-len(part) >= pos && strings.HasSuffix(path, part)
+		}
+		// Leftmost match, which is what makes the search linear: a later literal
+		// never needs an earlier one to give ground, because the pattern between
+		// them is a * that absorbs anything.
+		at := strings.Index(path[pos:], part)
+		if at < 0 {
+			return false
+		}
+		pos += at + len(part)
+	}
+	if anchored {
+		return pos == len(path)
+	}
+	return true
+}
+
+// robotsNormalize puts a path and a rule pattern into the same encoding, since
+// RFC 9309 section 2.2.2 compares them octet by octet. Existing escapes are
+// upper cased rather than decoded and anything outside printable US-ASCII is
+// escaped, so a URL and the rule written for it agree however the site spelled
+// them.
+func robotsNormalize(s string) string {
+	if !robotsNeedsEscape(s) {
+		return s
+	}
+	const hex = "0123456789ABCDEF"
+	var b strings.Builder
+	b.Grow(len(s) + 8)
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		switch {
+		case c == '%' && i+2 < len(s) && isHexDigit(s[i+1]) && isHexDigit(s[i+2]):
+			b.WriteByte('%')
+			b.WriteByte(upperHexDigit(s[i+1]))
+			b.WriteByte(upperHexDigit(s[i+2]))
+			i += 2
+		case c <= 0x20 || c >= 0x7f || c == '%':
+			b.WriteByte('%')
+			b.WriteByte(hex[c>>4])
+			b.WriteByte(hex[c&0x0f])
+		default:
+			b.WriteByte(c)
+		}
+	}
+	return b.String()
+}
+
+func robotsNeedsEscape(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if c := s[i]; c <= 0x20 || c >= 0x7f || c == '%' {
+			return true
+		}
+	}
+	return false
+}
+
+func isHexDigit(c byte) bool {
+	return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F'
+}
+
+func upperHexDigit(c byte) byte {
+	if c >= 'a' && c <= 'f' {
+		return c - 'a' + 'A'
+	}
+	return c
+}
+
+// ── parsing ───────────────────────────────────────────────────────────────────
+
+// robotsGroup is one group of records: the user agent lines that open it and
+// the rules that follow until the next user agent line.
+type robotsGroup struct {
+	agents   []string
+	rules    []RobotsRule
+	delay    time.Duration
+	hasDelay bool
+}
+
+// parseRobots parses a robots.txt body and returns the rules that apply to
+// userAgent. Records for other user agents are read and discarded, so the
+// result is a plain rule list the caller can match paths against without
+// knowing which group it came from.
+func parseRobots(r io.Reader, userAgent string) *RobotsEntry {
+	var (
+		groups   []*robotsGroup
+		cur      *robotsGroup
+		sitemaps []string
+		opening  bool // consecutive user agent lines open one group, not several
+	)
+	sc := bufio.NewScanner(io.LimitReader(r, robotsMaxBytes))
+	// Sitemap lines can be long and a single overlong line should not stop the
+	// parse, so the buffer is generous rather than the 64 KB default.
+	sc.Buffer(make([]byte, 0, 8<<10), 1<<20)
+	for sc.Scan() {
+		line := sc.Text()
+		if i := strings.IndexByte(line, '#'); i >= 0 {
+			line = line[:i]
+		}
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		key, val, ok := strings.Cut(line, ":")
+		if !ok {
+			continue
+		}
+		key = strings.ToLower(strings.TrimSpace(key))
+		val = strings.TrimSpace(val)
+		switch key {
+		case "user-agent", "useragent":
+			if !opening {
+				cur = &robotsGroup{}
+				groups = append(groups, cur)
+				opening = true
+			}
+			// A group still opens on a nameless User-agent line, so the rules
+			// under it stay out of the previous group, but it names nobody. The
+			// empty string is a prefix of every product token, so keeping it
+			// would let one malformed line beat the wildcard group for every
+			// crawler that reads the file.
+			if val != "" {
+				cur.agents = append(cur.agents, strings.ToLower(val))
+			}
+		case "allow", "disallow":
+			opening = false
+			// Rules before any user agent line belong to no group, and RFC 9309
+			// section 2.2.1 says to ignore them. An empty Disallow is the
+			// documented way to say "nothing is disallowed", which is the absence
+			// of a rule rather than a rule matching everything.
+			if cur == nil || val == "" {
+				continue
+			}
+			cur.rules = append(cur.rules, RobotsRule{
+				Allow:   key == "allow",
+				Pattern: robotsNormalize(val),
+			})
+		case "crawl-delay":
+			opening = false
+			if cur == nil {
+				continue
+			}
+			if d, err := strconv.ParseFloat(val, 64); err == nil && d > 0 {
+				cur.delay, cur.hasDelay = time.Duration(d*float64(time.Second)), true
+			}
+		case "sitemap":
+			// Not a group record. It applies to the whole file wherever it
+			// appears, and it does not interrupt the group being read.
+			if val != "" {
+				sitemaps = append(sitemaps, val)
+			}
+		}
+	}
+	entry := matchGroups(groups, robotsToken(userAgent))
+	entry.Sitemaps = sitemaps
+	return entry
+}
+
+// matchGroups reduces the parsed groups to the one that applies to token.
+//
+// RFC 9309 section 2.2.1: the most specific matching product token wins, the
+// wildcard group is the fallback and never merges with a specific one, and
+// groups that name the same token are followed as one.
+func matchGroups(groups []*robotsGroup, token string) *RobotsEntry {
+	best := -1
+	for _, g := range groups {
+		for _, a := range g.agents {
+			if a == "*" || !strings.HasPrefix(token, a) {
+				continue
+			}
+			if len(a) > best {
+				best = len(a)
+			}
+		}
+	}
+	entry := &RobotsEntry{}
+	for _, g := range groups {
+		if !groupApplies(g, token, best) {
+			continue
+		}
+		entry.Rules = append(entry.Rules, g.rules...)
+		// The longest delay across merged groups, because the point of the field
+		// is a floor on how hard we hit the host.
+		if g.hasDelay && g.delay > entry.CrawlDelay {
+			entry.CrawlDelay = g.delay
+		}
+	}
+	return entry
+}
+
+// groupApplies reports whether a group is part of the set we follow, given the
+// specificity of the best matching product token in the file. A best of -1
+// means nothing named us and the wildcard group is what is left.
+func groupApplies(g *robotsGroup, token string, best int) bool {
+	for _, a := range g.agents {
+		if best < 0 {
+			if a == "*" {
+				return true
+			}
+			continue
+		}
+		if a != "*" && len(a) == best && strings.HasPrefix(token, a) {
+			return true
+		}
+	}
+	return false
+}
+
+// robotsToken reduces a User-Agent header to the product token robots.txt is
+// written against, so that "CCrawl/2.0 (+https://example.com/bot)" is matched
+// by a group naming ccrawl. Matching is case insensitive, and a group naming a
+// prefix of our token applies to us, which is how a site writes one rule for a
+// crawler and all of its variants.
+func robotsToken(userAgent string) string {
+	token := strings.TrimSpace(userAgent)
+	if i := strings.IndexAny(token, "/ \t"); i >= 0 {
+		token = token[:i]
+	}
+	return strings.ToLower(token)
+}
+
+// ── fetching and caching ──────────────────────────────────────────────────────
+
+// RobotsCache caches parsed robots.txt per host with a TTL.
+type RobotsCache struct {
+	mu      sync.RWMutex
+	entries map[string]*RobotsEntry
+	ttl     time.Duration
+	ua      string // the crawler's user agent, matched against the groups
+}
+
+// NewRobotsCache creates a cache with the given TTL and user agent string.
+func NewRobotsCache(ttl time.Duration, userAgent string) *RobotsCache {
+	return &RobotsCache{
+		entries: make(map[string]*RobotsEntry),
+		ttl:     ttl,
+		ua:      userAgent,
+	}
+}
+
+// Get returns the cached robots entry for host, or nil if it is missing or
+// expired.
+func (rc *RobotsCache) Get(host string) *RobotsEntry {
+	rc.mu.RLock()
+	defer rc.mu.RUnlock()
+	e, ok := rc.entries[host]
+	if !ok || time.Now().Unix() >= e.ExpiresAt {
+		return nil
+	}
+	return e
+}
+
+// Put stores a robots entry for host. An entry carrying its own TTL keeps it,
+// which is how a disallow from a failed fetch expires in minutes instead of
+// sticking for the cache's usual day.
+func (rc *RobotsCache) Put(host string, e *RobotsEntry) {
+	ttl := rc.ttl
+	if e.TTL > 0 {
+		ttl = e.TTL
+	}
+	e.ExpiresAt = time.Now().Add(ttl).Unix()
+	rc.mu.Lock()
+	rc.entries[host] = e
+	rc.mu.Unlock()
+}
+
+// Fetch returns the robots entry for a host, fetching and caching it on a miss.
+func (rc *RobotsCache) Fetch(ctx context.Context, h *HTTPClient, host, scheme string) *RobotsEntry {
+	if e := rc.Get(host); e != nil {
+		return e
+	}
+	e := FetchRobots(ctx, h, host, scheme, rc.ua)
+	rc.Put(host, e)
+	return e
+}
+
+// FetchRobots fetches and parses robots.txt for one host.
+//
+// The status handling is RFC 9309 section 2.3.1 and it is not symmetric. A 2xx
+// is parsed. A 4xx means there is no robots.txt, which the spec reads as the
+// whole site being open. A 3xx that outlived the client's redirect following is
+// unresolvable and section 2.3.1.2 treats it the same way. A 5xx or a network
+// failure means the site could not tell us anything, and section 2.3.1.4 is
+// explicit that this is a complete disallow rather than a shrug.
+func FetchRobots(ctx context.Context, h *HTTPClient, host, scheme, userAgent string) *RobotsEntry {
+	resp, err := h.getStatus(ctx, scheme+"://"+host+"/robots.txt")
+	if err != nil {
+		return robotsUnreachable()
+	}
+	defer func() { _ = resp.Body.Close() }()
+	switch {
+	case resp.StatusCode >= 200 && resp.StatusCode < 300:
+		return parseRobots(resp.Body, userAgent)
+	case resp.StatusCode >= 500:
+		return robotsUnreachable()
+	default:
+		return &RobotsEntry{}
+	}
+}
+
+// robotsUnreachable is the entry for a host that could not be asked: everything
+// disallowed, and remembered only briefly so the host is retried soon.
+func robotsUnreachable() *RobotsEntry {
+	return &RobotsEntry{
+		Rules: []RobotsRule{{Pattern: "/"}},
+		TTL:   robotsErrorTTL,
+	}
+}
diff --git a/ccrawl/robots_live_test.go b/ccrawl/robots_live_test.go
new file mode 100644
index 0000000..f8d1c1d
--- /dev/null
+++ b/ccrawl/robots_live_test.go
@@ -0,0 +1,87 @@
+package ccrawl
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestRobotsAgainstRealSites replays a corpus of robots.txt files served by
+// real sites and checks every verdict against a second implementation of the
+// same spec.
+//
+// The hand written tests prove the parser matches RFC 9309 as we read it, which
+// is worth something and is not the same as being right. The web is full of
+// files with typos, duplicate groups, rules before any user agent line, and
+// patterns nobody would think to invent, and the way to find out what we do
+// with those is to run them.
+//
+// The corpus is built by scripts/robots-corpus.py, which fetches the files and
+// records what protego, the parser Scrapy uses, concludes about each one:
+//
+//	CCRAWL_ROBOTS_CORPUS=/tmp/robots-corpus go test ./ccrawl -run RealSites -v
+//
+// It is skipped by default because it depends on files this repository does not
+// own and on a Python package to regenerate.
+func TestRobotsAgainstRealSites(t *testing.T) {
+	dir := os.Getenv("CCRAWL_ROBOTS_CORPUS")
+	if dir == "" {
+		t.Skip("set CCRAWL_ROBOTS_CORPUS to a directory built by scripts/robots-corpus.py")
+	}
+	var corpus struct {
+		UserAgent string `json:"user_agent"`
+		Sites     map[string]struct {
+			Allowed    map[string]bool `json:"allowed"`
+			Skipped    []string        `json:"skipped"`
+			CrawlDelay float64         `json:"crawl_delay"`
+			Sitemaps   []string        `json:"sitemaps"`
+		} `json:"sites"`
+	}
+	raw, err := os.ReadFile(filepath.Join(dir, "verdicts.json"))
+	if err != nil {
+		t.Fatalf("read verdicts: %v", err)
+	}
+	if err := json.Unmarshal(raw, &corpus); err != nil {
+		t.Fatalf("parse verdicts: %v", err)
+	}
+
+	var probes, skipped, disagreements int
+	for site, want := range corpus.Sites {
+		body, err := os.ReadFile(filepath.Join(dir, site+".txt"))
+		if err != nil {
+			t.Fatalf("read %s: %v", site, err)
+		}
+		e := parseRobots(strings.NewReader(string(body)), corpus.UserAgent)
+		skipped += len(want.Skipped)
+
+		// Sorted, because a map iterates differently every run and a failure
+		// should name the same first path every time.
+		paths := make([]string, 0, len(want.Allowed))
+		for p := range want.Allowed {
+			paths = append(paths, p)
+		}
+		sort.Strings(paths)
+		for _, p := range paths {
+			probes++
+			if got := e.IsAllowed(p); got != want.Allowed[p] {
+				disagreements++
+				t.Errorf("%s: IsAllowed(%q) = %v, protego says %v", site, p, got, want.Allowed[p])
+			}
+		}
+		if wantDelay := time.Duration(want.CrawlDelay * float64(time.Second)); e.CrawlDelay != wantDelay {
+			t.Errorf("%s: CrawlDelay = %s, protego says %s", site, e.CrawlDelay, wantDelay)
+		}
+		if got := len(e.Sitemaps); got != len(want.Sitemaps) {
+			t.Errorf("%s: found %d sitemaps, protego found %d", site, got, len(want.Sitemaps))
+		}
+	}
+	if probes == 0 {
+		t.Fatal("the corpus is empty, rebuild it with scripts/robots-corpus.py")
+	}
+	t.Logf("%d sites, %d probes, %d disagreements, %d probes protego cannot answer",
+		len(corpus.Sites), probes, disagreements, skipped)
+}
diff --git a/ccrawl/robots_test.go b/ccrawl/robots_test.go
new file mode 100644
index 0000000..1991d34
--- /dev/null
+++ b/ccrawl/robots_test.go
@@ -0,0 +1,469 @@
+package ccrawl
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+// The path matching table is RFC 9309 section 2.2.2, transcribed. Every case
+// below is one the RFC spells out, which is the point: this is the part of the
+// spec sites actually write against, and a parser that gets it approximately
+// right blocks pages it should fetch and fetches pages it was told not to.
+func TestRobotsPathMatching(t *testing.T) {
+	cases := []struct {
+		pattern string
+		match   []string
+		miss    []string
+	}{{
+		pattern: "/",
+		match:   []string{"/", "/anything", "/deep/path.html"},
+	}, {
+		pattern: "/*",
+		match:   []string{"/", "/anything", "/deep/path.html"},
+	}, {
+		pattern: "/fish",
+		match: []string{"/fish", "/fish.html", "/fish/salmon.html", "/fishheads",
+			"/fishheads/yummy.html", "/fish.html?id=anything"},
+		miss: []string{"/Fish.asp", "/catfish", "/?id=fish", "/desert/fish"},
+	}, {
+		pattern: "/fish*",
+		match:   []string{"/fish", "/fish.html", "/fish/salmon.html", "/fishheads"},
+		miss:    []string{"/Fish.asp", "/catfish", "/?id=fish"},
+	}, {
+		pattern: "/fish/",
+		match:   []string{"/fish/", "/fish/?id=anything", "/fish/salmon.html"},
+		miss:    []string{"/fish", "/fish.html", "/Fish/Salmon.asp"},
+	}, {
+		pattern: "/*.php",
+		match: []string{"/index.php", "/filename.php", "/folder/filename.php",
+			"/folder/filename.php?parameters", "/folder/any.php.file.html", "/filename.php/"},
+		miss: []string{"/", "/windows.PHP"},
+	}, {
+		pattern: "/*.php$",
+		match:   []string{"/filename.php", "/folder/filename.php"},
+		miss:    []string{"/filename.php?parameters", "/filename.php/", "/filename.php5", "/windows.PHP"},
+	}, {
+		pattern: "/fish*.php",
+		match:   []string{"/fish.php", "/fishheads/catfish.php?parameters"},
+		miss:    []string{"/Fish.PHP"},
+	}}
+
+	for _, c := range cases {
+		e := &RobotsEntry{Rules: []RobotsRule{{Pattern: c.pattern}}}
+		for _, p := range c.match {
+			if e.IsAllowed(p) {
+				t.Errorf("Disallow: %s should have matched %s", c.pattern, p)
+			}
+		}
+		for _, p := range c.miss {
+			if !e.IsAllowed(p) {
+				t.Errorf("Disallow: %s should not have matched %s", c.pattern, p)
+			}
+		}
+	}
+}
+
+// The file is the example from RFC 9309 section 5.2, and so are the four
+// verdicts. quxbot is the interesting one: an empty group naming a crawler
+// still counts as that crawler's group, so it does not fall back to the
+// wildcard rules and everything is allowed.
+const rfc9309Example = `User-Agent: *
+Disallow: *.gif$
+Disallow: /example/
+Allow: /publications/
+
+User-Agent: foobot
+Disallow:/
+Allow:/example/page.html
+Allow:/example/allowed.gif
+
+User-Agent: barbot
+User-Agent: bazbot
+Disallow: /example/page.html
+
+User-Agent: quxbot
+`
+
+func TestRobotsGroupSelection(t *testing.T) {
+	cases := []struct {
+		ua      string
+		allowed []string
+		denied  []string
+	}{{
+		// Nothing names this one, so the wildcard group applies.
+		ua:      "randombot",
+		allowed: []string{"/publications/paper.html", "/index.html"},
+		denied:  []string{"/example/page.html", "/img/logo.gif"},
+	}, {
+		ua:      "foobot",
+		allowed: []string{"/example/page.html", "/example/allowed.gif"},
+		denied:  []string{"/", "/index.html", "/publications/paper.html", "/img/logo.gif"},
+	}, {
+		// Matching is case insensitive and runs on the product token, so the
+		// version and the contact URL in a real User-Agent header do not matter.
+		ua:      "FooBot/2.1 (+https://example.com/bot)",
+		allowed: []string{"/example/page.html"},
+		denied:  []string{"/index.html"},
+	}, {
+		ua:      "barbot",
+		allowed: []string{"/index.html", "/img/logo.gif", "/example/other.html"},
+		denied:  []string{"/example/page.html"},
+	}, {
+		ua:      "bazbot",
+		allowed: []string{"/index.html", "/img/logo.gif"},
+		denied:  []string{"/example/page.html"},
+	}, {
+		ua:      "quxbot",
+		allowed: []string{"/", "/example/page.html", "/img/logo.gif"},
+	}}
+
+	for _, c := range cases {
+		e := parseRobots(strings.NewReader(rfc9309Example), c.ua)
+		for _, p := range c.allowed {
+			if !e.IsAllowed(p) {
+				t.Errorf("%s: %s should be allowed", c.ua, p)
+			}
+		}
+		for _, p := range c.denied {
+			if e.IsAllowed(p) {
+				t.Errorf("%s: %s should be disallowed", c.ua, p)
+			}
+		}
+	}
+}
+
+func TestRobotsMostSpecificGroupWins(t *testing.T) {
+	const txt = `User-agent: *
+Disallow: /
+
+User-agent: ccrawl
+Disallow: /private/
+
+User-agent: ccrawl-images
+Disallow: /
+`
+	// A group naming a prefix of our token applies to us, which is how a site
+	// writes one rule for a crawler and all of its variants. The longest such
+	// token wins, and a more specific group for a sibling crawler is not ours.
+	e := parseRobots(strings.NewReader(txt), "CCrawl/2.0 (+https://example.com/bot)")
+	if !e.IsAllowed("/public/page.html") {
+		t.Error("the ccrawl group should have replaced the wildcard group")
+	}
+	if e.IsAllowed("/private/page.html") {
+		t.Error("/private/ should be disallowed by the ccrawl group")
+	}
+
+	img := parseRobots(strings.NewReader(txt), "ccrawl-images")
+	if img.IsAllowed("/public/page.html") {
+		t.Error("ccrawl-images should have taken its own group, which disallows everything")
+	}
+}
+
+func TestRobotsMergesGroupsNamingTheSameAgent(t *testing.T) {
+	const txt = `User-agent: ccrawl
+Disallow: /a/
+
+User-agent: ccrawl
+Disallow: /b/
+Crawl-delay: 3
+`
+	e := parseRobots(strings.NewReader(txt), "ccrawl")
+	if e.IsAllowed("/a/x") || e.IsAllowed("/b/x") {
+		t.Error("rules from both groups naming ccrawl should be followed")
+	}
+	if e.CrawlDelay != 3*time.Second {
+		t.Errorf("CrawlDelay = %s, want 3s", e.CrawlDelay)
+	}
+}
+
+func TestRobotsLongestMatchWinsAndAllowBreaksTies(t *testing.T) {
+	const txt = `User-agent: *
+Disallow: /private/
+Allow: /private/public.html
+Disallow: /both
+Allow: /both
+`
+	e := parseRobots(strings.NewReader(txt), "ccrawl")
+	if e.IsAllowed("/private/secret.html") {
+		t.Error("/private/secret.html should be disallowed")
+	}
+	if !e.IsAllowed("/private/public.html") {
+		t.Error("the longer Allow should win over the shorter Disallow")
+	}
+	if !e.IsAllowed("/both") {
+		t.Error("an Allow and a Disallow of equal length should resolve to allow")
+	}
+	if !e.IsAllowed("/elsewhere") {
+		t.Error("a path no rule matches should be allowed")
+	}
+}
+
+func TestRobotsEmptyDisallowAllowsEverything(t *testing.T) {
+	e := parseRobots(strings.NewReader("User-agent: *\nDisallow:\n"), "ccrawl")
+	if !e.IsAllowed("/anything") {
+		t.Error("an empty Disallow means nothing is disallowed")
+	}
+}
+
+func TestRobotsIgnoresNamelessGroup(t *testing.T) {
+	const txt = `User-agent:
+Disallow: /
+
+User-agent: *
+Disallow: /admin/
+`
+	e := parseRobots(strings.NewReader(txt), "ccrawl")
+	if !e.IsAllowed("/page.html") {
+		t.Error("a User-agent line naming nobody should not disallow the site")
+	}
+	if e.IsAllowed("/admin/x") {
+		t.Error("the wildcard group should still apply")
+	}
+}
+
+func TestRobotsIgnoresRulesBeforeAnyGroup(t *testing.T) {
+	e := parseRobots(strings.NewReader("Disallow: /\n\nUser-agent: *\nAllow: /\n"), "ccrawl")
+	if !e.IsAllowed("/page.html") {
+		t.Error("a rule outside any group should be ignored")
+	}
+}
+
+func TestRobotsComments(t *testing.T) {
+	const txt = `# leading comment
+User-agent: *   # trailing comment
+Disallow: /admin/ # another
+`
+	e := parseRobots(strings.NewReader(txt), "ccrawl")
+	if e.IsAllowed("/admin/x") {
+		t.Error("/admin/ should be disallowed with comments stripped")
+	}
+	if !e.IsAllowed("/admin") {
+		t.Error("the trailing comment should not have become part of the pattern")
+	}
+}
+
+func TestRobotsPercentEncoding(t *testing.T) {
+	// The rule and the URL spell the same path differently. RFC 9309 section
+	// 2.2.2 compares the percent encoded forms, so both sides get normalized and
+	// the rule bites.
+	e := parseRobots(strings.NewReader("User-agent: *\nDisallow: /caf%c3%a9/\n"), "ccrawl")
+	if e.IsAllowed("/caf%C3%A9/menu") {
+		t.Error("an escape written in lower case should match the same path in upper case")
+	}
+	if e.IsAllowed("/café/menu") {
+		t.Error("an unescaped path should match the escaped rule written for it")
+	}
+}
+
+func TestRobotsCollectsSitemaps(t *testing.T) {
+	const txt = `Sitemap: https://example.com/sitemap.xml
+
+User-agent: *
+Disallow: /admin/
+Sitemap: https://example.com/sitemap-news.xml
+`
+	e := parseRobots(strings.NewReader(txt), "ccrawl")
+	want := []string{"https://example.com/sitemap.xml", "https://example.com/sitemap-news.xml"}
+	if len(e.Sitemaps) != len(want) {
+		t.Fatalf("Sitemaps = %v, want %v", e.Sitemaps, want)
+	}
+	for i, w := range want {
+		if e.Sitemaps[i] != w {
+			t.Errorf("Sitemaps[%d] = %q, want %q", i, e.Sitemaps[i], w)
+		}
+	}
+	// A Sitemap line is not a group record and must not end the group it sits in.
+	if e.IsAllowed("/admin/x") {
+		t.Error("the Disallow before the second Sitemap should still apply")
+	}
+}
+
+func TestRobotsStopsAtTheSizeLimit(t *testing.T) {
+	// RFC 9309 section 2.5 asks for at least 500 kibibytes and says the rest is
+	// discarded. A rule past the limit is not an error and does not invalidate
+	// the rules before it.
+	var b strings.Builder
+	b.WriteString("User-agent: *\nDisallow: /early/\n")
+	for b.Len() < robotsMaxBytes {
+		b.WriteString("# padding to push the next rule past the parse limit\n")
+	}
+	b.WriteString("Disallow: /late/\n")
+
+	e := parseRobots(strings.NewReader(b.String()), "ccrawl")
+	if e.IsAllowed("/early/x") {
+		t.Error("the rule before the limit should have been parsed")
+	}
+	if !e.IsAllowed("/late/x") {
+		t.Error("the rule past the limit should have been discarded")
+	}
+}
+
+// ── fetching ──────────────────────────────────────────────────────────────────
+
+// robotsServer serves /robots.txt with a fixed status and body.
+func robotsServer(t *testing.T, status int, body string) (*httptest.Server, string) {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/robots.txt" {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(srv.Close)
+	return srv, strings.TrimPrefix(srv.URL, "http://")
+}
+
+// robotsClient is an HTTP client with the retries turned off, so a test that
+// asks for a 503 gets one back in milliseconds instead of after the default
+// five attempts and their backoff.
+func robotsClient() *HTTPClient {
+	cfg := DefaultConfig()
+	cfg.Retries = 0
+	cfg.Delay = 0
+	return NewHTTPClient(cfg)
+}
+
+func TestFetchRobots(t *testing.T) {
+	_, host := robotsServer(t, 200, "User-agent: *\nDisallow: /secret/\nCrawl-delay: 4\n")
+	e := FetchRobots(context.Background(), robotsClient(), host, "http", "ccrawl")
+	if e.IsAllowed("/secret/page") {
+		t.Error("/secret/ should be disallowed")
+	}
+	if !e.IsAllowed("/public/page") {
+		t.Error("/public/ should be allowed")
+	}
+	if e.CrawlDelay != 4*time.Second {
+		t.Errorf("CrawlDelay = %s, want 4s", e.CrawlDelay)
+	}
+}
+
+// The done-when from E5, and the reason the issue is filed as a bug: a host
+// whose robots endpoint is failing cannot tell us to stop, so we stop.
+func TestFetchRobotsDisallowsWhenUnreachable(t *testing.T) {
+	for _, status := range []int{500, 502, 503} {
+		_, host := robotsServer(t, status, "")
+		e := FetchRobots(context.Background(), robotsClient(), host, "http", "ccrawl")
+		if e.IsAllowed("/") || e.IsAllowed("/anything.html") {
+			t.Errorf("HTTP %d on robots.txt should disallow the whole host", status)
+		}
+		if e.TTL != robotsErrorTTL {
+			t.Errorf("HTTP %d: TTL = %s, want the short negative TTL %s", status, e.TTL, robotsErrorTTL)
+		}
+	}
+}
+
+func TestFetchRobotsDisallowsWhenHostIsDown(t *testing.T) {
+	srv, host := robotsServer(t, 200, "")
+	srv.Close() // nothing is listening, which is the network failure case
+	e := FetchRobots(context.Background(), robotsClient(), host, "http", "ccrawl")
+	if e.IsAllowed("/anything.html") {
+		t.Error("a host that cannot be reached at all should be disallowed")
+	}
+}
+
+func TestFetchRobotsAllowsWhenMissing(t *testing.T) {
+	// 404 and 403 both mean there is no robots.txt to read, which RFC 9309
+	// section 2.3.1.3 says leaves the site open. 403 is the one worth a test:
+	// the HTTP client treats it as retryable, and if the retry gave up with a
+	// plain error this would come back as a disallow.
+	for _, status := range []int{403, 404, 410} {
+		_, host := robotsServer(t, status, "")
+		e := FetchRobots(context.Background(), robotsClient(), host, "http", "ccrawl")
+		if !e.IsAllowed("/anything.html") {
+			t.Errorf("HTTP %d on robots.txt should leave the host allowed", status)
+		}
+	}
+}
+
+func TestRobotsCache(t *testing.T) {
+	rc := NewRobotsCache(24*time.Hour, "ccrawl")
+	if rc.Get("example.com") != nil {
+		t.Error("empty cache should return nil")
+	}
+	rc.Put("example.com", &RobotsEntry{Rules: []RobotsRule{{Pattern: "/admin/"}}})
+	got := rc.Get("example.com")
+	if got == nil {
+		t.Fatal("Put then Get should return the entry")
+	}
+	if len(got.Rules) != 1 {
+		t.Errorf("expected 1 rule, got %d", len(got.Rules))
+	}
+}
+
+func TestRobotsCacheKeepsFailuresBriefly(t *testing.T) {
+	rc := NewRobotsCache(24*time.Hour, "ccrawl")
+	rc.Put("example.com", robotsUnreachable())
+	e := rc.Get("example.com")
+	if e == nil {
+		t.Fatal("the negative entry should be cached")
+	}
+	// The failure expires on its own short clock rather than the cache's day, so
+	// a host that comes back up is crawled again in minutes.
+	if wait := time.Until(time.Unix(e.ExpiresAt, 0)); wait > robotsErrorTTL+time.Second {
+		t.Errorf("negative entry expires in %s, want at most %s", wait, robotsErrorTTL)
+	}
+}
+
+func TestRobotsCacheFetchesOnce(t *testing.T) {
+	var hits int
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits++
+		_, _ = w.Write([]byte("User-agent: *\nDisallow: /secret/\n"))
+	}))
+	defer srv.Close()
+	host := strings.TrimPrefix(srv.URL, "http://")
+
+	rc := NewRobotsCache(time.Hour, "ccrawl")
+	h := robotsClient()
+	for i := 0; i < 3; i++ {
+		if rc.Fetch(context.Background(), h, host, "http").IsAllowed("/secret/x") {
+			t.Fatal("/secret/ should be disallowed")
+		}
+	}
+	if hits != 1 {
+		t.Errorf("fetched robots.txt %d times, want 1", hits)
+	}
+}
+
+// The last done-when from E5. A Crawl-delay is only worth parsing if something
+// enforces it, and the thing that decides when a host may be touched again is
+// the frontier.
+func TestCrawlDelayHoldsTheHostInTheFrontier(t *testing.T) {
+	f := NewFrontier(time.Second) // the crawl's own delay, shorter than robots asks for
+	defer func() { _ = f.Close() }()
+
+	f.Add(FrontierEntry{URL: "https://slow.example/a", Host: "slow.example"})
+	f.Add(FrontierEntry{URL: "https://slow.example/b", Host: "slow.example"})
+
+	now := time.Now().Unix()
+	first, ok, err := f.Pop(now)
+	if err != nil || !ok {
+		t.Fatalf("first Pop: %v %v", ok, err)
+	}
+
+	delay := parseRobots(strings.NewReader("User-agent: *\nCrawl-delay: 10\n"), "ccrawl").CrawlDelay
+	if delay != 10*time.Second {
+		t.Fatalf("CrawlDelay = %s, want 10s", delay)
+	}
+	f.Hold(first.Host, now+int64(delay.Seconds()))
+
+	for _, after := range []int64{1, 5, 9} {
+		if _, ok, err := f.Pop(now + after); err != nil {
+			t.Fatalf("Pop: %v", err)
+		} else if ok {
+			t.Fatalf("second URL came out %ds after the first, want at least 10s", after)
+		}
+	}
+	if _, ok, err := f.Pop(now + 10); err != nil {
+		t.Fatalf("Pop: %v", err)
+	} else if !ok {
+		t.Error("the second URL should be available once the crawl delay has passed")
+	}
+}
diff --git a/cli/crawl_cmd.go b/cli/crawl_cmd.go
index 0d53e83..95fd9ba 100644
--- a/cli/crawl_cmd.go
+++ b/cli/crawl_cmd.go
@@ -138,7 +138,7 @@ Examples:
 				return err
 			}
 			h := in.App.HTTP
-			rc := ccrawl.NewRobotsCache(24*time.Hour, "ccrawl")
+			rc := ccrawl.NewRobotsCache(24*time.Hour, ccrawl.DefaultCrawlConfig.UserAgent)
 			entry := FetchRobotsForHost(ctx, h, rc, u.Hostname(), u.Scheme)
 			if !entry.IsAllowed(u.Path) {
 				return fmt.Errorf("robots.txt disallows %s", rawURL)
@@ -164,12 +164,7 @@ Examples:
 
 // FetchRobotsForHost retrieves (with caching) the robots.txt for a host.
 func FetchRobotsForHost(ctx context.Context, h *ccrawl.HTTPClient, rc *ccrawl.RobotsCache, host, scheme string) *ccrawl.RobotsEntry {
-	if e := rc.Get(host); e != nil {
-		return e
-	}
-	e := ccrawl.FetchRobots(ctx, h, host, scheme)
-	rc.Put(host, e)
-	return e
+	return rc.Fetch(ctx, h, host, scheme)
 }
 
 // ── crawl status ──────────────────────────────────────────────────────────────
diff --git a/docs/content/guides/recrawl-engine.md b/docs/content/guides/recrawl-engine.md
index 812b062..114e0b1 100644
--- a/docs/content/guides/recrawl-engine.md
+++ b/docs/content/guides/recrawl-engine.md
@@ -67,6 +67,10 @@ ccrawl crawl fetch https://example.com/ --robots
 It is off by default because a single manual fetch of a URL you already chose does not need it.
 Turn it on for anything automated.
 
+The parser follows [RFC 9309](https://www.rfc-editor.org/rfc/rfc9309.html): `*` and `$` wildcards, longest match wins with an allow breaking ties, the most specific user agent group taking precedence over `*`, and `Sitemap` lines collected.
+A missing `robots.txt` leaves the host open, and a `robots.txt` that could not be fetched at all, whether the host returned a 5xx or never answered, disallows the whole host until the fetch is retried a few minutes later.
+That last rule is the spec's and it is deliberate: a site whose robots endpoint is down cannot tell you to stop, so you stop.
+
 The digest is the useful part for recrawl work.
 Fetch a URL, compare the digest with the `content_digest` the columnar index has for the same URL, and you know whether the page changed since the crawl without diffing any text.
 
@@ -110,16 +114,15 @@ See the [search index guide](/guides/search-index/) for the details.
 
 ## Planned
 
-The library under `ccrawl/crawl.go` already has the pieces a real crawl loop needs, and they compile and are tested:
+The library already has the pieces a real crawl loop needs, and they compile and are tested:
 
-- `Frontier`, a priority queue with a per-host politeness delay and anti-starvation
-- `RobotsCache`, a per-host cache of parsed rules with a TTL
+- `Frontier`, a SQLite-backed priority queue with a per-host politeness delay, resumable across a restart
+- `RobotsCache`, a per-host cache of RFC 9309 rules with a TTL
 - `WriteWARCResponse`, a WARC/1.1 response record writer
 - a shared connection pool, 200 idle connections and 10 per host
 
 None of it is reachable from a command yet.
 Wiring it into a `ccrawl crawl run` that takes a seed list, walks the frontier, honours robots, and writes WARC is tracked in [#54](https://github.com/tamnd/ccrawl-cli/issues/54).
-The frontier also has to survive a restart before that is useful at scale, which is [#51](https://github.com/tamnd/ccrawl-cli/issues/51), and the robots parser needs to be RFC 9309 correct first, which is [#52](https://github.com/tamnd/ccrawl-cli/issues/52).
 
 Until those land, if you need a bulk pipeline over Common Crawl URLs today, use [`ccrawl markdown refetch`](/guides/markdown-corpus/).
 It takes the URL list from a WARC shard, fetches every page live at high concurrency, and writes Parquet.
diff --git a/scripts/robots-corpus.py b/scripts/robots-corpus.py
new file mode 100644
index 0000000..3c9b23b
--- /dev/null
+++ b/scripts/robots-corpus.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""Build a real robots.txt corpus and the verdicts a reference parser gives for it.
+
+The Go side is TestRobotsAgainstRealSites, which is skipped unless
+CCRAWL_ROBOTS_CORPUS points at the directory this writes. Hand written test
+cases prove the parser matches the RFC as we read it; this proves it matches
+what the live web actually serves and what another implementation of the same
+spec concludes about it.
+
+    python3 -m venv .venv && .venv/bin/pip install protego
+    .venv/bin/python scripts/robots-corpus.py --out /tmp/robots-corpus
+    CCRAWL_ROBOTS_CORPUS=/tmp/robots-corpus go test ./ccrawl -run RealSites -v
+
+protego is the parser Scrapy uses and it follows the same Google derived rules
+RFC 9309 was written from, so a disagreement is worth looking at in both
+directions rather than assumed to be ours.
+"""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import json
+import os
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+
+# Sites picked for variety rather than popularity: news, commerce, code hosting,
+# government, and a few that are known to write long or unusual files.
+SITES = """
+wikipedia.org nytimes.com github.com reddit.com amazon.com apple.com
+microsoft.com cloudflare.com mozilla.org python.org golang.org arxiv.org
+nasa.gov europa.eu gov.uk imdb.com etsy.com wordpress.org theguardian.com
+cnn.com booking.com craigslist.org archive.org quora.com linkedin.com
+pinterest.com ebay.com target.com walmart.com espn.com un.org
+""".split()
+
+USER_AGENT = "CCrawl/2.0 (+https://ccrawl.tamnd.com/bot)"
+TOKEN = "ccrawl"
+
+# Paths every file is probed with, so a file with no rules of its own still
+# contributes something to compare.
+COMMON = [
+    "/",
+    "/index.html",
+    "/search?q=test",
+    "/admin/",
+    "/api/v1/items",
+    "/images/logo.gif",
+    "/page.php?id=1",
+    "/a/b/c/d/e.html",
+]
+
+
+def reliable(path: str) -> bool:
+    """Report whether protego's verdict on a path is worth comparing against.
+
+    protego runs a URL through urlparse and percent encodes the result before
+    matching, and it does not do the same to the pattern, so three kinds of path
+    come back with an answer that contradicts the file's own text:
+
+      - an = in the path segment is escaped in the URL and left alone in the
+        pattern, so Disallow: /gp/aw/help/id=sss does not match /gp/aw/help/id=sss
+        and neither does a %3D spelling of the same character
+      - a trailing ? is an empty query that urlparse drops from the URL and a
+        corner case in protego preserves in the pattern
+      - a leading // is read as a network location, which empties the path
+
+    Every one of these is protego reading a path as if it were a URL. They are
+    filtered here rather than papered over in the Go test so that the thing
+    being skipped, and the reason, sit next to the tool they are about.
+    """
+    if path.endswith("?") or path.startswith("//"):
+        return False
+    return "=" not in urllib.parse.unquote(path.split("?", 1)[0])
+
+
+def fetch(site: str) -> tuple[str, int, str]:
+    req = urllib.request.Request(
+        f"https://{site}/robots.txt", headers={"User-Agent": USER_AGENT}
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=20) as resp:
+            return site, resp.status, resp.read(2_000_000).decode("utf-8", "replace")
+    except urllib.error.HTTPError as err:
+        return site, err.code, ""
+    except Exception:
+        return site, 0, ""
+
+
+def probes(body: str) -> list[str]:
+    """Concrete paths derived from the file's own rules, plus the common set.
+
+    A rule is only interesting if something is tested against it, and the paths
+    a file cares about are the ones it names. Each pattern becomes the path it
+    was written for and a couple of near misses around it.
+    """
+    paths = list(COMMON)
+    for line in body.splitlines():
+        line = line.split("#", 1)[0].strip()
+        match = re.match(r"(?i)^(allow|disallow)\s*:\s*(\S.*)$", line)
+        if not match:
+            continue
+        pattern = match.group(2).strip()
+        base = pattern.rstrip("$").replace("*", "x")
+        if not base.startswith("/"):
+            base = "/" + base.lstrip("/")
+        paths.extend([base, base + "extra", base.rstrip("/") + "/deep/page.html", base + "?q=1"])
+    seen, out = set(), []
+    for p in paths:
+        if p not in seen and len(p) < 300:
+            seen.add(p)
+            out.append(p)
+    return out[:400]
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--out", default="/tmp/robots-corpus")
+    args = ap.parse_args()
+    os.makedirs(args.out, exist_ok=True)
+
+    from protego import Protego
+
+    verdicts: dict[str, dict[str, object]] = {}
+    with concurrent.futures.ThreadPoolExecutor(8) as pool:
+        for site, status, body in pool.map(fetch, SITES):
+            if status != 200 or not body.strip():
+                print(f"skip {site}: HTTP {status}")
+                continue
+            with open(os.path.join(args.out, f"{site}.txt"), "w") as fh:
+                fh.write(body)
+            parser = Protego.parse(body)
+            paths = probes(body)
+            allowed = {p: bool(parser.can_fetch(p, TOKEN)) for p in paths if reliable(p)}
+            delay = parser.crawl_delay(TOKEN)
+            verdicts[site] = {
+                "allowed": allowed,
+                "skipped": [p for p in paths if not reliable(p)],
+                "crawl_delay": float(delay) if delay else 0.0,
+                "sitemaps": sorted(parser.sitemaps),
+            }
+            print(f"{site}: {len(body)} bytes, {len(allowed)} probes")
+
+    path = os.path.join(args.out, "verdicts.json")
+    with open(path, "w") as fh:
+        json.dump({"user_agent": TOKEN, "sites": verdicts}, fh, indent=1, sort_keys=True)
+    total = sum(len(v["allowed"]) for v in verdicts.values())
+    skipped = sum(len(v["skipped"]) for v in verdicts.values())
+    print(f"wrote {path}: {len(verdicts)} sites, {total} probes, {skipped} skipped")
+
+
+if __name__ == "__main__":
+    main()