Skip to content

crawl: rewrite robots.txt to RFC 9309 #97

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tamnd merged 1 commit into from
Aug 10, 2026
Merged

crawl: rewrite robots.txt to RFC 9309 #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 20 additions & 8 deletions ccrawl/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,19 +137,28 @@ func (h *HTTPClient) DataURL(path string) string { return FileURL(path, h.source

// Get fetches url with retries.
func (h *HTTPClient) Get(ctx context.Context, url string) (*http.Response, error) {
return h.doWith(ctx, h.c, url, "")
return h.doWith(ctx, h.c, url, "", false)
}

// getStatus fetches url and hands back whatever the server finally said, where
// Get would report a retryable status as an error. Callers that have to tell a
// 403 from a dead host need the difference: to robots.txt a 403 is "no rules
// here" and a failed connection is "assume everything is off limits", and one
// error value cannot say which happened.
func (h *HTTPClient) getStatus(ctx context.Context, url string) (*http.Response, error) {
return h.doWith(ctx, h.c, url, "", true)
}

// GetRange fetches the [offset, offset+length) byte span of url.
func (h *HTTPClient) GetRange(ctx context.Context, url string, offset, length int64) (*http.Response, error) {
rangeHdr := fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)
return h.doWith(ctx, h.c, url, rangeHdr)
return h.doWith(ctx, h.c, url, rangeHdr, false)
}

// GetDownload fetches url with no client timeout (relies on ctx cancellation),
// for large archive bodies.
func (h *HTTPClient) GetDownload(ctx context.Context, url string) (*http.Response, error) {
return h.doWith(ctx, h.download, url, "")
return h.doWith(ctx, h.download, url, "", false)
}

// ContentLength returns the total size of url. It sends a one-byte range request
Expand Down Expand Up @@ -189,7 +198,10 @@ func (h *HTTPClient) FetchBytes(ctx context.Context, url string) ([]byte, error)
return io.ReadAll(resp.Body)
}

func (h *HTTPClient) doWith(ctx context.Context, client *http.Client, url, rangeHdr string) (*http.Response, error) {
// doWith runs the request with the retry, throttle and backoff policy. When
// keepStatus is set, a retryable status that survives the last attempt is
// returned as a response instead of an error.
func (h *HTTPClient) doWith(ctx context.Context, client *http.Client, url, rangeHdr string, keepStatus bool) (*http.Response, error) {
var last error
var retryAfter time.Duration // server-advised wait carried from the prior attempt
var serverWait bool // whether the prior attempt sent a Retry-After
Expand All @@ -204,12 +216,9 @@ func (h *HTTPClient) doWith(ctx context.Context, client *http.Client, url, range
return nil, err
}
// s3://bucket/key is not something net/http can dial. The bucket is
// public, so the REST endpoint for it is a plain anonymous GET and every
// public, so the REST endpoint for it is a plain anonymous GET, and every
// other line in this loop, the throttle, the retries, the backoff, applies
// to it unchanged.
// s3://bucket/key is not something net/http can dial. It becomes a GET
// against the bucket's REST endpoint, and every other line in this loop,
// the throttle, the retries, the backoff, applies to it unchanged.
fetch, isS3 := s3Endpoint(url, s3BucketRegion)
if !isS3 {
fetch = url
Expand Down Expand Up @@ -250,6 +259,9 @@ func (h *HTTPClient) doWith(ctx context.Context, client *http.Client, url, range
}
}
if retryableStatus(resp.StatusCode) {
if keepStatus && attempt == h.retries {
return resp, nil
}
// A 503/429 commonly carries Retry-After; honor it on the next loop.
retryAfter, serverWait = parseRetryAfter(resp.Header.Get("Retry-After"))
_ = resp.Body.Close()
Expand Down
126 changes: 2 additions & 124 deletions ccrawl/crawl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package ccrawl

import (
"bufio"
"bytes"
"compress/gzip"
"context"
Expand All @@ -12,14 +11,15 @@ import (
"net/http"
"net/url"
"strings"
"sync"
"time"

"github.com/andybalholm/brotli"
)

// The frontier itself lives in frontier.go, on disk. It used to be a heap and
// a map right here, which is fine until the seed set is bigger than memory.
// robots.txt lives in robots.go for the same reason: RFC 9309 is more than the
// dozen lines of prefix matching that used to sit in this file.

// ContentSHA1 returns the hex SHA-1 of raw content bytes (matches CC's digest
// field in CDX records).
Expand All @@ -29,128 +29,6 @@ func ContentSHA1(content []byte) string {
return hex.EncodeToString(h.Sum(nil))
}

// ── robots.txt cache ──────────────────────────────────────────────────────────

// RobotsRule is one allow/disallow rule from robots.txt.
type RobotsRule struct {
Allow bool
Pattern string
}

// RobotsEntry is a cached robots.txt for one host.
type RobotsEntry struct {
Rules []RobotsRule
CrawlDelay time.Duration
ExpiresAt int64 // Unix timestamp
}

// IsAllowed reports whether the given path is allowed. The most specific
// (longest) matching rule wins, following the standard robots.txt precedence.
func (e *RobotsEntry) IsAllowed(path string) bool {
best := -1
bestAllow := true
for _, r := range e.Rules {
if strings.HasPrefix(path, r.Pattern) && len(r.Pattern) > best {
best = len(r.Pattern)
bestAllow = r.Allow
}
}
return bestAllow // default allow when no rule matches (best == -1 → true)
}

// RobotsCache caches parsed robots.txt per host with a TTL.
type RobotsCache struct {
mu sync.RWMutex
entries map[string]*RobotsEntry
ttl time.Duration
ua string // user-agent to match in robots.txt
}

// NewRobotsCache creates a cache with the given TTL and user-agent string.
func NewRobotsCache(ttl time.Duration, userAgent string) *RobotsCache {
return &RobotsCache{
entries: make(map[string]*RobotsEntry),
ttl: ttl,
ua: userAgent,
}
}

// Get returns the cached robots entry for host, or nil if not cached or expired.
func (rc *RobotsCache) Get(host string) *RobotsEntry {
rc.mu.RLock()
defer rc.mu.RUnlock()
e, ok := rc.entries[host]
if !ok || time.Now().Unix() >= e.ExpiresAt {
return nil
}
return e
}

// Put stores a robots entry for host.
func (rc *RobotsCache) Put(host string, e *RobotsEntry) {
e.ExpiresAt = time.Now().Add(rc.ttl).Unix()
rc.mu.Lock()
rc.entries[host] = e
rc.mu.Unlock()
}

// FetchRobots fetches and parses robots.txt for the given host. On timeout or
// 5xx, returns a permissive entry (allow all). On 404, returns allow all
// permanently. The caller should Put the result into the cache.
func FetchRobots(ctx context.Context, h *HTTPClient, host, scheme string) *RobotsEntry {
robotsURL := scheme + "://" + host + "/robots.txt"
resp, err := h.Get(ctx, robotsURL)
if err != nil || resp.StatusCode == 404 {
return &RobotsEntry{} // allow all
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != 200 {
return &RobotsEntry{} // allow all on 4xx/5xx
}
return parseRobots(resp.Body)
}

// parseRobots parses a robots.txt body (simplified: only User-agent:*/all,
// Disallow, Allow, and Crawl-delay).
func parseRobots(r io.Reader) *RobotsEntry {
entry := &RobotsEntry{}
var active bool // true when current User-agent block applies to us
sc := bufio.NewScanner(r)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if strings.HasPrefix(line, "#") || line == "" {
continue
}
key, val, ok := strings.Cut(line, ":")
if !ok {
continue
}
key = strings.TrimSpace(strings.ToLower(key))
val = strings.TrimSpace(val)
switch key {
case "user-agent":
active = val == "*" || strings.EqualFold(val, "ccrawl")
case "disallow":
if active && val != "" {
entry.Rules = append(entry.Rules, RobotsRule{Allow: false, Pattern: val})
}
case "allow":
if active && val != "" {
entry.Rules = append(entry.Rules, RobotsRule{Allow: true, Pattern: val})
}
case "crawl-delay":
if active {
var d float64
_, _ = fmt.Sscan(val, &d)
if d > 0 {
entry.CrawlDelay = time.Duration(d * float64(time.Second))
}
}
}
}
return entry
}

// ── crawl result ──────────────────────────────────────────────────────────────

// CrawlResult is the output of fetching a single URL.
Expand Down
68 changes: 0 additions & 68 deletions ccrawl/crawl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,74 +108,6 @@ func TestContentSHA1(t *testing.T) {
}
}

func TestParseRobots(t *testing.T) {
robotsTxt := `User-agent: *
Disallow: /private/
Allow: /private/public.html
Crawl-delay: 2

User-agent: googlebot
Disallow: /admin/`
entry := parseRobots(strings.NewReader(robotsTxt))
if len(entry.Rules) == 0 {
t.Fatal("expected rules to be parsed")
}
if entry.CrawlDelay != 2*time.Second {
t.Errorf("Crawl-delay = %s, want 2s", entry.CrawlDelay)
}
// /private/ should be disallowed
if entry.IsAllowed("/private/secret.html") {
t.Error("/private/secret.html should be disallowed")
}
// /private/public.html should be allowed (Allow rule)
if !entry.IsAllowed("/private/public.html") {
t.Error("/private/public.html should be allowed")
}
// /public/ should be allowed (not in rules)
if !entry.IsAllowed("/public/page.html") {
t.Error("/public/page.html should be allowed")
}
}

func TestRobotsCache(t *testing.T) {
rc := NewRobotsCache(24*time.Hour, "ccrawl")
if rc.Get("example.com") != nil {
t.Error("empty cache should return nil")
}
e := &RobotsEntry{Rules: []RobotsRule{{Allow: false, Pattern: "/admin/"}}}
rc.Put("example.com", e)
got := rc.Get("example.com")
if got == nil {
t.Fatal("Put then Get should return entry")
}
if len(got.Rules) != 1 {
t.Errorf("expected 1 rule, got %d", len(got.Rules))
}
}

func TestFetchRobots(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/robots.txt" {
w.Header().Set("Content-Type", "text/plain")
_, _ = w.Write([]byte("User-agent: *\nDisallow: /secret/\n"))
}
}))
defer srv.Close()

h := NewHTTPClient(DefaultConfig())
host := strings.TrimPrefix(srv.URL, "http://")
entry := FetchRobots(context.Background(), h, host, "http")
if entry == nil {
t.Fatal("FetchRobots returned nil")
}
if entry.IsAllowed("/secret/page") {
t.Error("/secret/ should be disallowed")
}
if !entry.IsAllowed("/public/page") {
t.Error("/public/ should be allowed")
}
}

func TestCrawlURL(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
Expand Down
16 changes: 16 additions & 0 deletions ccrawl/frontier.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,22 @@ func (f *Frontier) Retry(e FrontierEntry, at int64) error {
return nil
}

// Hold keeps a host idle until at least the given time, on top of whatever the
// frontier's own politeness delay already reserved.
//
// This is where a robots.txt Crawl-delay lands. Pop reserves the configured
// delay when it hands a URL out, because at that point nobody has read the
// host's robots.txt yet; the caller that fetches it learns the host wants ten
// seconds and says so here. Nothing is ever brought forward, so the effective
// delay is the longest anyone asked for.
func (f *Frontier) Hold(host string, until int64) {
f.mu.Lock()
defer f.mu.Unlock()
if at := f.hostNextAt(host); at < until {
f.setHostNextAt(host, until)
}
}

// hostNextAt reads a host's next eligible time, from the LRU when it is hot,
// from the unflushed writes when it is not, and from the database otherwise. A
// crawl touches a small set of hosts hard and a long tail once, which is the
Expand Down
Loading
Loading