diff --git a/ccrawl/extractor.go b/ccrawl/extractor.go new file mode 100644 index 0000000..9834fbb --- /dev/null +++ b/ccrawl/extractor.go @@ -0,0 +1,246 @@ +package ccrawl + +import ( + "bytes" + "fmt" + "io" + "net/url" + "runtime/debug" + "sort" + "strings" + "unicode/utf8" + + "github.com/tamnd/h2m" + "github.com/tamnd/yomi/extract" + "github.com/tamnd/yomi/mdconv" + "golang.org/x/net/html" + "golang.org/x/net/html/charset" +) + +// Extractor is one way of turning a captured page into the Markdown that lands +// in the dataset. Which one you use is a corpus quality decision, not a build +// time constant: the same shard run through two engines is two different +// corpora, and the only way to find out which one suits a downstream task is to +// build both and compare. +// +// Convert returns "" when the page yields nothing worth keeping, which the +// pipelines treat as a skipped record rather than an error. +type Extractor struct { + // Name is the value written to the extractor column, before the version. + Name string + // Summary is the one line shown in help. + Summary string + // Module is the Go module whose version identifies this engine. Empty when + // the engine is not ours to version, which is the WET case. + Module string + // SourceKind is the manifest a run using this extractor has to read, "warc" + // or "wet". They are not interchangeable: a WET file holds no HTML and a + // WARC holds no pre-extracted text. + SourceKind string + + convert func(body []byte, pageURL string) string +} + +// Extractors are the engines --extractor accepts, keyed by name. +var Extractors = map[string]*Extractor{ + "h2m": { + Name: "h2m", + Summary: "go-trafilatura tuned for recall, rendered as GFM (default)", + Module: "github.com/tamnd/h2m", + SourceKind: "warc", + convert: htmlToMarkdownH2M, + }, + "readability": { + Name: "readability", + Summary: "go-readability extraction, the engine open-markdown-v2 shipped", + Module: "github.com/tamnd/yomi", + SourceKind: "warc", + convert: htmlToMarkdownReadability, + }, + "raw": { + Name: "raw", + Summary: "the whole document rendered as Markdown, no boilerplate removal", + Module: "github.com/tamnd/yomi", + SourceKind: "warc", + convert: htmlToMarkdownRaw, + }, + "wet": { + Name: "wet", + Summary: "the plain text Common Crawl already extracted, from the WET files", + SourceKind: "wet", + convert: wetTextPassthrough, + }, +} + +// DefaultExtractor is the engine a run uses when nobody says otherwise. +const DefaultExtractor = "h2m" + +// LookupExtractor resolves an --extractor value. +func LookupExtractor(name string) (*Extractor, error) { + if name == "" { + name = DefaultExtractor + } + e, ok := Extractors[strings.ToLower(strings.TrimSpace(name))] + if !ok { + return nil, fmt.Errorf("unknown extractor %q, want one of %s", name, strings.Join(ExtractorNames(), ", ")) + } + return e, nil +} + +// ExtractorNames lists the known engines in a stable order. +func ExtractorNames() []string { + names := make([]string, 0, len(Extractors)) + for n := range Extractors { + names = append(names, n) + } + sort.Strings(names) + return names +} + +// Convert turns one captured body into Markdown. +func (e *Extractor) Convert(body []byte, pageURL string) string { + if e == nil { + return htmlToMarkdownH2M(body, pageURL) + } + return e.convert(body, pageURL) +} + +// ID is what goes in the extractor column and the dataset card: the engine name +// and the version of the code that produced the text, as name@version. +// +// The version matters more than the name. Extraction changes between releases, +// sometimes visibly, and a dataset that only records "h2m" cannot answer why two +// shards built three months apart disagree about the same page. +// +// The WET engine is Common Crawl's own and has no version we can read, so it is +// stamped with the crawl instead, which is the closest thing to a version the +// text has. +func (e *Extractor) ID(crawlID string) string { + if e == nil { + return DefaultExtractor + "@" + moduleVersion(Extractors[DefaultExtractor].Module) + } + if e.Module == "" { + if crawlID == "" { + return e.Name + } + return e.Name + "@" + crawlID + } + return e.Name + "@" + moduleVersion(e.Module) +} + +// moduleVersion reads a dependency's version out of the build info of the +// running binary, so the stamp describes the build that actually ran rather +// than whatever go.mod said when this file was written. +func moduleVersion(path string) string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "unknown" + } + for _, d := range info.Deps { + if d.Path == path { + if d.Replace != nil && d.Replace.Version != "" { + return d.Replace.Version + } + return d.Version + } + } + return "unknown" +} + +// htmlToMarkdownH2M converts one HTML body to Markdown via h2m: go-trafilatura +// tuned for recall strips boilerplate and isolates the main content, then h2m +// renders GitHub-flavored Markdown with links resolved against pageURL. h2m +// transcodes non-UTF-8 bodies (GBK, Shift-JIS, Latin-1, and so on) from the +// page's declared charset before parsing, so Common Crawl's mixed-encoding +// pages convert correctly. +func htmlToMarkdownH2M(body []byte, pageURL string) string { + if len(body) == 0 { + return "" + } + res := h2m.Convert(body, pageURL) + if !res.HasContent { + return "" + } + return validUTF8(res.Markdown) +} + +// htmlToMarkdownReadability is the engine open-markdown-v2 shipped: yomi's +// go-readability extraction plus mdconv rendering. It is faster than h2m and +// keeps less of the page, which on a news article is usually the same text and +// on a listing page is much less of it. +func htmlToMarkdownReadability(body []byte, pageURL string) string { + utf8Body := transcode(body) + if len(utf8Body) == 0 { + return "" + } + art, err := extract.FromHTML(utf8Body, pageURL) + if err != nil || art.Node == nil { + return "" + } + base, _ := url.Parse(pageURL) + md, err := mdconv.Convert(art.Node, mdconv.Options{Base: base}) + if err != nil { + return "" + } + return validUTF8(strings.TrimSpace(md)) +} + +// htmlToMarkdownRaw renders the whole document and skips extraction entirely, +// so the nav bars, the footer, and the cookie banner all come along. +// +// That sounds useless and is not. Every extractor is a lossy judgement about +// what the page was for, and on the pages where that judgement goes wrong there +// is no way to tell from the output alone. Raw is the control: it says what was +// on the page, so a corpus built with a real extractor can be measured against +// what it threw away. +func htmlToMarkdownRaw(body []byte, pageURL string) string { + utf8Body := transcode(body) + if len(utf8Body) == 0 { + return "" + } + doc, err := html.Parse(bytes.NewReader(utf8Body)) + if err != nil { + return "" + } + base, _ := url.Parse(pageURL) + md, err := mdconv.Convert(doc, mdconv.Options{Base: base}) + if err != nil { + return "" + } + return validUTF8(strings.TrimSpace(md)) +} + +// wetTextPassthrough is the WET engine: Common Crawl already extracted the text, +// so there is nothing to do but keep it. It is not Markdown and does not pretend +// to be, which is the trade for text that costs no CPU and no HTML download. +func wetTextPassthrough(body []byte, _ string) string { + return validUTF8(strings.TrimSpace(string(body))) +} + +// transcode converts a body to UTF-8 using its declared charset. A page that +// declares nothing is sniffed, and one that declares a charset it does not +// honour comes out with replacement characters rather than an error. +func transcode(body []byte) []byte { + if len(body) == 0 { + return nil + } + r, err := charset.NewReader(bytes.NewReader(body), "") + if err != nil { + return nil + } + out, err := io.ReadAll(r) + if err != nil { + return nil + } + return out +} + +// validUTF8 drops invalid byte sequences. A handful of pages declare one charset +// and serve another, so transcoded text can still carry stray bytes, and parquet +// strings and the Arrow readers on top of them require valid UTF-8. +func validUTF8(s string) string { + if utf8.ValidString(s) { + return s + } + return strings.ToValidUTF8(s, "") +} diff --git a/ccrawl/extractor_test.go b/ccrawl/extractor_test.go new file mode 100644 index 0000000..ece8a62 --- /dev/null +++ b/ccrawl/extractor_test.go @@ -0,0 +1,288 @@ +package ccrawl + +import ( + "bytes" + "context" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + kgzip "github.com/klauspost/compress/gzip" + "github.com/parquet-go/parquet-go" +) + +// articleHTML is a page with a real article buried in the usual furniture, so +// the engines have something to disagree about: an extractor that works keeps +// the article, and the raw engine keeps all of it. +const articleHTML = `Quarterly report + +
+

Quarterly report

+

The committee published its quarterly report on Tuesday, noting that growth had slowed across most of the region while inflation remained broadly under control.

+

Analysts expect the central bank to hold rates steady until the end of the year, and several of them said the pause would last well into the next one.

+
+ +` + +func TestLookupExtractor(t *testing.T) { + tests := []struct { + name string + in string + wantName string + wantSource string + wantErr bool + }{ + {name: "default on empty", in: "", wantName: "h2m", wantSource: "warc"}, + {name: "by name", in: "readability", wantName: "readability", wantSource: "warc"}, + {name: "case and space insensitive", in: " RAW ", wantName: "raw", wantSource: "warc"}, + {name: "wet reads wet shards", in: "wet", wantName: "wet", wantSource: "wet"}, + {name: "unknown", in: "trafilatura", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ex, err := LookupExtractor(tt.in) + if tt.wantErr { + if err == nil { + t.Fatalf("LookupExtractor(%q) = %v, want an error", tt.in, ex.Name) + } + // The error has to list the options, since the whole point of + // the flag is that the caller is choosing between them. + for _, n := range ExtractorNames() { + if !strings.Contains(err.Error(), n) { + t.Fatalf("error %q does not mention %q", err, n) + } + } + return + } + if err != nil { + t.Fatalf("LookupExtractor(%q): %v", tt.in, err) + } + if ex.Name != tt.wantName || ex.SourceKind != tt.wantSource { + t.Fatalf("got %s reading %s shards, want %s reading %s", ex.Name, ex.SourceKind, tt.wantName, tt.wantSource) + } + }) + } +} + +// TestExtractorID pins the format the parquet column and the dataset card both +// use. A bare name is not enough: extraction changes between releases, so the +// version is part of the answer. +// +// The version itself reads "unknown" here and only here. A test binary's build +// info carries no dependency list, so this can check the shape but not the +// value; a real binary stamps h2m@v0.2.1. +func TestExtractorID(t *testing.T) { + for _, name := range []string{"h2m", "readability", "raw"} { + id := Extractors[name].ID("CC-MAIN-2026-30") + prefix, version, ok := strings.Cut(id, "@") + if !ok || prefix != name { + t.Fatalf("ID = %q, want %s@version", id, name) + } + if version == "" { + t.Fatalf("ID = %q, want a version after the name", id) + } + } + if v := moduleVersion("example.com/nothing/we/import"); v != "unknown" { + t.Fatalf("moduleVersion of a module we do not depend on = %q, want unknown", v) + } + // The WET text is Common Crawl's own and has no version we can read, so the + // crawl stands in for one. + if id := Extractors["wet"].ID("CC-MAIN-2026-30"); id != "wet@CC-MAIN-2026-30" { + t.Fatalf("wet ID = %q, want wet@CC-MAIN-2026-30", id) + } + if id := Extractors["wet"].ID(""); id != "wet" { + t.Fatalf("wet ID with no crawl = %q, want wet", id) + } +} + +// TestExtractorsAgreeOnPagesDisagreeOnText is the done-when from the issue: the +// same shard through three engines gives the same documents and different text. +// Same doc_id set is what makes the corpora comparable; different markdown is +// what makes the choice worth having. +func TestExtractorsAgreeOnPagesDisagreeOnText(t *testing.T) { + var shard bytes.Buffer + for i := range 4 { + shard.Write(warcMember(t, fmt.Sprintf("https://example.com/article/%d", i), articleHTML)) + } + + dir := t.TempDir() + md := map[string]map[string]string{} // engine -> doc_id -> markdown + for _, name := range []string{"h2m", "readability", "raw"} { + out := filepath.Join(dir, name+".parquet") + stats, err := packStream(context.Background(), + bytes.NewReader(shard.Bytes()), + MarkdownPackConfig{OutPath: out, Workers: 2, CrawlID: "CC-MAIN-2026-30", Extractor: Extractors[name]}, + MarkdownStats{}, time.Now()) + if err != nil { + t.Fatalf("%s: packStream: %v", name, err) + } + if stats.Rows != 4 { + t.Fatalf("%s: Rows = %d, want 4", name, stats.Rows) + } + rows, err := parquet.ReadFile[MarkdownRow](out) + if err != nil { + t.Fatalf("%s: read parquet: %v", name, err) + } + md[name] = map[string]string{} + for _, r := range rows { + md[name][r.DocID] = r.Markdown + if want := Extractors[name].ID("CC-MAIN-2026-30"); r.Extractor != want { + t.Fatalf("%s: row extractor = %q, want %q", name, r.Extractor, want) + } + } + } + + for _, name := range []string{"readability", "raw"} { + if len(md[name]) != len(md["h2m"]) { + t.Fatalf("%s produced %d documents, h2m produced %d", name, len(md[name]), len(md["h2m"])) + } + for id := range md["h2m"] { + if _, ok := md[name][id]; !ok { + t.Fatalf("%s is missing doc_id %s that h2m produced", name, id) + } + } + } + + // The article survives every engine. Only raw keeps the furniture, which is + // the difference the flag exists for. + for _, name := range []string{"h2m", "readability", "raw"} { + for id, text := range md[name] { + if !strings.Contains(text, "quarterly report") { + t.Fatalf("%s dropped the article from %s: %q", name, id, text) + } + hasFooter := strings.Contains(text, "Privacy policy") + if name == "raw" && !hasFooter { + t.Fatalf("raw dropped the footer, which is the one thing it is for: %q", text) + } + if name != "raw" && hasFooter { + t.Fatalf("%s kept the footer: %q", name, text) + } + } + } +} + +// wetShard builds a WET file: WARC conversion records holding plain text. +func wetShard(t *testing.T, pages map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + for uri, text := range pages { + rec := fmt.Sprintf("WARC/1.0\r\nWARC-Type: conversion\r\nWARC-Target-URI: %s\r\nWARC-Date: 2026-07-10T00:00:00Z\r\nWARC-Record-ID: \r\nContent-Type: text/plain\r\nContent-Length: %d\r\n\r\n%s\r\n\r\n", + uri, MarkdownDocID(uri)[:16], len(text), text) + zw := kgzip.NewWriter(&buf) + if _, err := zw.Write([]byte(rec)); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + } + return buf.Bytes() +} + +// TestPackStreamWETSource covers the other half of --source-kind: a WET shard +// holds no HTML at all, so the pipeline has to read conversion records and pass +// the text through instead of extracting anything. +func TestPackStreamWETSource(t *testing.T) { + pages := map[string]string{ + "https://example.com/vi": vieText, + "https://example.com/en": engText, + } + out := filepath.Join(t.TempDir(), "wet.parquet") + stats, err := packStream(context.Background(), + bytes.NewReader(wetShard(t, pages)), + MarkdownPackConfig{OutPath: out, Workers: 2, CrawlID: "CC-MAIN-2026-30", Extractor: Extractors["wet"]}, + MarkdownStats{}, time.Now()) + if err != nil { + t.Fatalf("packStream: %v", err) + } + if stats.Rows != 2 { + t.Fatalf("Rows = %d, want 2", stats.Rows) + } + rows, err := parquet.ReadFile[MarkdownRow](out) + if err != nil { + t.Fatalf("read parquet: %v", err) + } + for _, r := range rows { + want, ok := pages[r.URL] + if !ok { + t.Fatalf("unexpected url %q", r.URL) + } + if r.Markdown != want { + t.Fatalf("%s: text was changed on the way through:\n got %q\nwant %q", r.URL, r.Markdown, want) + } + if r.Extractor != "wet@CC-MAIN-2026-30" { + t.Fatalf("%s: extractor = %q", r.URL, r.Extractor) + } + // The language filter reads the same column whatever produced it, which + // is the reason WET is worth having as a source at all. + if r.URL == "https://example.com/vi" && r.Language != "vie" { + t.Fatalf("wet text was not identified: %q at %.2f", r.Language, r.LangConfidence) + } + } +} + +// TestWETSourceRespectsLangFilter checks the filter still applies when the text +// came from Common Crawl rather than from our own extractor. +func TestWETSourceRespectsLangFilter(t *testing.T) { + out := filepath.Join(t.TempDir(), "wet.parquet") + stats, err := packStream(context.Background(), + bytes.NewReader(wetShard(t, map[string]string{ + "https://example.com/vi": vieText, + "https://example.com/en": engText, + })), + MarkdownPackConfig{OutPath: out, Workers: 2, CrawlID: "CC-MAIN-2026-30", Extractor: Extractors["wet"], Lang: "vie"}, + MarkdownStats{}, time.Now()) + if err != nil { + t.Fatalf("packStream: %v", err) + } + if stats.Rows != 1 || stats.LangDropped != 1 { + t.Fatalf("Rows = %d, LangDropped = %d, want 1 and 1", stats.Rows, stats.LangDropped) + } +} + +// TestREADMENamesTheEngine covers the second done-when: the dataset card has to +// say which engine produced the data, and it has to say the version too. +func TestREADMENamesTheEngine(t *testing.T) { + for _, name := range []string{"h2m", "readability", "raw", "wet"} { + id := Extractors[name].ID("CC-MAIN-2026-30") + card := GenerateMarkdownREADME(MarkdownDatasetStats{ + CrawlID: "CC-MAIN-2026-30", + CommittedShards: 1, + TotalShards: 1, + Rows: 100, + Extractor: id, + }) + if !strings.Contains(card, id) { + t.Fatalf("card for %s never names %q", name, id) + } + if !strings.Contains(card, "`extractor`") { + t.Fatalf("card for %s does not document the extractor column", name) + } + } + + // A WET dataset came from a different manifest and skipped extraction, so + // the steps that describe the input have to say so. Telling a reader to + // download .warc.gz for a dataset built from WET files is wrong in the one + // place they go to find out where the text came from. + wetCard := GenerateMarkdownREADME(MarkdownDatasetStats{ + CrawlID: "CC-MAIN-2026-30", CommittedShards: 1, TotalShards: 1, Rows: 100, + Extractor: Extractors["wet"].ID("CC-MAIN-2026-30"), + }) + if !strings.Contains(wetCard, ".warc.wet.gz") || strings.Contains(wetCard, "**Download** raw .warc.gz") { + t.Fatal("the WET card still describes downloading WARC files") + } + if strings.Contains(wetCard, "HTTP 200 responses") { + t.Fatal("the WET card still describes the response filter, which WET records do not have") + } + + // And a filtered dataset says so, because "this is the Vietnamese subset" is + // not something a reader should have to infer from the rows. + card := GenerateMarkdownREADME(MarkdownDatasetStats{ + CrawlID: "CC-MAIN-2026-30", CommittedShards: 1, TotalShards: 1, Rows: 100, Lang: "vie", + }) + if !strings.Contains(card, "filtered to `vie`") { + t.Fatal("card does not mention the language filter") + } +} diff --git a/ccrawl/markdown.go b/ccrawl/markdown.go index 10e4eea..8c7ff08 100644 --- a/ccrawl/markdown.go +++ b/ccrawl/markdown.go @@ -14,22 +14,24 @@ import ( "strings" "sync" "time" - "unicode/utf8" "github.com/parquet-go/parquet-go" "github.com/parquet-go/parquet-go/compress/zstd" - "github.com/tamnd/h2m" ) // MarkdownRow is the parquet schema for one converted HTML document. It matches // the open-index/open-markdown dataset layout so the output can be appended to // existing crawls without a schema migration. // -// language and language_confidence make this open-markdown-v3. They are appended -// at the end and every earlier column keeps its name and type, so a v2 reader -// projecting the columns it knows about reads a v3 file without changes: parquet -// is read by name, and a reader that never asks for the two new columns never -// touches them. +// language, language_confidence, and extractor make this open-markdown-v3. They +// are appended at the end and every earlier column keeps its name and type, so a +// v2 reader projecting the columns it knows about reads a v3 file without +// changes: parquet is read by name, and a reader that never asks for the new +// columns never touches them. +// +// extractor is name@version, not just the name. Extraction changes between +// releases, sometimes visibly, and a dataset that only recorded the name could +// not answer why two shards built months apart disagree about the same page. type MarkdownRow struct { DocID string `parquet:"doc_id"` URL string `parquet:"url"` @@ -41,6 +43,7 @@ type MarkdownRow struct { Markdown string `parquet:"markdown"` Language string `parquet:"language"` LangConfidence float64 `parquet:"language_confidence"` + Extractor string `parquet:"extractor"` } // MarkdownDocID returns a stable 16-byte hex document ID derived from the URL. @@ -122,6 +125,9 @@ type MarkdownPackConfig struct { // MinLangConfidence is the floor Lang applies. 0 selects // DefaultMinLangConfidence. MinLangConfidence float64 + // Extractor is the engine that turns a captured page into Markdown. nil + // selects the default, which keeps every existing caller on h2m. + Extractor *Extractor // Progress is called after each row is written. It may be nil. Progress func(MarkdownStats) } @@ -180,48 +186,21 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta if minConf <= 0 { minConf = DefaultMinLangConfidence } + ex := cfg.Extractor + if ex == nil { + ex = Extractors[DefaultExtractor] + } + exID := ex.ID(cfg.CrawlID) records := make(chan htmlRecord, workers*4) rows := make(chan candidateRow, workers*4) langCounts := map[string]int64{} - // Reader: iterate the WARC multi-member gzip stream, push HTML records. + // Reader: iterate the source stream and push one record per page. var readErr error go func() { defer close(records) - queued := 0 - errStop := errors.New("record limit reached") - readErr = IterateWARC(body, func(rec WARCRecord) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if rec.Header.Type != "response" || rec.Header.HTTPStatus != 200 { - return nil - } - if !isHTMLMIME(rec.Header.HTTPMIME) { - return nil - } - body := HTTPBody(rec.Block) - if len(body) == 0 { - return nil - } - records <- htmlRecord{ - url: rec.Header.TargetURI, - date: rec.Header.Date.Format("2006-01-02"), - recordID: rec.Header.RecordID, - html: body, - } - queued++ - if cfg.MaxRecords > 0 && queued >= cfg.MaxRecords { - return errStop - } - return nil - }) - if readErr == errStop { - readErr = nil // a deliberate early stop is not a read failure - } + readErr = streamSourceRecords(ctx, body, cfg, ex, records) }() tConvert := time.Now() @@ -233,7 +212,7 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta go func() { defer wg.Done() for r := range records { - md := convertGated(cfg.ConvertSem, r.html, r.url) + md := convertGated(cfg.ConvertSem, ex, r.html, r.url) if md == "" { continue } @@ -254,6 +233,7 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta Markdown: md, Language: code, LangConfidence: conf, + Extractor: exID, }, } } @@ -321,41 +301,73 @@ func newMarkdownParquetWriter(path string) (*ParquetWriter[MarkdownRow], error) return &ParquetWriter[MarkdownRow]{f: f, w: w}, nil } -// htmlToMarkdown converts one HTML body to Markdown via h2m: go-trafilatura -// tuned for recall strips boilerplate and isolates the main content, then h2m -// renders GitHub-flavored Markdown with links resolved against pageURL. h2m -// transcodes non-UTF-8 bodies (GBK, Shift-JIS, Latin-1, and so on) from the -// page's declared charset before parsing, so Common Crawl's mixed-encoding -// pages convert correctly. -// -// Returns "" when the body yields no extractable article or conversion fails. -func htmlToMarkdown(body []byte, pageURL string) string { - if len(body) == 0 { - return "" +// streamSourceRecords reads the shard and pushes one record per page. A WARC +// shard carries HTML in response records; a WET shard carries text Common Crawl +// already extracted, in conversion records. Both come out as the same struct, +// so everything downstream of here is identical for the two sources and only +// the extractor knows the difference. +func streamSourceRecords(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, ex *Extractor, records chan<- htmlRecord) error { + queued := 0 + errStop := errors.New("record limit reached") + + push := func(rec htmlRecord) error { + select { + case <-ctx.Done(): + return ctx.Err() + case records <- rec: + } + queued++ + if cfg.MaxRecords > 0 && queued >= cfg.MaxRecords { + return errStop + } + return nil } - res := h2m.Convert(body, pageURL) - if !res.HasContent { - return "" + + var err error + if ex.SourceKind == "wet" { + err = IterateWET(body, cfg.CrawlID, func(rec WETRecord) error { + return push(htmlRecord{ + url: rec.URL, + date: rec.Date.Format("2006-01-02"), + recordID: rec.RecordID, + html: []byte(rec.Text), + }) + }) + } else { + err = IterateWARC(body, func(rec WARCRecord) error { + if rec.Header.Type != "response" || rec.Header.HTTPStatus != 200 { + return nil + } + if !isHTMLMIME(rec.Header.HTTPMIME) { + return nil + } + html := HTTPBody(rec.Block) + if len(html) == 0 { + return nil + } + return push(htmlRecord{ + url: rec.Header.TargetURI, + date: rec.Header.Date.Format("2006-01-02"), + recordID: rec.Header.RecordID, + html: html, + }) + }) } - // A handful of pages declare one charset but serve another, so the - // transcoded Markdown can still carry stray invalid byte sequences. Parquet - // strings and the Arrow readers on top of them require valid UTF-8, so drop - // any invalid bytes before the value reaches the writer. - if !utf8.ValidString(res.Markdown) { - return strings.ToValidUTF8(res.Markdown, "") + if errors.Is(err, errStop) { + return nil // a deliberate early stop is not a read failure } - return res.Markdown + return err } -// convertGated runs htmlToMarkdown while holding a slot in sem, so the total +// convertGated runs the extractor while holding a slot in sem, so the total // number of concurrent conversions across all in-flight shards stays bounded. -// A nil sem runs the conversion ungated. -func convertGated(sem chan struct{}, body []byte, pageURL string) string { +// A nil sem runs the conversion ungated, and a nil extractor runs the default. +func convertGated(sem chan struct{}, ex *Extractor, body []byte, pageURL string) string { if sem != nil { sem <- struct{}{} defer func() { <-sem }() } - return htmlToMarkdown(body, pageURL) + return ex.Convert(body, pageURL) } // isHTMLMIME reports whether a MIME type string names an HTML document. diff --git a/ccrawl/markdown_bench_test.go b/ccrawl/markdown_bench_test.go index 427bbb7..e44ecc3 100644 --- a/ccrawl/markdown_bench_test.go +++ b/ccrawl/markdown_bench_test.go @@ -4,45 +4,10 @@ import ( "io" "os" "strconv" - "strings" "testing" "time" - - "net/url" - - "github.com/tamnd/yomi/extract" - "github.com/tamnd/yomi/mdconv" - "golang.org/x/net/html/charset" ) -// htmlToMarkdownReadability is the engine open-markdown-v2 originally shipped: -// yomi's go-readability extraction plus mdconv rendering. It is kept here only -// so TestEngineComparison can measure it head to head against the h2m -// (trafilatura, FavorRecall) engine that htmlToMarkdown now uses. -func htmlToMarkdownReadability(body []byte, pageURL string) string { - if len(body) == 0 { - return "" - } - r, err := charset.NewReader(strings.NewReader(string(body)), "") - if err != nil { - return "" - } - utf8Body, err := io.ReadAll(r) - if err != nil || len(utf8Body) == 0 { - return "" - } - art, err := extract.FromHTML(utf8Body, pageURL) - if err != nil || art.Node == nil { - return "" - } - base, _ := url.Parse(pageURL) - md, err := mdconv.Convert(art.Node, mdconv.Options{Base: base}) - if err != nil { - return "" - } - return strings.TrimSpace(md) -} - type engineRun struct { name string rows int @@ -121,8 +86,9 @@ func TestEngineComparison(t *testing.T) { name string conv func([]byte, string) string }{ - {"h2m (trafilatura, FavorRecall)", htmlToMarkdown}, - {"v2 (yomi readability + mdconv)", htmlToMarkdownReadability}, + {"h2m (trafilatura, FavorRecall)", htmlToMarkdownH2M}, + {"readability (yomi + mdconv)", htmlToMarkdownReadability}, + {"raw (whole document, no extraction)", htmlToMarkdownRaw}, } var runs []engineRun @@ -169,7 +135,7 @@ func TestEngineComparison(t *testing.T) { if dumped >= n { break } - a := htmlToMarkdown(p.html, p.url) + a := htmlToMarkdownH2M(p.html, p.url) b := htmlToMarkdownReadability(p.html, p.url) if a == "" || b == "" { continue diff --git a/ccrawl/markdown_readme.go b/ccrawl/markdown_readme.go index b882ded..62b3c43 100644 --- a/ccrawl/markdown_readme.go +++ b/ccrawl/markdown_readme.go @@ -25,6 +25,15 @@ type MarkdownDatasetStats struct { ConvertS int64 ExportS int64 PublishS int64 + + // Extractor is the engine that produced the text, as name@version. A dataset + // that does not say which extractor built it cannot be reproduced or + // compared against another one, so the card says it in the processing steps + // and the schema carries it per row. Empty falls back to the default engine's + // description. + Extractor string + // Lang, when set, is the ISO 639-3 language the run filtered to. + Lang string } // GenerateMarkdownREADME produces the HuggingFace dataset card for @@ -65,6 +74,28 @@ func GenerateMarkdownREADME(s MarkdownDatasetStats) string { approx, fmtBytes(html), approx, fmtBytes(md), pct) } + extractorID := s.Extractor + if extractorID == "" { + extractorID = Extractors[DefaultExtractor].ID(s.CrawlID) + } + convertStep := extractorCardStep(extractorID) + // The WET source reads a different manifest and skips extraction, so the + // steps describing the input have to change with it. A card that says + // "download .warc.gz" over a dataset built from WET files is wrong in the one + // place a reader goes to find out where the text came from. + downloadStep := "raw .warc.gz files from Common Crawl S3 (each file is roughly 1 GB compressed)" + filterStep := "to keep only HTTP 200 responses with a `text/html` content type, discarding images, scripts, redirects, and error pages" + streamLine := "The pipeline streams from the compressed WARC through conversion directly into Parquet. Pages that produce empty conversions are dropped." + if name, _, _ := strings.Cut(extractorID, "@"); name == "wet" { + downloadStep = "raw .warc.wet.gz files from Common Crawl S3, which hold the text Common Crawl extracted and are a fraction of the size of the WARC files they came from" + filterStep = "to keep only conversion records, which is every page Common Crawl produced text for" + streamLine = "The pipeline streams from the compressed WET file straight into Parquet. Pages that came with no text are dropped." + } + langStep := "" + if s.Lang != "" { + langStep = fmt.Sprintf(". This dataset was filtered to `%s`, so documents identified as anything else were dropped", s.Lang) + } + var b strings.Builder // Frontmatter. The default config reads every crawl; a per-crawl config @@ -193,7 +224,10 @@ The following is an example row from the dataset: "warc_record_id": "", "html_length": 48210, "markdown_length": 3847, - "markdown": "# Interesting Topic\n\nThis is the main content of the page..." + "markdown": "# Interesting Topic\n\nThis is the main content of the page...", + "language": "eng", + "language_confidence": 1.0, + "extractor": "%s" } `+"```"+` @@ -209,6 +243,9 @@ The following is an example row from the dataset: | `+"`html_length`"+` | int64 | Byte length of the original HTML body before conversion | | `+"`markdown_length`"+` | int64 | Byte length of the converted Markdown body | | `+"`markdown`"+` | string | Clean Markdown content extracted from the page | +| `+"`language`"+` | string | ISO 639-3 language detected in the Markdown, empty when there was too little text to identify | +| `+"`language_confidence`"+` | double | How sure the identifier is, 0 to 1 | +| `+"`extractor`"+` | string | Engine that produced the Markdown, as `+"`name@version`"+`. Extraction changes between releases, so the version is part of the answer | ### Data Splits @@ -228,21 +265,31 @@ The source data consists of web pages crawled by the [Common Crawl](https://comm The processing pipeline runs as a single streaming pass with no intermediate files: -1. **Download** raw .warc.gz files from Common Crawl S3 (each file is roughly 1 GB compressed) -2. **Filter** to keep only HTTP 200 responses with a `+"`text/html`"+` content type, discarding images, scripts, redirects, and error pages -3. **Convert** HTML to clean Markdown. [go-trafilatura](https://github.com/markusmobius/go-trafilatura) tuned for recall (`+"`FavorRecall`"+`, with Readability and DomDistiller fallbacks) isolates the main content node and strips navigation, ads, and boilerplate, then a direct node-tree walk renders GitHub-flavored Markdown with links resolved to absolute URLs -4. **Export** straight to Apache Parquet with Zstd compression +1. **Download** %s +2. **Filter** %s +3. **Convert** %s +4. **Identify** the language of the converted text and record it with the identifier's confidence%s +5. **Export** straight to Apache Parquet with Zstd compression -The pipeline streams from the compressed WARC through conversion directly into Parquet. Pages that produce empty conversions are dropped. +%s + +Every row records the engine that produced it as `+"`%s`"+`, so two shards built with different engines are never mistaken for one corpus. `, s.CrawlID, docsStr, fmtInt(int64(shards)), reduction, // intro line - s.CrawlID, // file tree - s.CrawlID, // datasets name= - s.CrawlID, // datasets data_files= - s.CrawlID, // huggingface_hub allow_patterns= - s.CrawlID, // duckdb path - s.CrawlID, // data splits example + s.CrawlID, // file tree + s.CrawlID, // datasets name= + s.CrawlID, // datasets data_files= + s.CrawlID, // huggingface_hub allow_patterns= + s.CrawlID, // duckdb path + extractorID, // example row extractor + s.CrawlID, // data splits example + downloadStep, // processing step 1 + filterStep, // processing step 2 + convertStep, // processing step 3 + langStep, // processing step 4 + streamLine, // what the pipeline streams through + extractorID, // engine stamped on every row ) // Compression Ratios table. Rows are rendered only when we have data for @@ -433,3 +480,20 @@ func fmtInt(n int64) string { parts = append([]string{s}, parts...) return strings.Join(parts, ",") } + +// extractorCardStep describes one engine for the dataset card's processing +// steps. The name is taken from the recorded name@version so the prose and the +// column can never drift apart. +func extractorCardStep(id string) string { + name, _, _ := strings.Cut(id, "@") + switch name { + case "readability": + return "HTML to Markdown with [go-readability](https://github.com/go-shiori/go-readability), a port of Mozilla's Readability.js, which isolates the article node and drops the rest, then renders Markdown with links resolved to absolute URLs" + case "raw": + return "the whole HTML document to Markdown with no extraction step at all, so navigation, footers, and boilerplate are all still there. This is the control corpus: it says what was on the page before any extractor made a judgement about it" + case "wet": + return "nothing. The text is what Common Crawl's own extractor produced in the WET files, kept as it was given, so this dataset costs no extraction and inherits whatever that extractor decided" + default: + return "HTML to clean Markdown. [go-trafilatura](https://github.com/markusmobius/go-trafilatura) tuned for recall (`FavorRecall`, with Readability and DomDistiller fallbacks) isolates the main content node and strips navigation, ads, and boilerplate, then a direct node-tree walk renders GitHub-flavored Markdown with links resolved to absolute URLs" + } +} diff --git a/ccrawl/markdown_run.go b/ccrawl/markdown_run.go index f343e17..4c4d79f 100644 --- a/ccrawl/markdown_run.go +++ b/ccrawl/markdown_run.go @@ -116,6 +116,10 @@ type MarkdownExportConfig struct { Lang string MinLangConfidence float64 + // Extractor is the engine every shard in the run uses. nil selects the + // default. + Extractor *Extractor + // Progress is called once per committed batch with a snapshot of the run. // It may be nil. Progress func(MarkdownRunStats) @@ -279,6 +283,7 @@ func RunMarkdownExport(ctx context.Context, h *HTTPClient, hf *HFClient, cfg Mar Lang: cfg.Lang, MinLangConfidence: cfg.MinLangConfidence, + Extractor: cfg.Extractor, }) inflight.Add(-1) // Per-shard wall-clock is the useful convert figure for a parallel @@ -426,6 +431,8 @@ func runCommitter(ctx context.Context, hf *HFClient, cfg MarkdownExportConfig, k ParquetBytes: run.ParquetBytes, ConvertS: run.ConvertS, PublishS: run.PublishS, + Extractor: cfg.Extractor.ID(cfg.CrawlID), + Lang: cfg.Lang, } tmp, err := writeTempREADME(dstats) if err != nil { diff --git a/ccrawl/markdown_test.go b/ccrawl/markdown_test.go index 4e3d480..0f338a9 100644 --- a/ccrawl/markdown_test.go +++ b/ccrawl/markdown_test.go @@ -60,7 +60,7 @@ func TestHTMLToMarkdown(t *testing.T) {
copyright boilerplate
`) - md := htmlToMarkdown(body, "https://example.com/page") + md := htmlToMarkdownH2M(body, "https://example.com/page") if md == "" { t.Fatal("htmlToMarkdown returned empty for a valid article") } @@ -76,10 +76,10 @@ func TestHTMLToMarkdown(t *testing.T) { } func TestHTMLToMarkdownEdgeCases(t *testing.T) { - if got := htmlToMarkdown(nil, "https://example.com/"); got != "" { + if got := htmlToMarkdownH2M(nil, "https://example.com/"); got != "" { t.Errorf("nil body: expected empty, got %q", got) } - if got := htmlToMarkdown([]byte{}, "https://example.com/"); got != "" { + if got := htmlToMarkdownH2M([]byte{}, "https://example.com/"); got != "" { t.Errorf("empty body: expected empty, got %q", got) } } @@ -91,7 +91,7 @@ func TestHTMLToMarkdownLatin1(t *testing.T) { "

A paragraph about a caf\xe9 with enough words to read as an article body by readability and survive extraction into the markdown column cleanly.

" + "

Second paragraph to make the article substantial enough for the extractor heuristics to treat it as real content worth keeping.

" + "") - md := htmlToMarkdown(body, "https://example.com/cafe") + md := htmlToMarkdownH2M(body, "https://example.com/cafe") if !strings.Contains(md, "Café") { t.Errorf("latin-1 0xE9 byte not transcoded to UTF-8 é in:\n%s", md) } diff --git a/ccrawl/refetch.go b/ccrawl/refetch.go index c2b3f87..b8a8c04 100644 --- a/ccrawl/refetch.go +++ b/ccrawl/refetch.go @@ -49,6 +49,7 @@ type RefetchRow struct { Language string `parquet:"language"` LangConfidence float64 `parquet:"language_confidence"` + Extractor string `parquet:"extractor"` } // RefetchStats summarises one shard's refetch run with per-phase breakdown. @@ -130,6 +131,11 @@ type RefetchPackConfig struct { // DefaultMinLangConfidence. MinLangConfidence float64 + // Extractor is the engine that turns a fetched page into Markdown. nil + // selects the default. The wet engine is not usable here: a live fetch + // returns HTML, and there is no Common Crawl text to pass through. + Extractor *Extractor + // CacheDir, when set, is where the downloaded WARC is cached so a re-run of // the same shard skips the multi-second download. The download streams to a // .part file beside the final name and is renamed into place only once it is @@ -230,6 +236,11 @@ func PackRefetchShard(ctx context.Context, h *HTTPClient, cfg RefetchPackConfig) if minConf <= 0 { minConf = DefaultMinLangConfidence } + ex := cfg.Extractor + if ex == nil { + ex = Extractors[DefaultExtractor] + } + exID := ex.ID(cfg.CrawlID) var wg sync.WaitGroup wg.Add(convertWorkers) @@ -256,6 +267,7 @@ func PackRefetchShard(ctx context.Context, h *HTTPClient, cfg RefetchPackConfig) TTFBMS: res.TTFB.Milliseconds(), BodyLength: int64(len(res.Body)), Digest: res.Digest, + Extractor: exID, } if res.Header != nil { row.ContentType = res.Header.Get("Content-Type") @@ -273,7 +285,7 @@ func PackRefetchShard(ctx context.Context, h *HTTPClient, cfg RefetchPackConfig) // the CPU-bound conversion, so fetch is not throttled by it. row.HTML = string(res.Body) } else { - md := convertGated(cfg.ConvertSem, res.Body, res.URL) + md := convertGated(cfg.ConvertSem, ex, res.Body, res.URL) row.MarkdownLength = int64(len(md)) row.Markdown = md row.Language, row.LangConfidence = DetectLanguage(md) diff --git a/ccrawl/refetch_run.go b/ccrawl/refetch_run.go index 7c0f6ec..3516c60 100644 --- a/ccrawl/refetch_run.go +++ b/ccrawl/refetch_run.go @@ -51,6 +51,10 @@ type RefetchExportConfig struct { // still records the detected language on every converted row. Lang string MinLangConfidence float64 + + // Extractor is the engine every shard in the run uses. nil selects the + // default. + Extractor *Extractor // Ledger, when set, skips already-committed shards and records new ones. Ledger *Ledger @@ -212,6 +216,7 @@ func RunRefetchExport(ctx context.Context, h *HTTPClient, hf *HFClient, cfg Refe Lang: cfg.Lang, MinLangConfidence: cfg.MinLangConfidence, + Extractor: cfg.Extractor, }) inflight.Add(-1) if err != nil { diff --git a/cli/markdown_cmd.go b/cli/markdown_cmd.go index c86aea8..75d836b 100644 --- a/cli/markdown_cmd.go +++ b/cli/markdown_cmd.go @@ -38,6 +38,9 @@ type markdownExportCmd struct { lang string minConf float64 + + extractor string + sourceKind string } func newMarkdownExportCmd() kit.Command { @@ -45,10 +48,9 @@ func newMarkdownExportCmd() kit.Command { return kit.Command{ Use: "export", Short: "Stream CC WARCs → Markdown → Parquet → HuggingFace", - Long: `Download one or more Common Crawl WARC files, extract HTML bodies, convert to -Markdown with h2m (go-trafilatura tuned for recall plus a GFM renderer: cleaned -prose, tables, absolute links), and write each shard to a zstd-compressed Parquet -file. After each shard the Parquet is committed to a HuggingFace dataset repo. + Long: `Download one or more Common Crawl shards, turn each captured page into +Markdown, and write each shard to a zstd-compressed Parquet file. After each +shard the Parquet is committed to a HuggingFace dataset repo. Output schema (open-markdown-v3): doc_id stable SHA-256 URL hash (16 bytes hex) @@ -61,10 +63,27 @@ Output schema (open-markdown-v3): markdown converted Markdown text language ISO 639-3 code detected in the Markdown, "" if too short language_confidence 0 to 1, how sure the identifier is + extractor engine that produced the text, as name@version -v3 appends the last two columns and changes nothing else, so a v2 reader that +v3 appends the last three columns and changes nothing else, so a v2 reader that projects the columns it knows about reads a v3 file unchanged. +Extractors (--extractor): + h2m go-trafilatura tuned for recall, rendered as GFM (default) + readability go-readability extraction, the engine open-markdown-v2 shipped + raw the whole document as Markdown, no boilerplate removal + wet the plain text Common Crawl already extracted, needs --source-kind wet + +Which extractor you use is a corpus quality decision, so it is a flag and not a +build time constant. The same shard run through two engines is two different +corpora, and the version is recorded next to the name because extraction changes +between releases. + +--source-kind picks the manifest: warc reads warc.paths.gz and extracts the HTML +yourself, wet reads wet.paths.gz and takes the text Common Crawl already +extracted. They are not interchangeable, so wet pairs with --extractor wet and +nothing else. + --lang keeps only the documents identified as that language, which is a document level check on the text actually extracted, not the CLD2 label Common Crawl computed over the raw HTML. It is a coarse pre-filter: a trigram identifier that @@ -87,6 +106,8 @@ Examples: ccrawl markdown export --shards all --parallel 4 --commit-batch 10 ccrawl markdown export --shards 0-99 --push=false --out ~/data/md ccrawl markdown export --shards 0-9 --lang vie --min-lang-confidence 0.8 + ccrawl markdown export --shards 0 --extractor readability --push=false + ccrawl markdown export --shards 0 --source-kind wet --extractor wet --push=false HF_TOKEN=hf_... ccrawl markdown export --shards 0 -c 2026-25`, Flags: v.flags, Run: v.run, @@ -108,6 +129,8 @@ func (v *markdownExportCmd) flags(f *kit.FlagSet) { f.StringVar(&v.ledger, "ledger", "", "resume ledger file (default: /.committed)") f.StringVar(&v.lang, "lang", "", "keep only documents detected as this ISO 639-3 language (e.g. vie)") f.Float64Var(&v.minConf, "min-lang-confidence", ccrawl.DefaultMinLangConfidence, "confidence a document must clear for --lang") + f.StringVar(&v.extractor, "extractor", ccrawl.DefaultExtractor, "conversion engine: "+strings.Join(ccrawl.ExtractorNames(), "|")) + f.StringVar(&v.sourceKind, "source-kind", "", "shard manifest to read: warc|wet (default: whatever the extractor needs)") } func (v *markdownExportCmd) run(ctx context.Context, _ []string) error { @@ -126,13 +149,28 @@ func (v *markdownExportCmd) run(ctx context.Context, _ []string) error { return err } - // Resolve the WARC manifest for this crawl (cached after first fetch). - fmt.Fprintf(os.Stderr, "markdown: fetching WARC manifest for %s …\n", crawlID) - paths, err := ccrawl.FetchPaths(ctx, app.HTTP, app.Cache, crawlID, "warc") + ex, err := ccrawl.LookupExtractor(v.extractor) + if err != nil { + return usageErr(err.Error()) + } + // The extractor decides which manifest makes sense, so --source-kind only + // has to be given when you want to be explicit, and disagreeing with the + // extractor is an error rather than a silent reinterpretation. + kind := v.sourceKind + if kind == "" { + kind = ex.SourceKind + } + if kind != ex.SourceKind { + return usageErr(fmt.Sprintf("extractor %s reads %s shards, so it cannot be used with --source-kind %s", ex.Name, ex.SourceKind, kind)) + } + + // Resolve the shard manifest for this crawl (cached after first fetch). + fmt.Fprintf(os.Stderr, "markdown: fetching %s manifest for %s …\n", strings.ToUpper(kind), crawlID) + paths, err := ccrawl.FetchPaths(ctx, app.HTTP, app.Cache, crawlID, kind) if err != nil { - return fmt.Errorf("fetch WARC manifest: %w", err) + return fmt.Errorf("fetch %s manifest: %w", kind, err) } - fmt.Fprintf(os.Stderr, "markdown: manifest has %d WARC files\n", len(paths)) + fmt.Fprintf(os.Stderr, "markdown: manifest has %d %s files, extracting with %s\n", len(paths), strings.ToUpper(kind), ex.ID(crawlID)) indices, err := parseShardRange(v.shards, len(paths)) if err != nil { @@ -145,7 +183,7 @@ func (v *markdownExportCmd) run(ctx context.Context, _ []string) error { hf := ccrawl.NewHFClient("") if v.push { if !hf.Valid() { - return fmt.Errorf("HF_TOKEN not set — set it or pass --push=false") + return fmt.Errorf("HF_TOKEN not set, set it or pass --push=false") } if err := hf.CreateDatasetRepo(ctx, v.repo, false); err != nil { return fmt.Errorf("create HF repo: %w", err) @@ -193,6 +231,7 @@ func (v *markdownExportCmd) run(ctx context.Context, _ []string) error { Lang: v.lang, MinLangConfidence: v.minConf, + Extractor: ex, }) rep.Textf( diff --git a/cli/markdown_refetch_cmd.go b/cli/markdown_refetch_cmd.go index 40ee511..2de3a15 100644 --- a/cli/markdown_refetch_cmd.go +++ b/cli/markdown_refetch_cmd.go @@ -46,6 +46,7 @@ type markdownRefetchCmd struct { fetchOnly bool // store raw HTML, skip the convert phase lang string minConf float64 + extractor string } func newMarkdownRefetchCmd() kit.Command { @@ -79,8 +80,15 @@ Output schema (open-markdown-refetch-v1): html_length HTML body bytes (set only when content is HTML) markdown_length converted Markdown bytes markdown converted Markdown text + language ISO 639-3 code detected in the Markdown, "" if too short + language_confidence 0 to 1, how sure the identifier is + extractor engine that produced the text, as name@version error fetch error string (empty on success) +--extractor picks the conversion engine: h2m (default), readability, or raw. +The wet engine is not available here, since a live fetch returns HTML and there +is no Common Crawl text to pass through. + HF path layout (same as open-markdown): data/crawl=CC-MAIN-YYYY-WW/NNNNNN.parquet @@ -89,6 +97,7 @@ Examples: ccrawl markdown refetch --shards 0-9 --fetch-workers 400 --repo open-index/open-markdown-refetch-v1 ccrawl markdown refetch --shards all --parallel 2 --commit-batch 5 ccrawl markdown refetch --shards 0-99 --push=false --out ~/data/refetch + ccrawl markdown refetch --shards 0 --extractor readability --push=false HF_TOKEN=hf_... ccrawl markdown refetch --shards 0 -c 2026-25`, Flags: v.flags, Run: v.run, @@ -114,12 +123,29 @@ func (v *markdownRefetchCmd) flags(f *kit.FlagSet) { f.BoolVar(&v.noWARCCache, "no-warc-cache", false, "do not cache downloaded WARC shards to disk") f.StringVar(&v.lang, "lang", "", "keep only documents detected as this ISO 639-3 language (e.g. vie)") f.Float64Var(&v.minConf, "min-lang-confidence", ccrawl.DefaultMinLangConfidence, "confidence a document must clear for --lang") + f.StringVar(&v.extractor, "extractor", ccrawl.DefaultExtractor, "conversion engine: h2m|readability|raw") f.BoolVar(&v.fetchOnly, "fetch-only", false, "store raw HTML and skip the convert phase, so fetch runs at full speed (convert offline later over the html column)") } func (v *markdownRefetchCmd) run(ctx context.Context, _ []string) error { app := appFromCtx(ctx) + // Flag combinations that cannot work are settled before anything is + // downloaded, so a typo costs a message and not a manifest. + ex, err := ccrawl.LookupExtractor(v.extractor) + if err != nil { + return usageErr(err.Error()) + } + if ex.SourceKind != "warc" { + return usageErr(fmt.Sprintf("extractor %s cannot be used with refetch: it reads %s shards, and refetch converts pages it fetched live", ex.Name, ex.SourceKind)) + } + // Detection runs on the extracted Markdown, and --fetch-only never extracts + // any, so the two together would drop every row and report a filter working + // perfectly. Refuse instead. + if v.lang != "" && v.fetchOnly { + return usageErr("lang cannot be used with fetch-only: the language is detected in the converted Markdown, which fetch-only does not produce") + } + crawlID, err := app.Crawl(ctx) if err != nil { return err @@ -236,13 +262,6 @@ func (v *markdownRefetchCmd) run(ctx context.Context, _ []string) error { if v.fetchOnly { fmt.Fprintf(os.Stderr, "refetch: fetch-only mode, storing raw HTML and skipping convert\n") } - // Detection runs on the extracted Markdown, and --fetch-only never extracts - // any, so the two together would drop every row and report a filter working - // perfectly. Refuse instead. - if v.lang != "" && v.fetchOnly { - return usageErr("lang cannot be used with fetch-only: the language is detected in the converted Markdown, which fetch-only does not produce") - } - // The journal lands beside the ledger, so a resumed run's events sit next to // the record of what it resumed from. rep, stopRun, err := app.StartRun("markdown refetch", filepath.Join(outDir, "run.jsonl")) @@ -274,6 +293,7 @@ func (v *markdownRefetchCmd) run(ctx context.Context, _ []string) error { Lang: v.lang, MinLangConfidence: v.minConf, + Extractor: ex, }) n := int64(run.Committed) diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md index 2fb35b0..866d68f 100644 --- a/docs/content/reference/cli.md +++ b/docs/content/reference/cli.md @@ -308,6 +308,38 @@ ccrawl markdown refetch --shards 0-9 --fetch-workers 400 --repo org/name Both take `--shards` (`N`, `N-M`, `N,M`, or `all`), resume from a ledger at `/.committed`, and need `HF_TOKEN` unless `--push=false`. The [markdown reference](/reference/markdown/) has both schemas, the HuggingFace path layout, and the tuning flags. +### Choosing an extractor + +`--extractor` picks the engine that turns a captured page into the text in the row. + +| Engine | Reads | Does | +|---|---|---| +| `h2m` (default) | WARC | go-trafilatura tuned for recall, rendered as GitHub-flavored Markdown | +| `readability` | WARC | go-readability extraction plus mdconv, the engine `open-markdown-v2` shipped | +| `raw` | WARC | the whole document as Markdown, no boilerplate removal at all | +| `wet` | WET | the plain text Common Crawl already extracted, passed through unchanged | + +```sh +ccrawl markdown export --shards 0 --extractor readability --push=false --out ./md +ccrawl markdown export --shards 0 --extractor raw --push=false --out ./md +ccrawl markdown export --shards 0 --source-kind wet --extractor wet --push=false --out ./md +``` + +Which engine you use is a corpus quality decision, so it is a flag and not a build time constant. The same shard through two engines is two different corpora, and the only way to find out which one suits a downstream task is to build both and compare. Every row records the engine that produced it in the `extractor` column, as `name@version`, because extraction changes between releases and a name alone cannot explain why two shards built months apart disagree about the same page. + +`raw` keeps the nav bars and the footer, which sounds useless and is not. Every extractor is a lossy judgement about what a page was for, and on the pages where that judgement goes wrong the output alone does not say so. Raw is the control you measure the others against. + +`--source-kind` picks the manifest a run reads: `warc` takes `warc.paths.gz` and extracts the HTML itself, `wet` takes `wet.paths.gz` and uses the text Common Crawl already extracted. It defaults to whatever the extractor needs, so it rarely has to be given. The two are not interchangeable, since a WET file holds no HTML and a WARC holds no pre-extracted text, and asking for a pairing that cannot work is a usage error rather than a silently reinterpreted run: + +``` +$ ccrawl markdown export --shards 0 --source-kind wet --extractor h2m +ERROR Extractor h2m reads warc shards, so it cannot be used with --source-kind wet. +``` + +`markdown refetch` takes `--extractor` too, but only the WARC engines: a live fetch returns HTML, and there is no Common Crawl text to pass through. + +WET is much cheaper than any of the extractors, since the text is already there and the files are a fraction of the size of the WARCs. It is also somebody else's extraction decision, boilerplate and all, and it is plain text rather than Markdown, so headings and links are gone. + ### Language filtering Both subcommands label every row with a detected language and can keep only one of them. diff --git a/docs/content/reference/markdown.md b/docs/content/reference/markdown.md index 42ccc83..dbb89f2 100644 --- a/docs/content/reference/markdown.md +++ b/docs/content/reference/markdown.md @@ -53,8 +53,9 @@ An out-of-range index is a usage error, not a silent skip. | `markdown` | string | the converted Markdown | | `language` | string | ISO 639-3 language detected in the Markdown, empty when there was too little text | | `language_confidence` | double | the identifier's confidence, 0 to 1 | +| `extractor` | string | the engine that produced the text, as `name@version` | -v3 appends the last two columns and changes nothing else, so a reader written against v2 loads a v3 file and gets every column it asks for. +v3 appends the last three columns and changes nothing else, so a reader written against v2 loads a v3 file and gets every column it asks for. The default repo moved to `open-index/open-markdown-v3` to keep the two schemas in separate repos. ### open-markdown-refetch-v1, written by `markdown refetch` @@ -87,6 +88,7 @@ Everything the export schema has, plus the live response and its timings. | `markdown` | string | the converted Markdown | | `language` | string | ISO 639-3 language detected in the Markdown | | `language_confidence` | double | the identifier's confidence, 0 to 1 | +| `extractor` | string | the engine that produced the text, as `name@version` | | `error` | string | fetch error, empty on success | A failed fetch still produces a row. @@ -146,6 +148,64 @@ They exit 0 on success and 1 on a fatal error, and you re-run the same command t `urls publish` and `domains publish` are the commands that exit 75, because those runs commit into a single growing dataset and have a stall clock. See [exit codes](/reference/exit-codes/) for the full contract. +## Extractors + +`--extractor` chooses the engine that turns a captured page into the text in the `markdown` column. + +| Engine | Source | What it does | +|---|---|---| +| `h2m` (default) | WARC | go-trafilatura tuned for recall, rendered as GitHub-flavored Markdown | +| `readability` | WARC | go-readability plus mdconv, the engine `open-markdown-v2` shipped | +| `raw` | WARC | the whole document as Markdown, no boilerplate removal | +| `wet` | WET | the plain text Common Crawl already extracted, passed through unchanged | + +```sh +ccrawl markdown export --shards 0 --extractor readability --push=false --out ./md +ccrawl markdown export --shards 0 --source-kind wet --extractor wet --push=false --out ./md +``` + +Which engine you use is a corpus quality decision, so it is a flag rather than a build time constant. +The same shard through two engines is two different corpora, and the only way to find out which one suits a downstream task is to build both and compare them. + +Every row records the engine in the `extractor` column as `name@version`, and the dataset card names it too. +The version is there because extraction changes between releases: a dataset that only says `h2m` cannot explain why two shards built months apart disagree about the same page. +The WET engine has no version of ours to record, so it is stamped with the crawl instead, as `wet@CC-MAIN-2026-30`. + +`--source-kind` picks the manifest a run reads. +`warc` takes `warc.paths.gz` and extracts the HTML itself, `wet` takes `wet.paths.gz` and uses the text Common Crawl already extracted. +It defaults to whatever the extractor needs, so it rarely has to be given, and a pairing that cannot work is a usage error rather than a silently reinterpreted run: + +``` +$ ccrawl markdown export --shards 0 --source-kind wet --extractor h2m +ERROR Extractor h2m reads warc shards, so it cannot be used with --source-kind wet. +``` + +`markdown refetch` takes `--extractor` too, but only the WARC engines. +A live fetch returns HTML, so there is no Common Crawl text for the WET engine to pass through, and asking for it exits 2. + +### What the engines actually cost + +Shard 0 of `CC-MAIN-2026-30`, the same 3.5 GB of HTML through each engine on the same machine: + +| Engine | Rows | Markdown | Parquet | Mean doc | Wall clock | +|---|---|---|---|---|---| +| `h2m` | 20861 | 91.9 MB | 46.8 MB | 3990 B | 1m49s | +| `readability` | 20666 | 108.5 MB | 51.8 MB | 4865 B | 1m58s | +| `raw` | 20806 | 446.2 MB | 328.2 MB | 20724 B | 2m55s | +| `wet` | 21165 | 161.9 MB | 80.0 MB | 6604 B | 15s | + +`raw` produces roughly five times the text of `h2m` from the same pages, which is a fair measure of how much of a web page is not the page. +It keeps the nav bars and the footers, which sounds useless and is not: every extractor is a lossy judgement about what a page was for, and the output alone never says where that judgement went wrong. +Raw is the control you measure the others against. + +The three WARC engines agree on which pages are there and disagree on what is on them, which is the property that makes them comparable. +20353 documents are in all three, 97.6 percent of what `h2m` produced, and of the documents `h2m` and `readability` share only 284 have byte identical text. +The rest of the row count is pages one engine extracted nothing from and skipped rather than writing an empty row, and the engines do not give up on quite the same pages. + +`wet` is in a different cost class, since the text is already extracted and a WET file is a fraction of the size of the WARC it came from. +It is also somebody else's extraction decision, boilerplate included, and it is plain text rather than Markdown, so headings, links, and lists are gone. +Use it when you want volume cheaply and the structure does not matter. + ## Language filtering Every row is labelled with a detected language, and `--lang` keeps only one of them. @@ -264,12 +324,14 @@ Shared by `markdown export` and `markdown refetch`: | `--ledger` | Resume ledger path (default `/.committed`) | | `--lang` | Keep only documents detected as this ISO 639-3 language, for example `vie` | | `--min-lang-confidence` | Confidence a document has to clear for `--lang` (default 0.8) | +| `--extractor` | Conversion engine: `h2m`, `readability`, `raw`, or `wet` (default `h2m`, and `wet` is export only) | `markdown export` only: | Flag | Meaning | |---|---| | `--skip-errors` | Continue past per-shard failures instead of aborting | +| `--source-kind` | Manifest to read, `warc` or `wet` (default: whatever the extractor needs) | `markdown refetch` only: