diff --git a/ccrawl/extractor.go b/ccrawl/extractor.go
new file mode 100644
index 0000000..9834fbb
--- /dev/null
+++ b/ccrawl/extractor.go
@@ -0,0 +1,246 @@
+package ccrawl
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "net/url"
+ "runtime/debug"
+ "sort"
+ "strings"
+ "unicode/utf8"
+
+ "github.com/tamnd/h2m"
+ "github.com/tamnd/yomi/extract"
+ "github.com/tamnd/yomi/mdconv"
+ "golang.org/x/net/html"
+ "golang.org/x/net/html/charset"
+)
+
+// Extractor is one way of turning a captured page into the Markdown that lands
+// in the dataset. Which one you use is a corpus quality decision, not a build
+// time constant: the same shard run through two engines is two different
+// corpora, and the only way to find out which one suits a downstream task is to
+// build both and compare.
+//
+// Convert returns "" when the page yields nothing worth keeping, which the
+// pipelines treat as a skipped record rather than an error.
+type Extractor struct {
+ // Name is the value written to the extractor column, before the version.
+ Name string
+ // Summary is the one line shown in help.
+ Summary string
+ // Module is the Go module whose version identifies this engine. Empty when
+ // the engine is not ours to version, which is the WET case.
+ Module string
+ // SourceKind is the manifest a run using this extractor has to read, "warc"
+ // or "wet". They are not interchangeable: a WET file holds no HTML and a
+ // WARC holds no pre-extracted text.
+ SourceKind string
+
+ convert func(body []byte, pageURL string) string
+}
+
+// Extractors are the engines --extractor accepts, keyed by name.
+var Extractors = map[string]*Extractor{
+ "h2m": {
+ Name: "h2m",
+ Summary: "go-trafilatura tuned for recall, rendered as GFM (default)",
+ Module: "github.com/tamnd/h2m",
+ SourceKind: "warc",
+ convert: htmlToMarkdownH2M,
+ },
+ "readability": {
+ Name: "readability",
+ Summary: "go-readability extraction, the engine open-markdown-v2 shipped",
+ Module: "github.com/tamnd/yomi",
+ SourceKind: "warc",
+ convert: htmlToMarkdownReadability,
+ },
+ "raw": {
+ Name: "raw",
+ Summary: "the whole document rendered as Markdown, no boilerplate removal",
+ Module: "github.com/tamnd/yomi",
+ SourceKind: "warc",
+ convert: htmlToMarkdownRaw,
+ },
+ "wet": {
+ Name: "wet",
+ Summary: "the plain text Common Crawl already extracted, from the WET files",
+ SourceKind: "wet",
+ convert: wetTextPassthrough,
+ },
+}
+
+// DefaultExtractor is the engine a run uses when nobody says otherwise.
+const DefaultExtractor = "h2m"
+
+// LookupExtractor resolves an --extractor value.
+func LookupExtractor(name string) (*Extractor, error) {
+ if name == "" {
+ name = DefaultExtractor
+ }
+ e, ok := Extractors[strings.ToLower(strings.TrimSpace(name))]
+ if !ok {
+ return nil, fmt.Errorf("unknown extractor %q, want one of %s", name, strings.Join(ExtractorNames(), ", "))
+ }
+ return e, nil
+}
+
+// ExtractorNames lists the known engines in a stable order.
+func ExtractorNames() []string {
+ names := make([]string, 0, len(Extractors))
+ for n := range Extractors {
+ names = append(names, n)
+ }
+ sort.Strings(names)
+ return names
+}
+
+// Convert turns one captured body into Markdown.
+func (e *Extractor) Convert(body []byte, pageURL string) string {
+ if e == nil {
+ return htmlToMarkdownH2M(body, pageURL)
+ }
+ return e.convert(body, pageURL)
+}
+
+// ID is what goes in the extractor column and the dataset card: the engine name
+// and the version of the code that produced the text, as name@version.
+//
+// The version matters more than the name. Extraction changes between releases,
+// sometimes visibly, and a dataset that only records "h2m" cannot answer why two
+// shards built three months apart disagree about the same page.
+//
+// The WET engine is Common Crawl's own and has no version we can read, so it is
+// stamped with the crawl instead, which is the closest thing to a version the
+// text has.
+func (e *Extractor) ID(crawlID string) string {
+ if e == nil {
+ return DefaultExtractor + "@" + moduleVersion(Extractors[DefaultExtractor].Module)
+ }
+ if e.Module == "" {
+ if crawlID == "" {
+ return e.Name
+ }
+ return e.Name + "@" + crawlID
+ }
+ return e.Name + "@" + moduleVersion(e.Module)
+}
+
+// moduleVersion reads a dependency's version out of the build info of the
+// running binary, so the stamp describes the build that actually ran rather
+// than whatever go.mod said when this file was written.
+func moduleVersion(path string) string {
+ info, ok := debug.ReadBuildInfo()
+ if !ok {
+ return "unknown"
+ }
+ for _, d := range info.Deps {
+ if d.Path == path {
+ if d.Replace != nil && d.Replace.Version != "" {
+ return d.Replace.Version
+ }
+ return d.Version
+ }
+ }
+ return "unknown"
+}
+
+// htmlToMarkdownH2M converts one HTML body to Markdown via h2m: go-trafilatura
+// tuned for recall strips boilerplate and isolates the main content, then h2m
+// renders GitHub-flavored Markdown with links resolved against pageURL. h2m
+// transcodes non-UTF-8 bodies (GBK, Shift-JIS, Latin-1, and so on) from the
+// page's declared charset before parsing, so Common Crawl's mixed-encoding
+// pages convert correctly.
+func htmlToMarkdownH2M(body []byte, pageURL string) string {
+ if len(body) == 0 {
+ return ""
+ }
+ res := h2m.Convert(body, pageURL)
+ if !res.HasContent {
+ return ""
+ }
+ return validUTF8(res.Markdown)
+}
+
+// htmlToMarkdownReadability is the engine open-markdown-v2 shipped: yomi's
+// go-readability extraction plus mdconv rendering. It is faster than h2m and
+// keeps less of the page, which on a news article is usually the same text and
+// on a listing page is much less of it.
+func htmlToMarkdownReadability(body []byte, pageURL string) string {
+ utf8Body := transcode(body)
+ if len(utf8Body) == 0 {
+ return ""
+ }
+ art, err := extract.FromHTML(utf8Body, pageURL)
+ if err != nil || art.Node == nil {
+ return ""
+ }
+ base, _ := url.Parse(pageURL)
+ md, err := mdconv.Convert(art.Node, mdconv.Options{Base: base})
+ if err != nil {
+ return ""
+ }
+ return validUTF8(strings.TrimSpace(md))
+}
+
+// htmlToMarkdownRaw renders the whole document and skips extraction entirely,
+// so the nav bars, the footer, and the cookie banner all come along.
+//
+// That sounds useless and is not. Every extractor is a lossy judgement about
+// what the page was for, and on the pages where that judgement goes wrong there
+// is no way to tell from the output alone. Raw is the control: it says what was
+// on the page, so a corpus built with a real extractor can be measured against
+// what it threw away.
+func htmlToMarkdownRaw(body []byte, pageURL string) string {
+ utf8Body := transcode(body)
+ if len(utf8Body) == 0 {
+ return ""
+ }
+ doc, err := html.Parse(bytes.NewReader(utf8Body))
+ if err != nil {
+ return ""
+ }
+ base, _ := url.Parse(pageURL)
+ md, err := mdconv.Convert(doc, mdconv.Options{Base: base})
+ if err != nil {
+ return ""
+ }
+ return validUTF8(strings.TrimSpace(md))
+}
+
+// wetTextPassthrough is the WET engine: Common Crawl already extracted the text,
+// so there is nothing to do but keep it. It is not Markdown and does not pretend
+// to be, which is the trade for text that costs no CPU and no HTML download.
+func wetTextPassthrough(body []byte, _ string) string {
+ return validUTF8(strings.TrimSpace(string(body)))
+}
+
+// transcode converts a body to UTF-8 using its declared charset. A page that
+// declares nothing is sniffed, and one that declares a charset it does not
+// honour comes out with replacement characters rather than an error.
+func transcode(body []byte) []byte {
+ if len(body) == 0 {
+ return nil
+ }
+ r, err := charset.NewReader(bytes.NewReader(body), "")
+ if err != nil {
+ return nil
+ }
+ out, err := io.ReadAll(r)
+ if err != nil {
+ return nil
+ }
+ return out
+}
+
+// validUTF8 drops invalid byte sequences. A handful of pages declare one charset
+// and serve another, so transcoded text can still carry stray bytes, and parquet
+// strings and the Arrow readers on top of them require valid UTF-8.
+func validUTF8(s string) string {
+ if utf8.ValidString(s) {
+ return s
+ }
+ return strings.ToValidUTF8(s, "")
+}
diff --git a/ccrawl/extractor_test.go b/ccrawl/extractor_test.go
new file mode 100644
index 0000000..ece8a62
--- /dev/null
+++ b/ccrawl/extractor_test.go
@@ -0,0 +1,288 @@
+package ccrawl
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ kgzip "github.com/klauspost/compress/gzip"
+ "github.com/parquet-go/parquet-go"
+)
+
+// articleHTML is a page with a real article buried in the usual furniture, so
+// the engines have something to disagree about: an extractor that works keeps
+// the article, and the raw engine keeps all of it.
+const articleHTML = `
Quarterly report
+
+
+
Quarterly report
+
The committee published its quarterly report on Tuesday, noting that growth had slowed across most of the region while inflation remained broadly under control.
+
Analysts expect the central bank to hold rates steady until the end of the year, and several of them said the pause would last well into the next one.
+
+
+`
+
+func TestLookupExtractor(t *testing.T) {
+ tests := []struct {
+ name string
+ in string
+ wantName string
+ wantSource string
+ wantErr bool
+ }{
+ {name: "default on empty", in: "", wantName: "h2m", wantSource: "warc"},
+ {name: "by name", in: "readability", wantName: "readability", wantSource: "warc"},
+ {name: "case and space insensitive", in: " RAW ", wantName: "raw", wantSource: "warc"},
+ {name: "wet reads wet shards", in: "wet", wantName: "wet", wantSource: "wet"},
+ {name: "unknown", in: "trafilatura", wantErr: true},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ ex, err := LookupExtractor(tt.in)
+ if tt.wantErr {
+ if err == nil {
+ t.Fatalf("LookupExtractor(%q) = %v, want an error", tt.in, ex.Name)
+ }
+ // The error has to list the options, since the whole point of
+ // the flag is that the caller is choosing between them.
+ for _, n := range ExtractorNames() {
+ if !strings.Contains(err.Error(), n) {
+ t.Fatalf("error %q does not mention %q", err, n)
+ }
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("LookupExtractor(%q): %v", tt.in, err)
+ }
+ if ex.Name != tt.wantName || ex.SourceKind != tt.wantSource {
+ t.Fatalf("got %s reading %s shards, want %s reading %s", ex.Name, ex.SourceKind, tt.wantName, tt.wantSource)
+ }
+ })
+ }
+}
+
+// TestExtractorID pins the format the parquet column and the dataset card both
+// use. A bare name is not enough: extraction changes between releases, so the
+// version is part of the answer.
+//
+// The version itself reads "unknown" here and only here. A test binary's build
+// info carries no dependency list, so this can check the shape but not the
+// value; a real binary stamps h2m@v0.2.1.
+func TestExtractorID(t *testing.T) {
+ for _, name := range []string{"h2m", "readability", "raw"} {
+ id := Extractors[name].ID("CC-MAIN-2026-30")
+ prefix, version, ok := strings.Cut(id, "@")
+ if !ok || prefix != name {
+ t.Fatalf("ID = %q, want %s@version", id, name)
+ }
+ if version == "" {
+ t.Fatalf("ID = %q, want a version after the name", id)
+ }
+ }
+ if v := moduleVersion("example.com/nothing/we/import"); v != "unknown" {
+ t.Fatalf("moduleVersion of a module we do not depend on = %q, want unknown", v)
+ }
+ // The WET text is Common Crawl's own and has no version we can read, so the
+ // crawl stands in for one.
+ if id := Extractors["wet"].ID("CC-MAIN-2026-30"); id != "wet@CC-MAIN-2026-30" {
+ t.Fatalf("wet ID = %q, want wet@CC-MAIN-2026-30", id)
+ }
+ if id := Extractors["wet"].ID(""); id != "wet" {
+ t.Fatalf("wet ID with no crawl = %q, want wet", id)
+ }
+}
+
+// TestExtractorsAgreeOnPagesDisagreeOnText is the done-when from the issue: the
+// same shard through three engines gives the same documents and different text.
+// Same doc_id set is what makes the corpora comparable; different markdown is
+// what makes the choice worth having.
+func TestExtractorsAgreeOnPagesDisagreeOnText(t *testing.T) {
+ var shard bytes.Buffer
+ for i := range 4 {
+ shard.Write(warcMember(t, fmt.Sprintf("https://example.com/article/%d", i), articleHTML))
+ }
+
+ dir := t.TempDir()
+ md := map[string]map[string]string{} // engine -> doc_id -> markdown
+ for _, name := range []string{"h2m", "readability", "raw"} {
+ out := filepath.Join(dir, name+".parquet")
+ stats, err := packStream(context.Background(),
+ bytes.NewReader(shard.Bytes()),
+ MarkdownPackConfig{OutPath: out, Workers: 2, CrawlID: "CC-MAIN-2026-30", Extractor: Extractors[name]},
+ MarkdownStats{}, time.Now())
+ if err != nil {
+ t.Fatalf("%s: packStream: %v", name, err)
+ }
+ if stats.Rows != 4 {
+ t.Fatalf("%s: Rows = %d, want 4", name, stats.Rows)
+ }
+ rows, err := parquet.ReadFile[MarkdownRow](out)
+ if err != nil {
+ t.Fatalf("%s: read parquet: %v", name, err)
+ }
+ md[name] = map[string]string{}
+ for _, r := range rows {
+ md[name][r.DocID] = r.Markdown
+ if want := Extractors[name].ID("CC-MAIN-2026-30"); r.Extractor != want {
+ t.Fatalf("%s: row extractor = %q, want %q", name, r.Extractor, want)
+ }
+ }
+ }
+
+ for _, name := range []string{"readability", "raw"} {
+ if len(md[name]) != len(md["h2m"]) {
+ t.Fatalf("%s produced %d documents, h2m produced %d", name, len(md[name]), len(md["h2m"]))
+ }
+ for id := range md["h2m"] {
+ if _, ok := md[name][id]; !ok {
+ t.Fatalf("%s is missing doc_id %s that h2m produced", name, id)
+ }
+ }
+ }
+
+ // The article survives every engine. Only raw keeps the furniture, which is
+ // the difference the flag exists for.
+ for _, name := range []string{"h2m", "readability", "raw"} {
+ for id, text := range md[name] {
+ if !strings.Contains(text, "quarterly report") {
+ t.Fatalf("%s dropped the article from %s: %q", name, id, text)
+ }
+ hasFooter := strings.Contains(text, "Privacy policy")
+ if name == "raw" && !hasFooter {
+ t.Fatalf("raw dropped the footer, which is the one thing it is for: %q", text)
+ }
+ if name != "raw" && hasFooter {
+ t.Fatalf("%s kept the footer: %q", name, text)
+ }
+ }
+ }
+}
+
+// wetShard builds a WET file: WARC conversion records holding plain text.
+func wetShard(t *testing.T, pages map[string]string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ for uri, text := range pages {
+ rec := fmt.Sprintf("WARC/1.0\r\nWARC-Type: conversion\r\nWARC-Target-URI: %s\r\nWARC-Date: 2026-07-10T00:00:00Z\r\nWARC-Record-ID: \r\nContent-Type: text/plain\r\nContent-Length: %d\r\n\r\n%s\r\n\r\n",
+ uri, MarkdownDocID(uri)[:16], len(text), text)
+ zw := kgzip.NewWriter(&buf)
+ if _, err := zw.Write([]byte(rec)); err != nil {
+ t.Fatal(err)
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return buf.Bytes()
+}
+
+// TestPackStreamWETSource covers the other half of --source-kind: a WET shard
+// holds no HTML at all, so the pipeline has to read conversion records and pass
+// the text through instead of extracting anything.
+func TestPackStreamWETSource(t *testing.T) {
+ pages := map[string]string{
+ "https://example.com/vi": vieText,
+ "https://example.com/en": engText,
+ }
+ out := filepath.Join(t.TempDir(), "wet.parquet")
+ stats, err := packStream(context.Background(),
+ bytes.NewReader(wetShard(t, pages)),
+ MarkdownPackConfig{OutPath: out, Workers: 2, CrawlID: "CC-MAIN-2026-30", Extractor: Extractors["wet"]},
+ MarkdownStats{}, time.Now())
+ if err != nil {
+ t.Fatalf("packStream: %v", err)
+ }
+ if stats.Rows != 2 {
+ t.Fatalf("Rows = %d, want 2", stats.Rows)
+ }
+ rows, err := parquet.ReadFile[MarkdownRow](out)
+ if err != nil {
+ t.Fatalf("read parquet: %v", err)
+ }
+ for _, r := range rows {
+ want, ok := pages[r.URL]
+ if !ok {
+ t.Fatalf("unexpected url %q", r.URL)
+ }
+ if r.Markdown != want {
+ t.Fatalf("%s: text was changed on the way through:\n got %q\nwant %q", r.URL, r.Markdown, want)
+ }
+ if r.Extractor != "wet@CC-MAIN-2026-30" {
+ t.Fatalf("%s: extractor = %q", r.URL, r.Extractor)
+ }
+ // The language filter reads the same column whatever produced it, which
+ // is the reason WET is worth having as a source at all.
+ if r.URL == "https://example.com/vi" && r.Language != "vie" {
+ t.Fatalf("wet text was not identified: %q at %.2f", r.Language, r.LangConfidence)
+ }
+ }
+}
+
+// TestWETSourceRespectsLangFilter checks the filter still applies when the text
+// came from Common Crawl rather than from our own extractor.
+func TestWETSourceRespectsLangFilter(t *testing.T) {
+ out := filepath.Join(t.TempDir(), "wet.parquet")
+ stats, err := packStream(context.Background(),
+ bytes.NewReader(wetShard(t, map[string]string{
+ "https://example.com/vi": vieText,
+ "https://example.com/en": engText,
+ })),
+ MarkdownPackConfig{OutPath: out, Workers: 2, CrawlID: "CC-MAIN-2026-30", Extractor: Extractors["wet"], Lang: "vie"},
+ MarkdownStats{}, time.Now())
+ if err != nil {
+ t.Fatalf("packStream: %v", err)
+ }
+ if stats.Rows != 1 || stats.LangDropped != 1 {
+ t.Fatalf("Rows = %d, LangDropped = %d, want 1 and 1", stats.Rows, stats.LangDropped)
+ }
+}
+
+// TestREADMENamesTheEngine covers the second done-when: the dataset card has to
+// say which engine produced the data, and it has to say the version too.
+func TestREADMENamesTheEngine(t *testing.T) {
+ for _, name := range []string{"h2m", "readability", "raw", "wet"} {
+ id := Extractors[name].ID("CC-MAIN-2026-30")
+ card := GenerateMarkdownREADME(MarkdownDatasetStats{
+ CrawlID: "CC-MAIN-2026-30",
+ CommittedShards: 1,
+ TotalShards: 1,
+ Rows: 100,
+ Extractor: id,
+ })
+ if !strings.Contains(card, id) {
+ t.Fatalf("card for %s never names %q", name, id)
+ }
+ if !strings.Contains(card, "`extractor`") {
+ t.Fatalf("card for %s does not document the extractor column", name)
+ }
+ }
+
+ // A WET dataset came from a different manifest and skipped extraction, so
+ // the steps that describe the input have to say so. Telling a reader to
+ // download .warc.gz for a dataset built from WET files is wrong in the one
+ // place they go to find out where the text came from.
+ wetCard := GenerateMarkdownREADME(MarkdownDatasetStats{
+ CrawlID: "CC-MAIN-2026-30", CommittedShards: 1, TotalShards: 1, Rows: 100,
+ Extractor: Extractors["wet"].ID("CC-MAIN-2026-30"),
+ })
+ if !strings.Contains(wetCard, ".warc.wet.gz") || strings.Contains(wetCard, "**Download** raw .warc.gz") {
+ t.Fatal("the WET card still describes downloading WARC files")
+ }
+ if strings.Contains(wetCard, "HTTP 200 responses") {
+ t.Fatal("the WET card still describes the response filter, which WET records do not have")
+ }
+
+ // And a filtered dataset says so, because "this is the Vietnamese subset" is
+ // not something a reader should have to infer from the rows.
+ card := GenerateMarkdownREADME(MarkdownDatasetStats{
+ CrawlID: "CC-MAIN-2026-30", CommittedShards: 1, TotalShards: 1, Rows: 100, Lang: "vie",
+ })
+ if !strings.Contains(card, "filtered to `vie`") {
+ t.Fatal("card does not mention the language filter")
+ }
+}
diff --git a/ccrawl/markdown.go b/ccrawl/markdown.go
index 10e4eea..8c7ff08 100644
--- a/ccrawl/markdown.go
+++ b/ccrawl/markdown.go
@@ -14,22 +14,24 @@ import (
"strings"
"sync"
"time"
- "unicode/utf8"
"github.com/parquet-go/parquet-go"
"github.com/parquet-go/parquet-go/compress/zstd"
- "github.com/tamnd/h2m"
)
// MarkdownRow is the parquet schema for one converted HTML document. It matches
// the open-index/open-markdown dataset layout so the output can be appended to
// existing crawls without a schema migration.
//
-// language and language_confidence make this open-markdown-v3. They are appended
-// at the end and every earlier column keeps its name and type, so a v2 reader
-// projecting the columns it knows about reads a v3 file without changes: parquet
-// is read by name, and a reader that never asks for the two new columns never
-// touches them.
+// language, language_confidence, and extractor make this open-markdown-v3. They
+// are appended at the end and every earlier column keeps its name and type, so a
+// v2 reader projecting the columns it knows about reads a v3 file without
+// changes: parquet is read by name, and a reader that never asks for the new
+// columns never touches them.
+//
+// extractor is name@version, not just the name. Extraction changes between
+// releases, sometimes visibly, and a dataset that only recorded the name could
+// not answer why two shards built months apart disagree about the same page.
type MarkdownRow struct {
DocID string `parquet:"doc_id"`
URL string `parquet:"url"`
@@ -41,6 +43,7 @@ type MarkdownRow struct {
Markdown string `parquet:"markdown"`
Language string `parquet:"language"`
LangConfidence float64 `parquet:"language_confidence"`
+ Extractor string `parquet:"extractor"`
}
// MarkdownDocID returns a stable 16-byte hex document ID derived from the URL.
@@ -122,6 +125,9 @@ type MarkdownPackConfig struct {
// MinLangConfidence is the floor Lang applies. 0 selects
// DefaultMinLangConfidence.
MinLangConfidence float64
+ // Extractor is the engine that turns a captured page into Markdown. nil
+ // selects the default, which keeps every existing caller on h2m.
+ Extractor *Extractor
// Progress is called after each row is written. It may be nil.
Progress func(MarkdownStats)
}
@@ -180,48 +186,21 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta
if minConf <= 0 {
minConf = DefaultMinLangConfidence
}
+ ex := cfg.Extractor
+ if ex == nil {
+ ex = Extractors[DefaultExtractor]
+ }
+ exID := ex.ID(cfg.CrawlID)
records := make(chan htmlRecord, workers*4)
rows := make(chan candidateRow, workers*4)
langCounts := map[string]int64{}
- // Reader: iterate the WARC multi-member gzip stream, push HTML records.
+ // Reader: iterate the source stream and push one record per page.
var readErr error
go func() {
defer close(records)
- queued := 0
- errStop := errors.New("record limit reached")
- readErr = IterateWARC(body, func(rec WARCRecord) error {
- select {
- case <-ctx.Done():
- return ctx.Err()
- default:
- }
- if rec.Header.Type != "response" || rec.Header.HTTPStatus != 200 {
- return nil
- }
- if !isHTMLMIME(rec.Header.HTTPMIME) {
- return nil
- }
- body := HTTPBody(rec.Block)
- if len(body) == 0 {
- return nil
- }
- records <- htmlRecord{
- url: rec.Header.TargetURI,
- date: rec.Header.Date.Format("2006-01-02"),
- recordID: rec.Header.RecordID,
- html: body,
- }
- queued++
- if cfg.MaxRecords > 0 && queued >= cfg.MaxRecords {
- return errStop
- }
- return nil
- })
- if readErr == errStop {
- readErr = nil // a deliberate early stop is not a read failure
- }
+ readErr = streamSourceRecords(ctx, body, cfg, ex, records)
}()
tConvert := time.Now()
@@ -233,7 +212,7 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta
go func() {
defer wg.Done()
for r := range records {
- md := convertGated(cfg.ConvertSem, r.html, r.url)
+ md := convertGated(cfg.ConvertSem, ex, r.html, r.url)
if md == "" {
continue
}
@@ -254,6 +233,7 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta
Markdown: md,
Language: code,
LangConfidence: conf,
+ Extractor: exID,
},
}
}
@@ -321,41 +301,73 @@ func newMarkdownParquetWriter(path string) (*ParquetWriter[MarkdownRow], error)
return &ParquetWriter[MarkdownRow]{f: f, w: w}, nil
}
-// htmlToMarkdown converts one HTML body to Markdown via h2m: go-trafilatura
-// tuned for recall strips boilerplate and isolates the main content, then h2m
-// renders GitHub-flavored Markdown with links resolved against pageURL. h2m
-// transcodes non-UTF-8 bodies (GBK, Shift-JIS, Latin-1, and so on) from the
-// page's declared charset before parsing, so Common Crawl's mixed-encoding
-// pages convert correctly.
-//
-// Returns "" when the body yields no extractable article or conversion fails.
-func htmlToMarkdown(body []byte, pageURL string) string {
- if len(body) == 0 {
- return ""
+// streamSourceRecords reads the shard and pushes one record per page. A WARC
+// shard carries HTML in response records; a WET shard carries text Common Crawl
+// already extracted, in conversion records. Both come out as the same struct,
+// so everything downstream of here is identical for the two sources and only
+// the extractor knows the difference.
+func streamSourceRecords(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, ex *Extractor, records chan<- htmlRecord) error {
+ queued := 0
+ errStop := errors.New("record limit reached")
+
+ push := func(rec htmlRecord) error {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case records <- rec:
+ }
+ queued++
+ if cfg.MaxRecords > 0 && queued >= cfg.MaxRecords {
+ return errStop
+ }
+ return nil
}
- res := h2m.Convert(body, pageURL)
- if !res.HasContent {
- return ""
+
+ var err error
+ if ex.SourceKind == "wet" {
+ err = IterateWET(body, cfg.CrawlID, func(rec WETRecord) error {
+ return push(htmlRecord{
+ url: rec.URL,
+ date: rec.Date.Format("2006-01-02"),
+ recordID: rec.RecordID,
+ html: []byte(rec.Text),
+ })
+ })
+ } else {
+ err = IterateWARC(body, func(rec WARCRecord) error {
+ if rec.Header.Type != "response" || rec.Header.HTTPStatus != 200 {
+ return nil
+ }
+ if !isHTMLMIME(rec.Header.HTTPMIME) {
+ return nil
+ }
+ html := HTTPBody(rec.Block)
+ if len(html) == 0 {
+ return nil
+ }
+ return push(htmlRecord{
+ url: rec.Header.TargetURI,
+ date: rec.Header.Date.Format("2006-01-02"),
+ recordID: rec.Header.RecordID,
+ html: html,
+ })
+ })
}
- // A handful of pages declare one charset but serve another, so the
- // transcoded Markdown can still carry stray invalid byte sequences. Parquet
- // strings and the Arrow readers on top of them require valid UTF-8, so drop
- // any invalid bytes before the value reaches the writer.
- if !utf8.ValidString(res.Markdown) {
- return strings.ToValidUTF8(res.Markdown, "")
+ if errors.Is(err, errStop) {
+ return nil // a deliberate early stop is not a read failure
}
- return res.Markdown
+ return err
}
-// convertGated runs htmlToMarkdown while holding a slot in sem, so the total
+// convertGated runs the extractor while holding a slot in sem, so the total
// number of concurrent conversions across all in-flight shards stays bounded.
-// A nil sem runs the conversion ungated.
-func convertGated(sem chan struct{}, body []byte, pageURL string) string {
+// A nil sem runs the conversion ungated, and a nil extractor runs the default.
+func convertGated(sem chan struct{}, ex *Extractor, body []byte, pageURL string) string {
if sem != nil {
sem <- struct{}{}
defer func() { <-sem }()
}
- return htmlToMarkdown(body, pageURL)
+ return ex.Convert(body, pageURL)
}
// isHTMLMIME reports whether a MIME type string names an HTML document.
diff --git a/ccrawl/markdown_bench_test.go b/ccrawl/markdown_bench_test.go
index 427bbb7..e44ecc3 100644
--- a/ccrawl/markdown_bench_test.go
+++ b/ccrawl/markdown_bench_test.go
@@ -4,45 +4,10 @@ import (
"io"
"os"
"strconv"
- "strings"
"testing"
"time"
-
- "net/url"
-
- "github.com/tamnd/yomi/extract"
- "github.com/tamnd/yomi/mdconv"
- "golang.org/x/net/html/charset"
)
-// htmlToMarkdownReadability is the engine open-markdown-v2 originally shipped:
-// yomi's go-readability extraction plus mdconv rendering. It is kept here only
-// so TestEngineComparison can measure it head to head against the h2m
-// (trafilatura, FavorRecall) engine that htmlToMarkdown now uses.
-func htmlToMarkdownReadability(body []byte, pageURL string) string {
- if len(body) == 0 {
- return ""
- }
- r, err := charset.NewReader(strings.NewReader(string(body)), "")
- if err != nil {
- return ""
- }
- utf8Body, err := io.ReadAll(r)
- if err != nil || len(utf8Body) == 0 {
- return ""
- }
- art, err := extract.FromHTML(utf8Body, pageURL)
- if err != nil || art.Node == nil {
- return ""
- }
- base, _ := url.Parse(pageURL)
- md, err := mdconv.Convert(art.Node, mdconv.Options{Base: base})
- if err != nil {
- return ""
- }
- return strings.TrimSpace(md)
-}
-
type engineRun struct {
name string
rows int
@@ -121,8 +86,9 @@ func TestEngineComparison(t *testing.T) {
name string
conv func([]byte, string) string
}{
- {"h2m (trafilatura, FavorRecall)", htmlToMarkdown},
- {"v2 (yomi readability + mdconv)", htmlToMarkdownReadability},
+ {"h2m (trafilatura, FavorRecall)", htmlToMarkdownH2M},
+ {"readability (yomi + mdconv)", htmlToMarkdownReadability},
+ {"raw (whole document, no extraction)", htmlToMarkdownRaw},
}
var runs []engineRun
@@ -169,7 +135,7 @@ func TestEngineComparison(t *testing.T) {
if dumped >= n {
break
}
- a := htmlToMarkdown(p.html, p.url)
+ a := htmlToMarkdownH2M(p.html, p.url)
b := htmlToMarkdownReadability(p.html, p.url)
if a == "" || b == "" {
continue
diff --git a/ccrawl/markdown_readme.go b/ccrawl/markdown_readme.go
index b882ded..62b3c43 100644
--- a/ccrawl/markdown_readme.go
+++ b/ccrawl/markdown_readme.go
@@ -25,6 +25,15 @@ type MarkdownDatasetStats struct {
ConvertS int64
ExportS int64
PublishS int64
+
+ // Extractor is the engine that produced the text, as name@version. A dataset
+ // that does not say which extractor built it cannot be reproduced or
+ // compared against another one, so the card says it in the processing steps
+ // and the schema carries it per row. Empty falls back to the default engine's
+ // description.
+ Extractor string
+ // Lang, when set, is the ISO 639-3 language the run filtered to.
+ Lang string
}
// GenerateMarkdownREADME produces the HuggingFace dataset card for
@@ -65,6 +74,28 @@ func GenerateMarkdownREADME(s MarkdownDatasetStats) string {
approx, fmtBytes(html), approx, fmtBytes(md), pct)
}
+ extractorID := s.Extractor
+ if extractorID == "" {
+ extractorID = Extractors[DefaultExtractor].ID(s.CrawlID)
+ }
+ convertStep := extractorCardStep(extractorID)
+ // The WET source reads a different manifest and skips extraction, so the
+ // steps describing the input have to change with it. A card that says
+ // "download .warc.gz" over a dataset built from WET files is wrong in the one
+ // place a reader goes to find out where the text came from.
+ downloadStep := "raw .warc.gz files from Common Crawl S3 (each file is roughly 1 GB compressed)"
+ filterStep := "to keep only HTTP 200 responses with a `text/html` content type, discarding images, scripts, redirects, and error pages"
+ streamLine := "The pipeline streams from the compressed WARC through conversion directly into Parquet. Pages that produce empty conversions are dropped."
+ if name, _, _ := strings.Cut(extractorID, "@"); name == "wet" {
+ downloadStep = "raw .warc.wet.gz files from Common Crawl S3, which hold the text Common Crawl extracted and are a fraction of the size of the WARC files they came from"
+ filterStep = "to keep only conversion records, which is every page Common Crawl produced text for"
+ streamLine = "The pipeline streams from the compressed WET file straight into Parquet. Pages that came with no text are dropped."
+ }
+ langStep := ""
+ if s.Lang != "" {
+ langStep = fmt.Sprintf(". This dataset was filtered to `%s`, so documents identified as anything else were dropped", s.Lang)
+ }
+
var b strings.Builder
// Frontmatter. The default config reads every crawl; a per-crawl config
@@ -193,7 +224,10 @@ The following is an example row from the dataset:
"warc_record_id": "",
"html_length": 48210,
"markdown_length": 3847,
- "markdown": "# Interesting Topic\n\nThis is the main content of the page..."
+ "markdown": "# Interesting Topic\n\nThis is the main content of the page...",
+ "language": "eng",
+ "language_confidence": 1.0,
+ "extractor": "%s"
}
`+"```"+`
@@ -209,6 +243,9 @@ The following is an example row from the dataset:
| `+"`html_length`"+` | int64 | Byte length of the original HTML body before conversion |
| `+"`markdown_length`"+` | int64 | Byte length of the converted Markdown body |
| `+"`markdown`"+` | string | Clean Markdown content extracted from the page |
+| `+"`language`"+` | string | ISO 639-3 language detected in the Markdown, empty when there was too little text to identify |
+| `+"`language_confidence`"+` | double | How sure the identifier is, 0 to 1 |
+| `+"`extractor`"+` | string | Engine that produced the Markdown, as `+"`name@version`"+`. Extraction changes between releases, so the version is part of the answer |
### Data Splits
@@ -228,21 +265,31 @@ The source data consists of web pages crawled by the [Common Crawl](https://comm
The processing pipeline runs as a single streaming pass with no intermediate files:
-1. **Download** raw .warc.gz files from Common Crawl S3 (each file is roughly 1 GB compressed)
-2. **Filter** to keep only HTTP 200 responses with a `+"`text/html`"+` content type, discarding images, scripts, redirects, and error pages
-3. **Convert** HTML to clean Markdown. [go-trafilatura](https://github.com/markusmobius/go-trafilatura) tuned for recall (`+"`FavorRecall`"+`, with Readability and DomDistiller fallbacks) isolates the main content node and strips navigation, ads, and boilerplate, then a direct node-tree walk renders GitHub-flavored Markdown with links resolved to absolute URLs
-4. **Export** straight to Apache Parquet with Zstd compression
+1. **Download** %s
+2. **Filter** %s
+3. **Convert** %s
+4. **Identify** the language of the converted text and record it with the identifier's confidence%s
+5. **Export** straight to Apache Parquet with Zstd compression
-The pipeline streams from the compressed WARC through conversion directly into Parquet. Pages that produce empty conversions are dropped.
+%s
+
+Every row records the engine that produced it as `+"`%s`"+`, so two shards built with different engines are never mistaken for one corpus.
`,
s.CrawlID, docsStr, fmtInt(int64(shards)), reduction, // intro line
- s.CrawlID, // file tree
- s.CrawlID, // datasets name=
- s.CrawlID, // datasets data_files=
- s.CrawlID, // huggingface_hub allow_patterns=
- s.CrawlID, // duckdb path
- s.CrawlID, // data splits example
+ s.CrawlID, // file tree
+ s.CrawlID, // datasets name=
+ s.CrawlID, // datasets data_files=
+ s.CrawlID, // huggingface_hub allow_patterns=
+ s.CrawlID, // duckdb path
+ extractorID, // example row extractor
+ s.CrawlID, // data splits example
+ downloadStep, // processing step 1
+ filterStep, // processing step 2
+ convertStep, // processing step 3
+ langStep, // processing step 4
+ streamLine, // what the pipeline streams through
+ extractorID, // engine stamped on every row
)
// Compression Ratios table. Rows are rendered only when we have data for
@@ -433,3 +480,20 @@ func fmtInt(n int64) string {
parts = append([]string{s}, parts...)
return strings.Join(parts, ",")
}
+
+// extractorCardStep describes one engine for the dataset card's processing
+// steps. The name is taken from the recorded name@version so the prose and the
+// column can never drift apart.
+func extractorCardStep(id string) string {
+ name, _, _ := strings.Cut(id, "@")
+ switch name {
+ case "readability":
+ return "HTML to Markdown with [go-readability](https://github.com/go-shiori/go-readability), a port of Mozilla's Readability.js, which isolates the article node and drops the rest, then renders Markdown with links resolved to absolute URLs"
+ case "raw":
+ return "the whole HTML document to Markdown with no extraction step at all, so navigation, footers, and boilerplate are all still there. This is the control corpus: it says what was on the page before any extractor made a judgement about it"
+ case "wet":
+ return "nothing. The text is what Common Crawl's own extractor produced in the WET files, kept as it was given, so this dataset costs no extraction and inherits whatever that extractor decided"
+ default:
+ return "HTML to clean Markdown. [go-trafilatura](https://github.com/markusmobius/go-trafilatura) tuned for recall (`FavorRecall`, with Readability and DomDistiller fallbacks) isolates the main content node and strips navigation, ads, and boilerplate, then a direct node-tree walk renders GitHub-flavored Markdown with links resolved to absolute URLs"
+ }
+}
diff --git a/ccrawl/markdown_run.go b/ccrawl/markdown_run.go
index f343e17..4c4d79f 100644
--- a/ccrawl/markdown_run.go
+++ b/ccrawl/markdown_run.go
@@ -116,6 +116,10 @@ type MarkdownExportConfig struct {
Lang string
MinLangConfidence float64
+ // Extractor is the engine every shard in the run uses. nil selects the
+ // default.
+ Extractor *Extractor
+
// Progress is called once per committed batch with a snapshot of the run.
// It may be nil.
Progress func(MarkdownRunStats)
@@ -279,6 +283,7 @@ func RunMarkdownExport(ctx context.Context, h *HTTPClient, hf *HFClient, cfg Mar
Lang: cfg.Lang,
MinLangConfidence: cfg.MinLangConfidence,
+ Extractor: cfg.Extractor,
})
inflight.Add(-1)
// Per-shard wall-clock is the useful convert figure for a parallel
@@ -426,6 +431,8 @@ func runCommitter(ctx context.Context, hf *HFClient, cfg MarkdownExportConfig, k
ParquetBytes: run.ParquetBytes,
ConvertS: run.ConvertS,
PublishS: run.PublishS,
+ Extractor: cfg.Extractor.ID(cfg.CrawlID),
+ Lang: cfg.Lang,
}
tmp, err := writeTempREADME(dstats)
if err != nil {
diff --git a/ccrawl/markdown_test.go b/ccrawl/markdown_test.go
index 4e3d480..0f338a9 100644
--- a/ccrawl/markdown_test.go
+++ b/ccrawl/markdown_test.go
@@ -60,7 +60,7 @@ func TestHTMLToMarkdown(t *testing.T) {