diff --git a/ccrawl/journal.go b/ccrawl/journal.go
index 04fd303..f9c1566 100644
--- a/ccrawl/journal.go
+++ b/ccrawl/journal.go
@@ -115,6 +115,14 @@ type RunEvent struct {
// not folded into Failed: a shard with a thousand dead hosts still succeeded.
FetchFailed int64 `json:"fetch_failed,omitempty"`
+ // LangDropped counts documents a --lang filter threw away, and LangCounts is
+ // the whole detected breakdown including the dropped ones. They ride on the
+ // events because the text summary at the end of a run is suppressed in the
+ // machine modes, and a filter's drop rate is the first thing to check when a
+ // shard comes out smaller than expected.
+ LangDropped int64 `json:"lang_dropped,omitempty"`
+ LangCounts map[string]int64 `json:"lang_counts,omitempty"`
+
Rate float64 `json:"rate_per_hour,omitempty"`
ETAS float64 `json:"eta_s,omitempty"`
ElapsedS float64 `json:"elapsed_s,omitempty"`
diff --git a/ccrawl/lang.go b/ccrawl/lang.go
new file mode 100644
index 0000000..e2201d7
--- /dev/null
+++ b/ccrawl/lang.go
@@ -0,0 +1,134 @@
+package ccrawl
+
+import (
+ "strings"
+ "unicode"
+
+ "github.com/RadhiFadlillah/whatlanggo"
+)
+
+// MinLangText is how much text a detection needs before the answer means
+// anything. Trigram identification on a nav bar and a cookie banner is a coin
+// flip, and a coin flip that reports 0.9 confidence is worse than no answer, so
+// anything shorter is reported as unknown rather than guessed at.
+const MinLangText = 40
+
+// DefaultMinLangConfidence is the confidence a document has to clear to be kept
+// by a --lang filter. It matches whatlanggo's own reliability threshold.
+const DefaultMinLangConfidence = 0.8
+
+// DetectLanguage identifies the language of a document and returns an ISO 639-3
+// code with the detector's confidence in it.
+//
+// This is a document level check on the extracted text, which is the point: the
+// columnar index carries CLD2 labels computed by Common Crawl over the raw HTML,
+// and those labels are unreliable for low resource languages and absent
+// entirely for a good share of rows. Running the identifier over the Markdown we
+// actually extracted asks the question about the text we are actually keeping.
+//
+// The code is "" with confidence 0 when there is too little text to be worth
+// identifying. Callers should treat that as unknown, not as a failed match.
+func DetectLanguage(text string) (string, float64) {
+ text = langSample(text)
+ if len([]rune(text)) < MinLangText {
+ return "", 0
+ }
+ info := whatlanggo.Detect(text)
+ code := whatlanggo.LangToString(info.Lang)
+ if code == "" {
+ return "", 0
+ }
+ return code, info.Confidence
+}
+
+// LangMatches reports whether a document's detected language satisfies a filter.
+// An empty want matches everything, which is what makes the filter opt in.
+//
+// A document with too little text to identify is dropped by a language filter
+// rather than kept. A filtered export is asking for documents known to be in one
+// language, and "we could not tell" is not that.
+func LangMatches(want, got string, conf, min float64) bool {
+ if want == "" {
+ return true
+ }
+ if got == "" {
+ return false
+ }
+ return got == want && conf >= min
+}
+
+// langSample trims the text handed to the identifier. Trigram profiling reaches
+// a stable answer within a few thousand characters and then spends the rest of
+// the document confirming it, so a long article is pure cost.
+//
+// The URLs have to go. A link heavy page, which on a news site means the
+// homepage and every section index, carries more characters of slug than of
+// prose, and a Vietnamese slug is stripped of its diacritics by definition. Fed
+// to the identifier raw, one of those pages reads as some Latin language nobody
+// asked about. The link text stays: it is the headline, and it is exactly the
+// prose we want to judge.
+func langSample(text string) string {
+ var b strings.Builder
+ b.Grow(min(len(text), langSampleBytes))
+ rs := []rune(text)
+ for i := 0; i < len(rs) && b.Len() < langSampleBytes; i++ {
+ r := rs[i]
+ switch {
+ case r == '(' && i > 0 && rs[i-1] == ']':
+ // A link target, right after its text. Skip to the close paren.
+ for i < len(rs) && rs[i] != ')' {
+ i++
+ }
+ b.WriteByte(' ')
+ case r == '<':
+ // An autolink or a stray tag, neither of which is prose.
+ for i < len(rs) && rs[i] != '>' {
+ i++
+ }
+ b.WriteByte(' ')
+ case isURLStart(rs, i):
+ for i < len(rs) && !unicode.IsSpace(rs[i]) && rs[i] != ')' {
+ i++
+ }
+ i--
+ b.WriteByte(' ')
+ case r == '#' || r == '*' || r == '_' || r == '|' || r == '`' || r == '[' || r == ']':
+ b.WriteByte(' ')
+ case unicode.IsSpace(r):
+ b.WriteByte(' ')
+ default:
+ b.WriteRune(r)
+ }
+ }
+ return b.String()
+}
+
+// isURLStart reports whether a bare URL begins at rs[i]. Only at a word
+// boundary, so it does not fire in the middle of an ordinary word.
+func isURLStart(rs []rune, i int) bool {
+ if i > 0 && !unicode.IsSpace(rs[i-1]) {
+ return false
+ }
+ for _, p := range [...]string{"http://", "https://", "www.", "javascript:"} {
+ if len(rs)-i >= len(p) && string(rs[i:i+len(p)]) == p {
+ return true
+ }
+ }
+ return false
+}
+
+// langSampleBytes is how much text the identifier gets. Measured on Vietnamese
+// news pages, the answer stops moving well before this.
+const langSampleBytes = 4096
+
+// LangSample exposes the text DetectLanguage judged, truncated to n runes, so
+// "content lang" can show what the answer was based on. A wrong answer is
+// usually a wrong input.
+func LangSample(text string, n int) string {
+ s := strings.Join(strings.Fields(langSample(text)), " ")
+ rs := []rune(s)
+ if len(rs) > n {
+ return string(rs[:n]) + "..."
+ }
+ return s
+}
diff --git a/ccrawl/lang_test.go b/ccrawl/lang_test.go
new file mode 100644
index 0000000..e06796b
--- /dev/null
+++ b/ccrawl/lang_test.go
@@ -0,0 +1,277 @@
+package ccrawl
+
+import (
+ "bytes"
+ "context"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/parquet-go/parquet-go"
+)
+
+const (
+ vieText = "Trong bối cảnh kinh tế toàn cầu nhiều biến động, Việt Nam vẫn giữ được đà tăng trưởng ổn định nhờ xuất khẩu và đầu tư nước ngoài tăng mạnh trong quý vừa qua. Các chuyên gia cho rằng chính sách tiền tệ linh hoạt đã góp phần kiềm chế lạm phát."
+ engText = "The committee published its quarterly report on Tuesday, noting that growth had slowed across most of the region while inflation remained broadly under control. Analysts expect the central bank to hold rates steady until the end of the year."
+)
+
+func TestDetectLanguage(t *testing.T) {
+ tests := []struct {
+ name string
+ text string
+ wantCode string
+ minConf float64
+ }{
+ {"vietnamese", vieText, "vie", 0.9},
+ {"english", engText, "eng", 0.9},
+ {"too short", "Hà Nội", "", 0},
+ {"empty", "", "", 0},
+ {"markdown syntax only", "| a | b |\n| --- | --- |\n# \n* \n", "", 0},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ code, conf := DetectLanguage(tt.text)
+ if code != tt.wantCode {
+ t.Fatalf("code = %q, want %q (confidence %.2f)", code, tt.wantCode, conf)
+ }
+ if conf < tt.minConf {
+ t.Fatalf("confidence = %.2f, want at least %.2f", conf, tt.minConf)
+ }
+ })
+ }
+}
+
+// TestDetectLanguageIgnoresLinkTargets pins the reason langSample exists. A
+// Vietnamese link list carries its prose in the link text and its noise in the
+// slugs, and the slugs have no diacritics, so leaving them in drags the answer
+// away from Vietnamese. Measured on a real vnexpress.net homepage this was the
+// difference between confidence 0.16 and confidence 1.
+func TestDetectLanguageIgnoresLinkTargets(t *testing.T) {
+ headlines := []struct{ text, slug string }{
+ {"Kiến nghị giảm tuần làm việc của lao động doanh nghiệp xuống bốn mươi giờ", "kien-nghi-giam-tuan-lam-viec-cua-lao-dong-doanh-nghiep-xuong-40-44-gio"},
+ {"Hà Nội sắp xếp còn mười bốn sở ngành sau đợt cải cách bộ máy hành chính", "ha-noi-sap-xep-con-14-so-nganh"},
+ {"Hơn một trăm ba mươi đại học công bố điểm chuẩn, nhiều ngành chạm trần", "diem-chuan-hon-200-dai-hoc-nam-2026"},
+ {"Bộ Khoa học và Công nghệ muốn đẩy mạnh nghiên cứu và đổi mới sáng tạo", "bo-khoa-hoc-va-cong-nghe-muon-day-manh-nghien-cuu"},
+ {"Thành phố Hải Phòng kiện toàn bộ máy lãnh đạo trong phiên họp sáng nay", "tp-hai-phong-kien-toan-bo-may-lanh-dao"},
+ {"Đề nghị cấm hòa giải vụ bạo lực gia đình để bảo vệ nạn nhân tốt hơn", "de-nghi-cam-hoa-giai-vu-bao-luc-gia-dinh-de-bao-ve-nan-nhan"},
+ }
+ var b strings.Builder
+ for _, h := range headlines {
+ b.WriteString("- [" + h.text + "](https://vnexpress.net/" + h.slug + "-5107278.html)\n")
+ }
+ code, conf := DetectLanguage(b.String())
+ if code != "vie" {
+ t.Fatalf("code = %q, want vie (confidence %.2f)", code, conf)
+ }
+ if conf < 0.9 {
+ t.Fatalf("confidence = %.2f, want at least 0.9", conf)
+ }
+ if s := LangSample(b.String(), 400); strings.Contains(s, "kien-nghi") {
+ t.Fatalf("sample still holds the slug: %q", s)
+ }
+}
+
+func TestLangMatches(t *testing.T) {
+ tests := []struct {
+ name string
+ want, got string
+ conf, min float64
+ expect bool
+ explainIfWrongIs string
+ }{
+ {name: "no filter keeps everything", want: "", got: "eng", conf: 1, min: 0.8, expect: true},
+ {name: "no filter keeps the unidentifiable", want: "", got: "", conf: 0, min: 0.8, expect: true},
+ {name: "match above the floor", want: "vie", got: "vie", conf: 0.95, min: 0.8, expect: true},
+ {name: "match at the floor", want: "vie", got: "vie", conf: 0.8, min: 0.8, expect: true},
+ {name: "match below the floor", want: "vie", got: "vie", conf: 0.5, min: 0.8, expect: false},
+ {name: "other language", want: "vie", got: "eng", conf: 1, min: 0.8, expect: false},
+ {name: "filter drops the unidentifiable", want: "vie", got: "", conf: 0, min: 0.8, expect: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := LangMatches(tt.want, tt.got, tt.conf, tt.min); got != tt.expect {
+ t.Fatalf("LangMatches(%q, %q, %v, %v) = %v, want %v", tt.want, tt.got, tt.conf, tt.min, got, tt.expect)
+ }
+ })
+ }
+}
+
+// langWARC builds a WARC stream of pages in the given languages, one record per
+// entry, and returns the bytes.
+func langWARC(t *testing.T, langs []string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ for i, l := range langs {
+ text := engText
+ switch l {
+ case "vie":
+ text = vieText
+ case "short":
+ text = "Xin chào"
+ }
+ html := "" + text + "
"
+ buf.Write(warcMember(t, "https://example.com/"+l+"/"+string(rune('a'+i)), html))
+ }
+ return buf.Bytes()
+}
+
+func TestPackStreamLanguageFilter(t *testing.T) {
+ langs := []string{"vie", "eng", "vie", "short", "eng", "vie"}
+ out := filepath.Join(t.TempDir(), "out.parquet")
+ cfg := MarkdownPackConfig{
+ OutPath: out,
+ Workers: 2,
+ Lang: "vie",
+ }
+ stats, err := packStream(context.Background(), bytes.NewReader(langWARC(t, langs)), cfg, MarkdownStats{}, time.Now())
+ if err != nil {
+ t.Fatalf("packStream: %v", err)
+ }
+ if stats.Rows != 3 {
+ t.Fatalf("Rows = %d, want 3", stats.Rows)
+ }
+ if stats.LangDropped != 3 {
+ t.Fatalf("LangDropped = %d, want 3", stats.LangDropped)
+ }
+ if got := stats.LangCounts["vie"]; got != 3 {
+ t.Fatalf("LangCounts[vie] = %d, want 3", got)
+ }
+ if got := stats.LangCounts["eng"]; got != 2 {
+ t.Fatalf("LangCounts[eng] = %d, want 2", got)
+ }
+ // The short page has no text worth identifying, so it lands under the empty
+ // code and the filter drops it rather than guessing.
+ if got := stats.LangCounts[""]; got != 1 {
+ t.Fatalf(`LangCounts[""] = %d, want 1`, got)
+ }
+
+ rows, err := parquet.ReadFile[MarkdownRow](out)
+ if err != nil {
+ t.Fatalf("read parquet: %v", err)
+ }
+ if len(rows) != 3 {
+ t.Fatalf("parquet holds %d rows, want 3", len(rows))
+ }
+ for _, r := range rows {
+ if r.Language != "vie" {
+ t.Fatalf("row %s has language %q, want vie", r.URL, r.Language)
+ }
+ if r.LangConfidence < 0.8 {
+ t.Fatalf("row %s has confidence %.2f, below the floor it was filtered on", r.URL, r.LangConfidence)
+ }
+ }
+}
+
+// TestPackStreamNoFilterStillLabels is the other half: without --lang nothing is
+// dropped, and every row still carries the language, which is what makes an
+// unfiltered v3 shard filterable later without re-extracting it.
+func TestPackStreamNoFilterStillLabels(t *testing.T) {
+ langs := []string{"vie", "eng", "short"}
+ out := filepath.Join(t.TempDir(), "out.parquet")
+ stats, err := packStream(context.Background(),
+ bytes.NewReader(langWARC(t, langs)),
+ MarkdownPackConfig{OutPath: out, Workers: 2},
+ MarkdownStats{}, time.Now())
+ if err != nil {
+ t.Fatalf("packStream: %v", err)
+ }
+ if stats.Rows != 3 || stats.LangDropped != 0 {
+ t.Fatalf("Rows = %d, LangDropped = %d, want 3 and 0", stats.Rows, stats.LangDropped)
+ }
+ rows, err := parquet.ReadFile[MarkdownRow](out)
+ if err != nil {
+ t.Fatalf("read parquet: %v", err)
+ }
+ var seen []string
+ for _, r := range rows {
+ seen = append(seen, r.Language)
+ }
+ for _, want := range []string{"vie", "eng", ""} {
+ if !slicesContains(seen, want) {
+ t.Fatalf("languages %q missing %q", seen, want)
+ }
+ }
+}
+
+// markdownRowV2 is the schema as it stood before the language columns, standing
+// in for a reader written against v2 that has not been updated.
+type markdownRowV2 struct {
+ DocID string `parquet:"doc_id"`
+ URL string `parquet:"url"`
+ Host string `parquet:"host"`
+ CrawlDate string `parquet:"crawl_date"`
+ WARCRecordID string `parquet:"warc_record_id"`
+ HTMLLength int64 `parquet:"html_length"`
+ MarkdownLength int64 `parquet:"markdown_length"`
+ Markdown string `parquet:"markdown"`
+}
+
+// TestV2ReaderParsesV3File is the compatibility promise in the issue: a reader
+// that only knows the v2 columns reads a v3 file and gets them all. Parquet
+// matches columns by name, so appending two at the end costs an old reader
+// nothing as long as it never asks for them.
+func TestV2ReaderParsesV3File(t *testing.T) {
+ out := filepath.Join(t.TempDir(), "v3.parquet")
+ if _, err := packStream(context.Background(),
+ bytes.NewReader(langWARC(t, []string{"vie", "eng"})),
+ MarkdownPackConfig{OutPath: out, Workers: 2},
+ MarkdownStats{}, time.Now()); err != nil {
+ t.Fatalf("packStream: %v", err)
+ }
+
+ v3, err := parquet.ReadFile[MarkdownRow](out)
+ if err != nil {
+ t.Fatalf("read as v3: %v", err)
+ }
+ v2, err := parquet.ReadFile[markdownRowV2](out)
+ if err != nil {
+ t.Fatalf("read as v2: %v", err)
+ }
+ if len(v2) != len(v3) {
+ t.Fatalf("v2 reader saw %d rows, v3 reader saw %d", len(v2), len(v3))
+ }
+ for i := range v2 {
+ if v2[i].DocID != v3[i].DocID || v2[i].URL != v3[i].URL || v2[i].Markdown != v3[i].Markdown {
+ t.Fatalf("row %d differs between readers:\n v2 %+v\n v3 %+v", i, v2[i], v3[i])
+ }
+ if v2[i].MarkdownLength != v3[i].MarkdownLength || v2[i].HTMLLength != v3[i].HTMLLength {
+ t.Fatalf("row %d lengths differ between readers", i)
+ }
+ }
+
+ // And the file really does carry the new columns, so the test above is
+ // proving compatibility rather than proving they were never written.
+ f, err := os.Open(out)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = f.Close() }()
+ fi, err := f.Stat()
+ if err != nil {
+ t.Fatal(err)
+ }
+ pf, err := parquet.OpenFile(f, fi.Size())
+ if err != nil {
+ t.Fatal(err)
+ }
+ var names []string
+ for _, c := range pf.Schema().Fields() {
+ names = append(names, c.Name())
+ }
+ for _, want := range []string{"language", "language_confidence"} {
+ if !slicesContains(names, want) {
+ t.Fatalf("schema %q is missing %q", names, want)
+ }
+ }
+}
+
+func slicesContains(hay []string, needle string) bool {
+ for _, s := range hay {
+ if s == needle {
+ return true
+ }
+ }
+ return false
+}
diff --git a/ccrawl/markdown.go b/ccrawl/markdown.go
index 512151a..10e4eea 100644
--- a/ccrawl/markdown.go
+++ b/ccrawl/markdown.go
@@ -24,15 +24,23 @@ import (
// MarkdownRow is the parquet schema for one converted HTML document. It matches
// the open-index/open-markdown dataset layout so the output can be appended to
// existing crawls without a schema migration.
+//
+// language and language_confidence make this open-markdown-v3. They are appended
+// at the end and every earlier column keeps its name and type, so a v2 reader
+// projecting the columns it knows about reads a v3 file without changes: parquet
+// is read by name, and a reader that never asks for the two new columns never
+// touches them.
type MarkdownRow struct {
- DocID string `parquet:"doc_id"`
- URL string `parquet:"url"`
- Host string `parquet:"host"`
- CrawlDate string `parquet:"crawl_date"`
- WARCRecordID string `parquet:"warc_record_id"`
- HTMLLength int64 `parquet:"html_length"`
- MarkdownLength int64 `parquet:"markdown_length"`
- Markdown string `parquet:"markdown"`
+ DocID string `parquet:"doc_id"`
+ URL string `parquet:"url"`
+ Host string `parquet:"host"`
+ CrawlDate string `parquet:"crawl_date"`
+ WARCRecordID string `parquet:"warc_record_id"`
+ HTMLLength int64 `parquet:"html_length"`
+ MarkdownLength int64 `parquet:"markdown_length"`
+ Markdown string `parquet:"markdown"`
+ Language string `parquet:"language"`
+ LangConfidence float64 `parquet:"language_confidence"`
}
// MarkdownDocID returns a stable 16-byte hex document ID derived from the URL.
@@ -55,6 +63,25 @@ type MarkdownStats struct {
DurConvert time.Duration
DurExport time.Duration
DurPublish time.Duration
+
+ // LangDropped counts documents a --lang filter threw away, and LangCounts
+ // breaks that down by the language they were identified as, with "" for the
+ // ones too short to identify. A drop rate on its own says the filter is
+ // working; the breakdown says what it is working on, which is the number
+ // worth looking at when the rate is not what you expected.
+ //
+ // LangCounts is filled in on the returned stats only, not on the snapshots
+ // handed to Progress, so nothing reads the map while it is being written.
+ LangDropped int64
+ LangCounts map[string]int64
+}
+
+// candidateRow is a converted document on its way to the writer, carrying the
+// language filter's verdict. Dropped rows travel too, so the counting stays in
+// the single writer goroutine where it needs no lock.
+type candidateRow struct {
+ row MarkdownRow
+ keep bool
}
// htmlRecord carries the extracted fields from one WARC response record.
@@ -88,6 +115,13 @@ type MarkdownPackConfig struct {
// sized to the CPU count so several shards can be in flight (hiding download
// latency) without oversubscribing the cores. nil means no global cap.
ConvertSem chan struct{}
+ // Lang, when set, keeps only documents identified as that ISO 639-3 language
+ // with at least MinLangConfidence. Empty keeps everything and still records
+ // the detected language on every row.
+ Lang string
+ // MinLangConfidence is the floor Lang applies. 0 selects
+ // DefaultMinLangConfidence.
+ MinLangConfidence float64
// Progress is called after each row is written. It may be nil.
Progress func(MarkdownStats)
}
@@ -142,8 +176,14 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta
return stats, err
}
+ minConf := cfg.MinLangConfidence
+ if minConf <= 0 {
+ minConf = DefaultMinLangConfidence
+ }
+
records := make(chan htmlRecord, workers*4)
- rows := make(chan MarkdownRow, workers*4)
+ rows := make(chan candidateRow, workers*4)
+ langCounts := map[string]int64{}
// Reader: iterate the WARC multi-member gzip stream, push HTML records.
var readErr error
@@ -197,15 +237,24 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta
if md == "" {
continue
}
- rows <- MarkdownRow{
- DocID: MarkdownDocID(r.url),
- URL: r.url,
- Host: urlHostname(r.url),
- CrawlDate: r.date,
- WARCRecordID: r.recordID,
- HTMLLength: int64(len(r.html)),
- MarkdownLength: int64(len(md)),
- Markdown: md,
+ // Identification runs on the extracted Markdown rather than the
+ // HTML, so the answer is about the text being kept and not about
+ // the nav bars and script tags that went in the bin.
+ code, conf := DetectLanguage(md)
+ rows <- candidateRow{
+ keep: LangMatches(cfg.Lang, code, conf, minConf),
+ row: MarkdownRow{
+ DocID: MarkdownDocID(r.url),
+ URL: r.url,
+ Host: urlHostname(r.url),
+ CrawlDate: r.date,
+ WARCRecordID: r.recordID,
+ HTMLLength: int64(len(r.html)),
+ MarkdownLength: int64(len(md)),
+ Markdown: md,
+ Language: code,
+ LangConfidence: conf,
+ },
}
}
}()
@@ -217,8 +266,13 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta
stats.DurDownload = time.Since(t0)
- for row := range rows {
- if werr := pw.Write(row); werr != nil {
+ for c := range rows {
+ langCounts[c.row.Language]++
+ if !c.keep {
+ stats.LangDropped++
+ continue
+ }
+ if werr := pw.Write(c.row); werr != nil {
go func() {
for range rows {
}
@@ -227,12 +281,13 @@ func packStream(ctx context.Context, body io.Reader, cfg MarkdownPackConfig, sta
return stats, fmt.Errorf("write parquet: %w", werr)
}
stats.Rows++
- stats.HTMLBytes += row.HTMLLength
- stats.MDBytes += row.MarkdownLength
+ stats.HTMLBytes += c.row.HTMLLength
+ stats.MDBytes += c.row.MarkdownLength
if cfg.Progress != nil {
cfg.Progress(stats)
}
}
+ stats.LangCounts = langCounts
stats.DurConvert = time.Since(tConvert)
diff --git a/ccrawl/markdown_run.go b/ccrawl/markdown_run.go
index ea010a2..f343e17 100644
--- a/ccrawl/markdown_run.go
+++ b/ccrawl/markdown_run.go
@@ -110,6 +110,12 @@ type MarkdownExportConfig struct {
// Ledger, when set, skips already-committed shards and records new ones.
Ledger *Ledger
+ // Lang keeps only documents identified as that ISO 639-3 language, and
+ // MinLangConfidence is the floor it applies. Empty Lang keeps everything and
+ // still records the detected language on every row.
+ Lang string
+ MinLangConfidence float64
+
// Progress is called once per committed batch with a snapshot of the run.
// It may be nil.
Progress func(MarkdownRunStats)
@@ -141,6 +147,14 @@ type MarkdownRunStats struct {
ETA time.Duration // estimated time to finish the remaining shards
FreeDiskBytes int64
RSSBytes int64 // resident set size at the last progress update
+
+ // LangDropped is how many converted documents the --lang filter threw away,
+ // and LangCounts breaks every converted document down by detected language
+ // with "" for the ones too short to identify. Kept per language is
+ // LangCounts minus the drops, which for a single language filter is just
+ // LangCounts[lang].
+ LangDropped int64
+ LangCounts map[string]int64
}
// packShardFn is the function the orchestrator uses to convert one shard. It
@@ -262,6 +276,9 @@ func RunMarkdownExport(ctx context.Context, h *HTTPClient, hf *HFClient, cfg Mar
OutPath: outPath,
Workers: c,
ConvertSem: sem,
+
+ Lang: cfg.Lang,
+ MinLangConfidence: cfg.MinLangConfidence,
})
inflight.Add(-1)
// Per-shard wall-clock is the useful convert figure for a parallel
@@ -340,6 +357,8 @@ func markdownTickEvent(kind string, s *MarkdownRunStats, cfg MarkdownExportConfi
HTMLBytes: s.HTMLBytes,
MDBytes: s.MDBytes,
ParquetBytes: s.ParquetBytes,
+ LangDropped: s.LangDropped,
+ LangCounts: s.LangCounts,
Rate: rate,
ETAS: eta,
ElapsedS: elapsed.Seconds(),
@@ -384,6 +403,13 @@ func runCommitter(ctx context.Context, hf *HFClient, cfg MarkdownExportConfig, k
run.MDBytes += r.stats.MDBytes
run.ParquetBytes += r.stats.ParquetBytes
run.ConvertS += int64(r.stats.DurConvert.Seconds())
+ run.LangDropped += r.stats.LangDropped
+ for code, n := range r.stats.LangCounts {
+ if run.LangCounts == nil {
+ run.LangCounts = map[string]int64{}
+ }
+ run.LangCounts[code] += n
+ }
}
var readmeTmp string
@@ -446,6 +472,8 @@ func runCommitter(ctx context.Context, hf *HFClient, cfg MarkdownExportConfig, k
HTMLBytes: r.stats.HTMLBytes,
MDBytes: r.stats.MDBytes,
ParquetBytes: r.stats.ParquetBytes,
+ LangDropped: r.stats.LangDropped,
+ LangCounts: r.stats.LangCounts,
ConvertS: r.stats.DurConvert.Seconds(),
})
}
diff --git a/ccrawl/refetch.go b/ccrawl/refetch.go
index c530375..c2b3f87 100644
--- a/ccrawl/refetch.go
+++ b/ccrawl/refetch.go
@@ -46,6 +46,9 @@ type RefetchRow struct {
MarkdownLength int64 `parquet:"markdown_length"`
Markdown string `parquet:"markdown"`
Error string `parquet:"error"`
+
+ Language string `parquet:"language"`
+ LangConfidence float64 `parquet:"language_confidence"`
}
// RefetchStats summarises one shard's refetch run with per-phase breakdown.
@@ -78,6 +81,13 @@ type RefetchStats struct {
MDBytes int64
DurConvert time.Duration
+ // LangDropped is how many rows the --lang filter kept out of the parquet, and
+ // LangCounts breaks every row down by detected language, with "" for the ones
+ // with no text to identify: the failures, the non-HTML, and the documents too
+ // short to call. LangCounts is set on the returned stats only.
+ LangDropped int64
+ LangCounts map[string]int64
+
// Phase 4: parquet write.
ParquetBytes int64
DurExport time.Duration
@@ -110,6 +120,16 @@ type RefetchPackConfig struct {
// cluster without holding back the crawl.
FetchOnly bool
+ // Lang, when set, keeps only documents identified as that ISO 639-3 language
+ // with at least MinLangConfidence, and drops every other row including the
+ // failures. The stats still count the whole attempt; it is the parquet that
+ // holds only what was asked for. Empty keeps everything and still records the
+ // detected language on every converted row.
+ Lang string
+ // MinLangConfidence is the floor Lang applies. 0 selects
+ // DefaultMinLangConfidence.
+ MinLangConfidence float64
+
// CacheDir, when set, is where the downloaded WARC is cached so a re-run of
// the same shard skips the multi-second download. The download streams to a
// .part file beside the final name and is renamed into place only once it is
@@ -205,6 +225,11 @@ func PackRefetchShard(ctx context.Context, h *HTTPClient, cfg RefetchPackConfig)
convertWorkers := runtime.NumCPU()
rows := make(chan RefetchRow, convertWorkers*4)
+ langCounts := map[string]int64{}
+ minConf := cfg.MinLangConfidence
+ if minConf <= 0 {
+ minConf = DefaultMinLangConfidence
+ }
var wg sync.WaitGroup
wg.Add(convertWorkers)
@@ -251,6 +276,7 @@ func PackRefetchShard(ctx context.Context, h *HTTPClient, cfg RefetchPackConfig)
md := convertGated(cfg.ConvertSem, res.Body, res.URL)
row.MarkdownLength = int64(len(md))
row.Markdown = md
+ row.Language, row.LangConfidence = DetectLanguage(md)
}
}
rows <- row
@@ -278,6 +304,14 @@ func PackRefetchShard(ctx context.Context, h *HTTPClient, cfg RefetchPackConfig)
stats.Rows++
}
}
+ // The stats above count the whole attempt, failures included, because
+ // that is what tells you how the crawl went. The filter decides only what
+ // reaches the parquet.
+ langCounts[row.Language]++
+ if !LangMatches(cfg.Lang, row.Language, row.LangConfidence, minConf) {
+ stats.LangDropped++
+ continue
+ }
if werr := pw.Write(row); werr != nil {
go func() {
for range rows {
@@ -294,6 +328,7 @@ func PackRefetchShard(ctx context.Context, h *HTTPClient, cfg RefetchPackConfig)
// By here, rows is drained → workers exited → results was closed by the
// FetchBatch goroutine → fetchEndTime has been written.
stats.DurFetch = fetchEndTime.Sub(t1)
+ stats.LangCounts = langCounts
stats.DurConvert = time.Since(t2)
tExport := time.Now()
diff --git a/ccrawl/refetch_run.go b/ccrawl/refetch_run.go
index a69b105..7c0f6ec 100644
--- a/ccrawl/refetch_run.go
+++ b/ccrawl/refetch_run.go
@@ -45,6 +45,12 @@ type RefetchExportConfig struct {
// fetch step runs at its true ceiling and Markdown is produced by a separate
// offline pass over the stored html column.
FetchOnly bool
+
+ // Lang keeps only documents identified as that ISO 639-3 language, and
+ // MinLangConfidence is the floor it applies. Empty Lang keeps everything and
+ // still records the detected language on every converted row.
+ Lang string
+ MinLangConfidence float64
// Ledger, when set, skips already-committed shards and records new ones.
Ledger *Ledger
@@ -90,6 +96,11 @@ type RefetchRunStats struct {
ETA time.Duration
FreeDiskBytes int64
RSSBytes int64 // resident set size at the last progress update
+
+ // LangDropped is how many rows the --lang filter kept out of the parquet, and
+ // LangCounts breaks every row down by detected language.
+ LangDropped int64
+ LangCounts map[string]int64
}
// packRefetchFn is the function the orchestrator uses to refetch one shard.
@@ -198,6 +209,9 @@ func RunRefetchExport(ctx context.Context, h *HTTPClient, hf *HFClient, cfg Refe
ConvertSem: sem,
CacheDir: cfg.CacheDir,
FetchOnly: cfg.FetchOnly,
+
+ Lang: cfg.Lang,
+ MinLangConfidence: cfg.MinLangConfidence,
})
inflight.Add(-1)
if err != nil {
@@ -271,6 +285,8 @@ func refetchTickEvent(kind string, s *RefetchRunStats, cfg RefetchExportConfig,
HTMLBytes: s.HTMLBytes,
MDBytes: s.MDBytes,
ParquetBytes: s.ParquetBytes,
+ LangDropped: s.LangDropped,
+ LangCounts: s.LangCounts,
Rate: rate,
ETAS: eta,
ElapsedS: elapsed.Seconds(),
@@ -320,6 +336,13 @@ func runRefetchCommitter(ctx context.Context, hf *HFClient, cfg RefetchExportCon
run.ErrRefused += r.stats.ErrRefused
run.ErrSkip += r.stats.ErrSkip
run.ErrOther += r.stats.ErrOther
+ run.LangDropped += r.stats.LangDropped
+ for code, n := range r.stats.LangCounts {
+ if run.LangCounts == nil {
+ run.LangCounts = map[string]int64{}
+ }
+ run.LangCounts[code] += n
+ }
}
if cfg.Push {
@@ -378,6 +401,8 @@ func runRefetchCommitter(ctx context.Context, hf *HFClient, cfg RefetchExportCon
HTMLBytes: r.stats.HTMLBytes,
MDBytes: r.stats.MDBytes,
ParquetBytes: r.stats.ParquetBytes,
+ LangDropped: r.stats.LangDropped,
+ LangCounts: r.stats.LangCounts,
FetchFailed: r.stats.Failed,
ExtractS: r.stats.DurExtract.Seconds(),
FetchS: r.stats.DurFetch.Seconds(),
diff --git a/cli/content_v2.go b/cli/content_v2.go
index b953f18..aec271d 100644
--- a/cli/content_v2.go
+++ b/cli/content_v2.go
@@ -14,6 +14,73 @@ func registerContentV2(app *kit.App) {
registerContentExtract(app)
registerContentQuality(app)
registerContentLinksV2(app)
+ registerContentLang(app)
+}
+
+// ── content lang ──────────────────────────────────────────────────────────────
+
+type contentLangIn struct {
+ App *App `kit:"inject"`
+ URL string `kit:"arg" name:"url" help:"URL to fetch and identify"`
+}
+
+// LangReport is what the markdown pipelines decide on for one URL.
+type LangReport struct {
+ URL string `json:"url" table:"url"`
+ Language string `json:"language" table:"language"`
+ Confidence float64 `json:"confidence" table:"confidence"`
+ CCLanguage string `json:"cc_language,omitempty" table:"cc_language"`
+ Chars int `json:"chars" table:"chars"`
+
+ // Sample is the text the identifier actually saw, truncated. When an answer
+ // looks wrong this is the first thing to look at, because it is usually the
+ // input that is wrong and not the identifier.
+ Sample string `json:"sample" table:"sample"`
+}
+
+func registerContentLang(app *kit.App) {
+ kit.Handle(app, kit.OpMeta{
+ Name: "lang",
+ Parent: "content",
+ Single: true,
+ Summary: "Identify the language of a page the way the markdown pipelines do",
+ Long: `Fetch a URL, convert it to Markdown, and run the same identifier that
+"markdown export --lang" applies, so a document that was kept or dropped can be
+asked about one URL at a time.
+
+The language is detected in the extracted Markdown, not in the raw HTML and not
+taken from the page's own lang attribute, because that is what the pipelines
+filter on. cc_language is what the page declares, shown alongside so a
+disagreement is visible rather than silent.
+
+This is a coarse pre-filter. A trigram identifier tells Vietnamese from Malay
+well enough to cut a corpus down, and it is not a substitute for a language
+specific classifier.
+
+Examples:
+ ccrawl content lang https://vnexpress.net/
+ ccrawl content lang https://example.com/ -o json`,
+ Args: []kit.Arg{{Name: "url"}},
+ }, func(ctx context.Context, in contentLangIn, emit func(LangReport) error) error {
+ rawURL := in.URL
+ if !strings.HasPrefix(rawURL, "http") {
+ rawURL = "https://" + rawURL
+ }
+ res, err := ccrawl.CrawlURL(ctx, rawURL, ccrawl.DefaultCrawlConfig)
+ if err != nil {
+ return fmt.Errorf("fetch %s: %w", rawURL, err)
+ }
+ md, _ := ccrawl.ExtractMarkdown(res.Body)
+ code, conf := ccrawl.DetectLanguage(md)
+ return emit(LangReport{
+ URL: res.FinalURL,
+ Language: code,
+ Confidence: conf,
+ CCLanguage: ccrawl.ExtractContent(res.Body).Language,
+ Chars: len([]rune(md)),
+ Sample: ccrawl.LangSample(md, 200),
+ })
+ })
}
// ── content extract ───────────────────────────────────────────────────────────
diff --git a/cli/markdown_cmd.go b/cli/markdown_cmd.go
index e5b15d8..c86aea8 100644
--- a/cli/markdown_cmd.go
+++ b/cli/markdown_cmd.go
@@ -5,6 +5,7 @@ import (
"fmt"
"os"
"path/filepath"
+ "sort"
"strconv"
"strings"
"time"
@@ -34,6 +35,9 @@ type markdownExportCmd struct {
keepParquet bool // keep local parquet after commit
minFreeGB int // pause downloads below this much free disk
ledger string
+
+ lang string
+ minConf float64
}
func newMarkdownExportCmd() kit.Command {
@@ -46,15 +50,26 @@ Markdown with h2m (go-trafilatura tuned for recall plus a GFM renderer: cleaned
prose, tables, absolute links), and write each shard to a zstd-compressed Parquet
file. After each shard the Parquet is committed to a HuggingFace dataset repo.
-Output schema (open-markdown-v2):
- doc_id stable SHA-256 URL hash (16 bytes hex)
- url original page URL
- host hostname
- crawl_date WARC-Date YYYY-MM-DD
- warc_record_id WARC record ID
- html_length raw HTML body bytes before conversion
- markdown_length converted Markdown bytes
- markdown converted Markdown text
+Output schema (open-markdown-v3):
+ doc_id stable SHA-256 URL hash (16 bytes hex)
+ url original page URL
+ host hostname
+ crawl_date WARC-Date YYYY-MM-DD
+ warc_record_id WARC record ID
+ html_length raw HTML body bytes before conversion
+ markdown_length converted Markdown bytes
+ markdown converted Markdown text
+ language ISO 639-3 code detected in the Markdown, "" if too short
+ language_confidence 0 to 1, how sure the identifier is
+
+v3 appends the last two columns and changes nothing else, so a v2 reader that
+projects the columns it knows about reads a v3 file unchanged.
+
+--lang keeps only the documents identified as that language, which is a document
+level check on the text actually extracted, not the CLD2 label Common Crawl
+computed over the raw HTML. It is a coarse pre-filter: a trigram identifier that
+answers "this looks like Vietnamese" well enough to cut a corpus down, and not a
+substitute for a language specific classifier if you need one.
HF path layout:
data/crawl=CC-MAIN-YYYY-WW/NNNNNN.parquet
@@ -67,10 +82,11 @@ local files, so the slow commit round trip stays off the per-shard critical path
A ledger file records committed shards, so a killed run resumes where it stopped.
Examples:
- ccrawl markdown export --shards 0 --repo open-index/open-markdown-v2
- ccrawl markdown export --shards 0-9 --repo open-index/open-markdown-v2
+ ccrawl markdown export --shards 0 --repo open-index/open-markdown-v3
+ ccrawl markdown export --shards 0-9 --repo open-index/open-markdown-v3
ccrawl markdown export --shards all --parallel 4 --commit-batch 10
ccrawl markdown export --shards 0-99 --push=false --out ~/data/md
+ ccrawl markdown export --shards 0-9 --lang vie --min-lang-confidence 0.8
HF_TOKEN=hf_... ccrawl markdown export --shards 0 -c 2026-25`,
Flags: v.flags,
Run: v.run,
@@ -80,7 +96,7 @@ Examples:
func (v *markdownExportCmd) flags(f *kit.FlagSet) {
f.StringVar(&v.shards, "shards", "0", "shard range: N, N-M, N,M, or all")
f.StringVar(&v.outDir, "out", "", "directory for parquet files (default: /markdown)")
- f.StringVar(&v.repo, "repo", "open-index/open-markdown-v2", "HuggingFace dataset repo (org/name)")
+ f.StringVar(&v.repo, "repo", "open-index/open-markdown-v3", "HuggingFace dataset repo (org/name)")
f.IntVar(&v.workers, "workers", 0, "total conversion workers shared across shards (0 = NumCPU)")
f.IntVar(&v.limit, "limit", 0, "process at most this many shards (0 = all)")
f.BoolVar(&v.skip, "skip-errors", false, "continue past per-shard failures instead of aborting")
@@ -90,6 +106,8 @@ func (v *markdownExportCmd) flags(f *kit.FlagSet) {
f.BoolVar(&v.keepParquet, "keep-parquet", false, "keep local parquet files after they are committed")
f.IntVar(&v.minFreeGB, "min-free-gb", 2, "pause new downloads when free disk drops below this many GiB")
f.StringVar(&v.ledger, "ledger", "", "resume ledger file (default: /.committed)")
+ f.StringVar(&v.lang, "lang", "", "keep only documents detected as this ISO 639-3 language (e.g. vie)")
+ f.Float64Var(&v.minConf, "min-lang-confidence", ccrawl.DefaultMinLangConfidence, "confidence a document must clear for --lang")
}
func (v *markdownExportCmd) run(ctx context.Context, _ []string) error {
@@ -172,6 +190,9 @@ func (v *markdownExportCmd) run(ctx context.Context, _ []string) error {
MinFreeBytes: int64(v.minFreeGB) << 30,
Ledger: ledger,
Reporter: rep,
+
+ Lang: v.lang,
+ MinLangConfidence: v.minConf,
})
rep.Textf(
@@ -179,6 +200,7 @@ func (v *markdownExportCmd) run(ctx context.Context, _ []string) error {
run.Committed, run.Skipped, run.Failed, run.Total, run.Rows,
humanBytes(run.HTMLBytes), humanBytes(run.MDBytes), humanBytes(run.ParquetBytes),
run.Elapsed.Round(time.Second), run.ShardsPerHour)
+ rep.Textf("%s", langSummary(v.lang, run.LangDropped, run.LangCounts))
// Per-shard conversion failures never abort the run; they are logged and
// counted (the committer keeps draining). runErr is only set for a fatal
@@ -238,3 +260,47 @@ func parseShardRange(spec string, total int) ([]int, error) {
}
return out, nil
}
+
+// langSummary renders the language breakdown for the end of a run. Without a
+// --lang filter it is still worth printing: it says what the shard turned out to
+// be, which is the thing you want before deciding what to filter on next time.
+// The empty code covers documents with no text to identify, and it is named
+// rather than left as a blank row.
+func langSummary(want string, dropped int64, counts map[string]int64) string {
+ if len(counts) == 0 {
+ return ""
+ }
+ type pair struct {
+ code string
+ n int64
+ }
+ var total int64
+ pairs := make([]pair, 0, len(counts))
+ for code, n := range counts {
+ total += n
+ if code == "" {
+ code = "unknown"
+ }
+ pairs = append(pairs, pair{code, n})
+ }
+ sort.Slice(pairs, func(i, j int) bool {
+ if pairs[i].n != pairs[j].n {
+ return pairs[i].n > pairs[j].n
+ }
+ return pairs[i].code < pairs[j].code
+ })
+ if len(pairs) > 8 {
+ pairs = pairs[:8]
+ }
+ var b strings.Builder
+ if want != "" {
+ fmt.Fprintf(&b, "language: --lang %s kept %d of %d documents, dropped %d (%.1f%%)\n",
+ want, total-dropped, total, dropped, 100*float64(dropped)/float64(total))
+ }
+ b.WriteString("language: detected")
+ for _, p := range pairs {
+ fmt.Fprintf(&b, " %s=%d", p.code, p.n)
+ }
+ b.WriteString("\n")
+ return b.String()
+}
diff --git a/cli/markdown_refetch_cmd.go b/cli/markdown_refetch_cmd.go
index ea6b9c5..40ee511 100644
--- a/cli/markdown_refetch_cmd.go
+++ b/cli/markdown_refetch_cmd.go
@@ -44,6 +44,8 @@ type markdownRefetchCmd struct {
warcCacheDir string // where to cache downloaded WARC shards ("" = default)
noWARCCache bool // disable WARC caching entirely
fetchOnly bool // store raw HTML, skip the convert phase
+ lang string
+ minConf float64
}
func newMarkdownRefetchCmd() kit.Command {
@@ -110,6 +112,8 @@ func (v *markdownRefetchCmd) flags(f *kit.FlagSet) {
f.StringVar(&v.ledger, "ledger", "", "resume ledger file (default: /.committed)")
f.StringVar(&v.warcCacheDir, "warc-cache-dir", "", "cache downloaded WARC shards here (default: /ami/warc)")
f.BoolVar(&v.noWARCCache, "no-warc-cache", false, "do not cache downloaded WARC shards to disk")
+ f.StringVar(&v.lang, "lang", "", "keep only documents detected as this ISO 639-3 language (e.g. vie)")
+ f.Float64Var(&v.minConf, "min-lang-confidence", ccrawl.DefaultMinLangConfidence, "confidence a document must clear for --lang")
f.BoolVar(&v.fetchOnly, "fetch-only", false, "store raw HTML and skip the convert phase, so fetch runs at full speed (convert offline later over the html column)")
}
@@ -232,6 +236,12 @@ func (v *markdownRefetchCmd) run(ctx context.Context, _ []string) error {
if v.fetchOnly {
fmt.Fprintf(os.Stderr, "refetch: fetch-only mode, storing raw HTML and skipping convert\n")
}
+ // Detection runs on the extracted Markdown, and --fetch-only never extracts
+ // any, so the two together would drop every row and report a filter working
+ // perfectly. Refuse instead.
+ if v.lang != "" && v.fetchOnly {
+ return usageErr("lang cannot be used with fetch-only: the language is detected in the converted Markdown, which fetch-only does not produce")
+ }
// The journal lands beside the ledger, so a resumed run's events sit next to
// the record of what it resumed from.
@@ -261,6 +271,9 @@ func (v *markdownRefetchCmd) run(ctx context.Context, _ []string) error {
CacheDir: warcCacheDir,
FetchOnly: v.fetchOnly,
Reporter: rep,
+
+ Lang: v.lang,
+ MinLangConfidence: v.minConf,
})
n := int64(run.Committed)
@@ -287,6 +300,7 @@ func (v *markdownRefetchCmd) run(ctx context.Context, _ []string) error {
"failures: %d total | dns=%d timeout=%d refused=%d skip=%d other=%d\n",
run.Failures, run.ErrDNS, run.ErrTimeout, run.ErrRefused, run.ErrSkip, run.ErrOther)
}
+ rep.Textf("%s", langSummary(v.lang, run.LangDropped, run.LangCounts))
return runErr
}
diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md
index 96d0670..2fb35b0 100644
--- a/docs/content/reference/cli.md
+++ b/docs/content/reference/cli.md
@@ -224,13 +224,33 @@ Unlike `extract`, these commands use the v2 crawler config (10 MB body limit, br
| `content extract ` | Clean text, title, description, canonical URL, language, word count |
| `content outlinks ` | Structured outbound links with anchor text |
| `content quality ` | Quality signals: word count, spam score, parked detection, short-content flag |
+| `content lang ` | The language the markdown pipelines would detect, with the confidence and the text it judged |
```sh
ccrawl content extract https://golang.org/
ccrawl content quality https://example.com/ -o json
ccrawl content outlinks https://news.ycombinator.com/ -n 20
+ccrawl content lang https://vnexpress.net/ -o json
```
+### content lang
+
+`content lang` runs the same identifier `markdown export --lang` applies, on one URL at a time, so a document that was kept or dropped can be asked about directly.
+
+```
+$ ccrawl content lang https://vnexpress.net/ -o json
+{
+ "url": "https://vnexpress.net/",
+ "language": "vie",
+ "confidence": 1,
+ "cc_language": "vi",
+ "chars": 24196,
+ "sample": "Báo tiếng Việt nhiều người xem nhất Thứ hai, 3/8/2026 ..."
+}
+```
+
+`language` is detected in the extracted Markdown, not in the raw HTML and not read off the page's own `lang` attribute, because the Markdown is what the pipelines keep and filter on. `cc_language` is what the page declares, printed next to it so a disagreement is visible instead of silent. `sample` is the text the identifier actually saw, truncated; when an answer looks wrong it is almost always the input that is wrong.
+
---
## news
@@ -288,6 +308,33 @@ ccrawl markdown refetch --shards 0-9 --fetch-workers 400 --repo org/name
Both take `--shards` (`N`, `N-M`, `N,M`, or `all`), resume from a ledger at `/.committed`, and need `HF_TOKEN` unless `--push=false`.
The [markdown reference](/reference/markdown/) has both schemas, the HuggingFace path layout, and the tuning flags.
+### Language filtering
+
+Both subcommands label every row with a detected language and can keep only one of them.
+
+| Flag | Default | Does |
+|---|---|---|
+| `--lang` | (off) | Keep only documents detected as this ISO 639-3 language, for example `vie` |
+| `--min-lang-confidence` | `0.8` | Confidence a document has to clear for `--lang` |
+
+```sh
+ccrawl markdown export --shards 0 --lang vie --push=false --out ./md
+ccrawl markdown export --shards 0 --lang vie --min-lang-confidence 0.9 --push=false --out ./md
+```
+
+The language is detected in the extracted Markdown, so it describes the text in the row rather than whatever the page declared. Without `--lang` nothing is dropped and every row still carries `language` and `language_confidence`, which is what makes an unfiltered shard filterable later without extracting it again.
+
+A document with too little text to identify is dropped by `--lang` rather than kept: a filtered export asks for documents known to be in one language, and "we could not tell" is not that. The run prints both the drop rate and the detected mix, so a filter that threw away more than expected says so:
+
+```
+language: --lang vie kept 21482 of 43110 documents, dropped 21628 (50.2%)
+language: detected vie=21482 eng=17903 unknown=2011 zho=884 ...
+```
+
+`--lang` cannot be combined with `markdown refetch --fetch-only`, since fetch-only never produces the Markdown the identifier reads.
+
+This is a coarse pre-filter. A trigram identifier tells Vietnamese from Malay well enough to cut a corpus down to something worth looking at, and it is not a substitute for a language specific classifier.
+
---
## rank
diff --git a/docs/content/reference/markdown.md b/docs/content/reference/markdown.md
index 5ed87a8..42ccc83 100644
--- a/docs/content/reference/markdown.md
+++ b/docs/content/reference/markdown.md
@@ -15,7 +15,7 @@ There are two pipelines and they answer different questions.
Both walk the same unit of work: one WARC file from a crawl's manifest is one shard, one shard becomes one Parquet file, and one Parquet file lands at a fixed path in the repo.
```sh
-ccrawl markdown export --shards 0 --repo open-index/open-markdown-v2
+ccrawl markdown export --shards 0 --repo open-index/open-markdown-v3
ccrawl markdown refetch --shards 0 --repo open-index/open-markdown-refetch-v1
```
@@ -39,7 +39,7 @@ An out-of-range index is a usage error, not a silent skip.
## Output schemas
-### open-markdown-v2, written by `markdown export`
+### open-markdown-v3, written by `markdown export`
| Column | Type | Meaning |
|---|---|---|
@@ -51,6 +51,11 @@ An out-of-range index is a usage error, not a silent skip.
| `html_length` | int | raw HTML body bytes before conversion |
| `markdown_length` | int | converted Markdown bytes |
| `markdown` | string | the converted Markdown |
+| `language` | string | ISO 639-3 language detected in the Markdown, empty when there was too little text |
+| `language_confidence` | double | the identifier's confidence, 0 to 1 |
+
+v3 appends the last two columns and changes nothing else, so a reader written against v2 loads a v3 file and gets every column it asks for.
+The default repo moved to `open-index/open-markdown-v3` to keep the two schemas in separate repos.
### open-markdown-refetch-v1, written by `markdown refetch`
@@ -80,6 +85,8 @@ Everything the export schema has, plus the live response and its timings.
| `html_length` | int | HTML body bytes, set only when the response is HTML |
| `markdown_length` | int | converted Markdown bytes |
| `markdown` | string | the converted Markdown |
+| `language` | string | ISO 639-3 language detected in the Markdown |
+| `language_confidence` | double | the identifier's confidence, 0 to 1 |
| `error` | string | fetch error, empty on success |
A failed fetch still produces a row.
@@ -139,6 +146,49 @@ They exit 0 on success and 1 on a fatal error, and you re-run the same command t
`urls publish` and `domains publish` are the commands that exit 75, because those runs commit into a single growing dataset and have a stall clock.
See [exit codes](/reference/exit-codes/) for the full contract.
+## Language filtering
+
+Every row is labelled with a detected language, and `--lang` keeps only one of them.
+
+```sh
+ccrawl markdown export --shards 0-9 --lang vie --push=false --out ./md
+ccrawl markdown refetch --shards 0-9 --lang vie --min-lang-confidence 0.9
+```
+
+The identifier reads the extracted Markdown, not the raw HTML and not the page's own `lang` attribute.
+That is the whole point of doing it here: the columnar index already carries CLD2 labels computed by Common Crawl over the raw HTML, and those labels are unreliable for low resource languages and missing outright on a good share of rows.
+Running the identifier over the Markdown asks the question about the text that ends up in the dataset.
+
+`--min-lang-confidence` defaults to 0.8, which is the identifier's own reliability threshold.
+Raise it when precision matters more than volume.
+
+Documents with too little text to identify come out with an empty `language`, and a `--lang` filter drops them.
+A filtered export asks for documents known to be in one language, and "we could not tell" is not that.
+Without `--lang` nothing is dropped at all and the columns are still filled in, so an unfiltered shard can be filtered later without extracting it again.
+
+Both commands report the drop rate and the detected mix at the end of the run.
+Three shards of `CC-MAIN-2026-30` filtered to Vietnamese look like this:
+
+```
+markdown: 3 committed, 0 skipped, 0 failed of 3 | 647 rows | html=110.1 MB md=3.7 MB parquet=1.5 MB | 4m46s elapsed
+language: --lang vie kept 647 of 62824 documents, dropped 62177 (99.0%)
+language: detected eng=24266 fra=4165 deu=3915 rus=3786 spa=3388 cmn=2984 jpn=2702 por=1814
+```
+
+Those two lines only print in text mode, so the same numbers ride on the journal events as `lang_dropped` and `lang_counts` for a run whose progress is JSON.
+The breakdown is worth reading even when you did not filter, because it tells you what the crawl actually holds: Vietnamese is roughly 1 percent of a general Common Crawl shard, so a `--lang vie` run throws away 99 percent of it by design.
+
+The failure mode to know about is mojibake.
+A page served as UTF-8 but labelled Latin-1 comes out of extraction as garbled text, and garbled Cyrillic in particular carries enough accented Latin characters to read as Vietnamese.
+In a 200 document manual sample of the run above, 199 were Vietnamese and the one miss was a mojibake Ukrainian page.
+Romanized Vietnamese written without tone marks is detected as Vietnamese, which is usually what you want and worth knowing if it is not.
+
+Two limits worth knowing before you rely on it.
+`--lang` cannot be combined with `markdown refetch --fetch-only`, because fetch-only never produces the Markdown the identifier reads, and asking for both is a usage error rather than a silently ignored flag.
+And this is a coarse pre-filter: a trigram identifier separates Vietnamese from Malay well enough to cut a corpus down to something worth looking at, and it is not a substitute for a language specific classifier.
+
+`ccrawl content lang ` runs the same identifier on a single URL and prints the text it judged, which is the way to find out why a page was kept or dropped.
+
## Tuning: parallel, workers, commit-batch
Three flags set throughput, and they control three different resources.
@@ -212,6 +262,8 @@ Shared by `markdown export` and `markdown refetch`:
| `--keep-parquet` | Keep local Parquet after it commits |
| `--min-free-gb` | Pause new downloads below this much free disk |
| `--ledger` | Resume ledger path (default `/.committed`) |
+| `--lang` | Keep only documents detected as this ISO 639-3 language, for example `vie` |
+| `--min-lang-confidence` | Confidence a document has to clear for `--lang` (default 0.8) |
`markdown export` only: