From dce3aadddc1dafeb059586ae49dc9d6c8ecf6072 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:47:48 +0700 Subject: [PATCH] Make crawls info an operation so -o works crawls info was an escape hatch that printed its own text, which meant -o was accepted and ignored: asking for jsonl or csv got the same key: value block either way. It was also a worse duplicate of stats, counting five kinds where stats counted five different ones, so the same question got two answers depending on which name you typed. Both names now register as the same operation over one count closure and one statKinds list, so the renderer is back in the path and the kinds match by construction. crawls info keeps the optional positional crawl it has always taken and gains --kinds; stats keeps reading -c. A kind whose manifest cannot be fetched still comes back as a row with files of -1 rather than being dropped. --- cli/cmd_test.go | 51 +++++++++++++++++++++++++-- cli/crawls.go | 28 --------------- cli/ops.go | 65 +++++++++++++++++++++++++++-------- cli/testdata/flags.golden | 2 +- docs/content/reference/cli.md | 7 +++- 5 files changed, 107 insertions(+), 46 deletions(-) diff --git a/cli/cmd_test.go b/cli/cmd_test.go index c184116..1d58ac1 100644 --- a/cli/cmd_test.go +++ b/cli/cmd_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "slices" "strings" "testing" @@ -389,15 +390,61 @@ func TestUnknownCommandAndFlagExitOne(t *testing.T) { } // crawls info walks every manifest a crawl publishes, so it is the one command -// that fails if a kind stops resolving. +// that fails if a kind stops resolving. The positional crawl is checked here +// too: it predates the move to an operation and scripts pass it. func TestCrawlsInfoCountsEveryKind(t *testing.T) { r := run(t, "crawls", "info", "2026-30").wantCode(t, 0) r.wantOut(t, "CC-MAIN-2026-30", "warc", "wet", "cc-index-table") - if strings.Contains(r.Out, "(unavailable)") { + if strings.Contains(r.Out, "-1") { t.Fatalf("a manifest did not resolve:\n%s", r.Out) } } +// TestCrawlsInfoRendersInEveryFormat is #140. It used to write its own text with +// fmt.Fprintf, so -o was accepted and ignored and a pipeline asking for JSONL +// got a paragraph and exit 0. +func TestCrawlsInfoRendersInEveryFormat(t *testing.T) { + jsonl := run(t, "crawls", "info", "2026-30", "-o", "jsonl").wantCode(t, 0) + for _, line := range jsonl.Lines() { + var row struct { + Crawl string `json:"crawl"` + Kind string `json:"kind"` + Files int `json:"files"` + } + if err := json.Unmarshal([]byte(line), &row); err != nil { + t.Fatalf("-o jsonl did not write JSON: %v\n%s", err, line) + } + if row.Crawl != "CC-MAIN-2026-30" || row.Kind == "" { + t.Fatalf("row is missing fields: %s", line) + } + } + + csv := run(t, "crawls", "info", "2026-30", "-o", "csv").wantCode(t, 0) + if !strings.HasPrefix(csv.Out, "crawl,kind,files") { + t.Fatalf("-o csv did not write a CSV header:\n%s", csv.Out) + } +} + +// TestCrawlsInfoAndStatsCountTheSameKinds pins the two spellings together. They +// used to carry a default kind list each, differing by one kind in each +// direction, so the same question answered differently depending on which name +// you typed. +func TestCrawlsInfoAndStatsCountTheSameKinds(t *testing.T) { + kinds := func(args ...string) []string { + r := run(t, append(args, "-o", "csv")...).wantCode(t, 0) + var out []string + for _, line := range r.Lines()[1:] { + out = append(out, strings.Split(line, ",")[1]) + } + return out + } + info := kinds("crawls", "info", "2026-30") + stats := kinds("stats", "-c", "2026-30") + if !slices.Equal(info, stats) { + t.Fatalf("crawls info counts %v, stats counts %v", info, stats) + } +} + // get -O writes the body to a file rather than stdout, and --at picks a capture // by date instead of taking the newest. Both are one line of wiring each and // both are silently wrong if that line is missing: the file is empty, or the diff --git a/cli/crawls.go b/cli/crawls.go index 47524ed..0092167 100644 --- a/cli/crawls.go +++ b/cli/crawls.go @@ -24,12 +24,6 @@ func crawlsEscapeHatches() []kit.Command { Args: kit.ExactArgs(1), Run: runCrawlsResolve, }, - { - Use: "info ", - Short: "Show details for a crawl (file counts per format)", - Args: kit.MaximumNArgs(1), - Run: runCrawlsInfo, - }, } } @@ -55,25 +49,3 @@ func runCrawlsResolve(ctx context.Context, args []string) error { _, _ = fmt.Fprintln(cmdOut, id) return nil } - -func runCrawlsInfo(ctx context.Context, args []string) error { - app := appFromCtx(ctx) - ref := app.Cfg.CrawlID - if len(args) == 1 { - ref = args[0] - } - id, err := ccrawl.ResolveCrawl(ctx, app.HTTP, app.Cache, ref) - if err != nil { - return err - } - _, _ = fmt.Fprintf(cmdOut, "Crawl: %s\n", id) - for _, kind := range []string{"warc", "wat", "wet", "robotstxt", "cc-index-table"} { - paths, err := ccrawl.FetchPaths(ctx, app.HTTP, app.Cache, id, kind) - if err != nil { - _, _ = fmt.Fprintf(cmdOut, " %-16s (unavailable)\n", kind) - continue - } - _, _ = fmt.Fprintf(cmdOut, " %-16s %d files\n", kind, len(paths)) - } - return nil -} diff --git a/cli/ops.go b/cli/ops.go index 4caacd0..91ae327 100644 --- a/cli/ops.go +++ b/cli/ops.go @@ -533,33 +533,46 @@ type statRow struct { type statsIn struct { App *App `kit:"inject"` - Kinds []string `kit:"flag" help:"archive kinds to count (default warc,wat,wet,robotstxt,non200responses)"` + Kinds []string `kit:"flag" help:"archive kinds to count (default warc,wat,wet,robotstxt,non200responses,cc-index-table)"` +} + +// statKinds are the manifests counted when --kinds says nothing. One list for +// both spellings of this command: stats and crawls info used to carry a default +// each, differing by one kind in each direction, which meant the same question +// got two answers depending on which name you happened to type. +var statKinds = []string{"warc", "wat", "wet", "robotstxt", "non200responses", "cc-index-table"} + +// crawlsInfoIn is statsIn plus the positional crawl that spelling has always +// accepted. +type crawlsInfoIn struct { + App *App `kit:"inject"` + ID string `kit:"arg" help:"crawl reference, defaults to -c"` + Kinds []string `kit:"flag" help:"archive kinds to count (default warc,wat,wet,robotstxt,non200responses,cc-index-table)"` } func registerStats(app *kit.App) { - handle(app, kit.OpMeta{ - Name: "stats", - Group: "read", - Summary: "Show the shape of a crawl: file counts per archive kind", - Long: `Summarise a crawl by counting the files in each published manifest (warc, wat, -wet, robotstxt, non200responses). This reads the small *.paths.gz manifests, not -the archives themselves, so it is quick and cheap. + long := `Summarise a crawl by counting the files in each published manifest (warc, wat, +wet, robotstxt, non200responses, cc-index-table). This reads the small *.paths.gz +manifests, not the archives themselves, so it is quick and cheap. Examples: ccrawl stats the latest crawl ccrawl stats -c 2024-51 a specific crawl - ccrawl stats --kinds warc,wet`, - }, func(ctx context.Context, in statsIn, emit func(statRow) error) error { - id, err := in.App.Crawl(ctx) + ccrawl stats --kinds warc,wet` + + count := func(ctx context.Context, app *App, ref string, kinds []string, emit func(statRow) error) error { + id, err := app.Crawl(ctx) + if ref != "" { + id, err = ccrawl.ResolveCrawl(ctx, app.HTTP, app.Cache, ref) + } if err != nil { return err } - kinds := in.Kinds if len(kinds) == 0 { - kinds = []string{"warc", "wat", "wet", "robotstxt", "non200responses"} + kinds = statKinds } for _, kind := range kinds { - paths, err := ccrawl.FetchPaths(ctx, in.App.HTTP, in.App.Cache, id, kind) + paths, err := ccrawl.FetchPaths(ctx, app.HTTP, app.Cache, id, kind) if err != nil { if err := emit(statRow{Crawl: id, Kind: kind, Files: -1}); err != nil { return err @@ -571,5 +584,29 @@ Examples: } } return nil + } + + handle(app, kit.OpMeta{ + Name: "stats", + Group: "read", + Summary: "Show the shape of a crawl: file counts per archive kind", + Long: long, + }, func(ctx context.Context, in statsIn, emit func(statRow) error) error { + return count(ctx, in.App, "", in.Kinds, emit) + }) + + // crawls info is the same question asked from the crawls parent. It used to + // be an escape-hatch command printing its own text, which meant -o was + // accepted and ignored; registering it as an operation is what puts the + // renderer back in the path. The positional crawl it has always taken stays, + // so a command someone has in a script keeps working. + handle(app, kit.OpMeta{ + Name: "info", + Parent: "crawls", + Summary: "Show details for a crawl: file counts per archive kind", + Long: long, + Args: []kit.Arg{{Name: "id", Help: "crawl reference, defaults to -c", Optional: true}}, + }, func(ctx context.Context, in crawlsInfoIn, emit func(statRow) error) error { + return count(ctx, in.App, in.ID, in.Kinds, emit) }) } diff --git a/cli/testdata/flags.golden b/cli/testdata/flags.golden index 6283b5e..b10d72a 100644 --- a/cli/testdata/flags.golden +++ b/cli/testdata/flags.golden @@ -57,7 +57,7 @@ ccrawl crawl status ccrawl crawls ccrawl crawls info - + --kinds ccrawl crawls latest ccrawl crawls list diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md index 600e9e2..58f3d16 100644 --- a/docs/content/reference/cli.md +++ b/docs/content/reference/cli.md @@ -56,7 +56,12 @@ Run `ccrawl --help` for the full flag list on any command. | `crawls list` | List the monthly crawls, newest first | | `crawls latest` | Print the newest crawl ID | | `crawls resolve ` | Resolve a year or `latest` to a crawl ID | -| `crawls info ` | File counts per archive kind for a crawl | +| `crawls info [id]` | File counts per archive kind for a crawl | + +`crawls info` and `stats` are the same command under two names, so they count the same kinds and answer with the same rows: `crawl`, `kind`, `files`. +Both take `--kinds` to narrow the list, and both honour `-o`, so `crawls info -o csv` writes CSV the way every other read command does. +The one difference is how you name the crawl: `crawls info` takes it as a positional argument, `stats` reads `-c`, and either one falls back to the configured crawl when you leave it out. +A kind whose manifest could not be fetched comes back with `files` of `-1` rather than being dropped, so a row is never silently missing. ---