Make crawls info an operation so -o works

cli/cmd_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 
@@ -389,15 +390,61 @@ func TestUnknownCommandAndFlagExitOne(t *testing.T) {
 }
 
 // crawls info walks every manifest a crawl publishes, so it is the one command
-// that fails if a kind stops resolving.
+// that fails if a kind stops resolving. The positional crawl is checked here
+// too: it predates the move to an operation and scripts pass it.
 func TestCrawlsInfoCountsEveryKind(t *testing.T) {
 	r := run(t, "crawls", "info", "2026-30").wantCode(t, 0)
 	r.wantOut(t, "CC-MAIN-2026-30", "warc", "wet", "cc-index-table")
-	if strings.Contains(r.Out, "(unavailable)") {
+	if strings.Contains(r.Out, "-1") {
 		t.Fatalf("a manifest did not resolve:\n%s", r.Out)
 	}
 }
 
+// TestCrawlsInfoRendersInEveryFormat is #140. It used to write its own text with
+// fmt.Fprintf, so -o was accepted and ignored and a pipeline asking for JSONL
+// got a paragraph and exit 0.
+func TestCrawlsInfoRendersInEveryFormat(t *testing.T) {
+	jsonl := run(t, "crawls", "info", "2026-30", "-o", "jsonl").wantCode(t, 0)
+	for _, line := range jsonl.Lines() {
+		var row struct {
+			Crawl string `json:"crawl"`
+			Kind  string `json:"kind"`
+			Files int    `json:"files"`
+		}
+		if err := json.Unmarshal([]byte(line), &row); err != nil {
+			t.Fatalf("-o jsonl did not write JSON: %v\n%s", err, line)
+		}
+		if row.Crawl != "CC-MAIN-2026-30" || row.Kind == "" {
+			t.Fatalf("row is missing fields: %s", line)
+		}
+	}
+
+	csv := run(t, "crawls", "info", "2026-30", "-o", "csv").wantCode(t, 0)
+	if !strings.HasPrefix(csv.Out, "crawl,kind,files") {
+		t.Fatalf("-o csv did not write a CSV header:\n%s", csv.Out)
+	}
+}
+
+// TestCrawlsInfoAndStatsCountTheSameKinds pins the two spellings together. They
+// used to carry a default kind list each, differing by one kind in each
+// direction, so the same question answered differently depending on which name
+// you typed.
+func TestCrawlsInfoAndStatsCountTheSameKinds(t *testing.T) {
+	kinds := func(args ...string) []string {
+		r := run(t, append(args, "-o", "csv")...).wantCode(t, 0)
+		var out []string
+		for _, line := range r.Lines()[1:] {
+			out = append(out, strings.Split(line, ",")[1])
+		}
+		return out
+	}
+	info := kinds("crawls", "info", "2026-30")
+	stats := kinds("stats", "-c", "2026-30")
+	if !slices.Equal(info, stats) {
+		t.Fatalf("crawls info counts %v, stats counts %v", info, stats)
+	}
+}
+
 // get -O writes the body to a file rather than stdout, and --at picks a capture
 // by date instead of taking the newest. Both are one line of wiring each and
 // both are silently wrong if that line is missing: the file is empty, or the

cli/crawls.go

@@ -24,12 +24,6 @@ func crawlsEscapeHatches() []kit.Command {
 			Args:  kit.ExactArgs(1),
 			Run:   runCrawlsResolve,
 		},
-		{
-			Use:   "info <id>",
-			Short: "Show details for a crawl (file counts per format)",
-			Args:  kit.MaximumNArgs(1),
-			Run:   runCrawlsInfo,
-		},
 	}
 }
 
@@ -55,25 +49,3 @@ func runCrawlsResolve(ctx context.Context, args []string) error {
 	_, _ = fmt.Fprintln(cmdOut, id)
 	return nil
 }
-
-func runCrawlsInfo(ctx context.Context, args []string) error {
-	app := appFromCtx(ctx)
-	ref := app.Cfg.CrawlID
-	if len(args) == 1 {
-		ref = args[0]
-	}
-	id, err := ccrawl.ResolveCrawl(ctx, app.HTTP, app.Cache, ref)
-	if err != nil {
-		return err
-	}
-	_, _ = fmt.Fprintf(cmdOut, "Crawl: %s\n", id)
-	for _, kind := range []string{"warc", "wat", "wet", "robotstxt", "cc-index-table"} {
-		paths, err := ccrawl.FetchPaths(ctx, app.HTTP, app.Cache, id, kind)
-		if err != nil {
-			_, _ = fmt.Fprintf(cmdOut, "  %-16s (unavailable)\n", kind)
-			continue
-		}
-		_, _ = fmt.Fprintf(cmdOut, "  %-16s %d files\n", kind, len(paths))
-	}
-	return nil
-}

cli/ops.go

@@ -533,33 +533,46 @@ type statRow struct {
 
 type statsIn struct {
 	App   *App     `kit:"inject"`
-	Kinds []string `kit:"flag" help:"archive kinds to count (default warc,wat,wet,robotstxt,non200responses)"`
+	Kinds []string `kit:"flag" help:"archive kinds to count (default warc,wat,wet,robotstxt,non200responses,cc-index-table)"`
+}
+
+// statKinds are the manifests counted when --kinds says nothing. One list for
+// both spellings of this command: stats and crawls info used to carry a default
+// each, differing by one kind in each direction, which meant the same question
+// got two answers depending on which name you happened to type.
+var statKinds = []string{"warc", "wat", "wet", "robotstxt", "non200responses", "cc-index-table"}
+
+// crawlsInfoIn is statsIn plus the positional crawl that spelling has always
+// accepted.
+type crawlsInfoIn struct {
+	App   *App     `kit:"inject"`
+	ID    string   `kit:"arg" help:"crawl reference, defaults to -c"`
+	Kinds []string `kit:"flag" help:"archive kinds to count (default warc,wat,wet,robotstxt,non200responses,cc-index-table)"`
 }
 
 func registerStats(app *kit.App) {
-	handle(app, kit.OpMeta{
-		Name:    "stats",
-		Group:   "read",
-		Summary: "Show the shape of a crawl: file counts per archive kind",
-		Long: `Summarise a crawl by counting the files in each published manifest (warc, wat,
-wet, robotstxt, non200responses). This reads the small *.paths.gz manifests, not
-the archives themselves, so it is quick and cheap.
+	long := `Summarise a crawl by counting the files in each published manifest (warc, wat,
+wet, robotstxt, non200responses, cc-index-table). This reads the small *.paths.gz
+manifests, not the archives themselves, so it is quick and cheap.
 
 Examples:
   ccrawl stats                 the latest crawl
   ccrawl stats -c 2024-51      a specific crawl
-  ccrawl stats --kinds warc,wet`,
-	}, func(ctx context.Context, in statsIn, emit func(statRow) error) error {
-		id, err := in.App.Crawl(ctx)
+  ccrawl stats --kinds warc,wet`
+
+	count := func(ctx context.Context, app *App, ref string, kinds []string, emit func(statRow) error) error {
+		id, err := app.Crawl(ctx)
+		if ref != "" {
+			id, err = ccrawl.ResolveCrawl(ctx, app.HTTP, app.Cache, ref)
+		}
 		if err != nil {
 			return err
 		}
-		kinds := in.Kinds
 		if len(kinds) == 0 {
-			kinds = []string{"warc", "wat", "wet", "robotstxt", "non200responses"}
+			kinds = statKinds
 		}
 		for _, kind := range kinds {
-			paths, err := ccrawl.FetchPaths(ctx, in.App.HTTP, in.App.Cache, id, kind)
+			paths, err := ccrawl.FetchPaths(ctx, app.HTTP, app.Cache, id, kind)
 			if err != nil {
 				if err := emit(statRow{Crawl: id, Kind: kind, Files: -1}); err != nil {
 					return err
@@ -571,5 +584,29 @@ Examples:
 			}
 		}
 		return nil
+	}
+
+	handle(app, kit.OpMeta{
+		Name:    "stats",
+		Group:   "read",
+		Summary: "Show the shape of a crawl: file counts per archive kind",
+		Long:    long,
+	}, func(ctx context.Context, in statsIn, emit func(statRow) error) error {
+		return count(ctx, in.App, "", in.Kinds, emit)
+	})
+
+	// crawls info is the same question asked from the crawls parent. It used to
+	// be an escape-hatch command printing its own text, which meant -o was
+	// accepted and ignored; registering it as an operation is what puts the
+	// renderer back in the path. The positional crawl it has always taken stays,
+	// so a command someone has in a script keeps working.
+	handle(app, kit.OpMeta{
+		Name:    "info",
+		Parent:  "crawls",
+		Summary: "Show details for a crawl: file counts per archive kind",
+		Long:    long,
+		Args:    []kit.Arg{{Name: "id", Help: "crawl reference, defaults to -c", Optional: true}},
+	}, func(ctx context.Context, in crawlsInfoIn, emit func(statRow) error) error {
+		return count(ctx, in.App, in.ID, in.Kinds, emit)
 	})
 }

cli/testdata/flags.golden

@@ -57,7 +57,7 @@ ccrawl crawl status
 ccrawl crawls
 
 ccrawl crawls info
-
+	--kinds
 ccrawl crawls latest
 
 ccrawl crawls list

docs/content/reference/cli.md

@@ -56,7 +56,12 @@ Run `ccrawl <command> --help` for the full flag list on any command.
 | `crawls list` | List the monthly crawls, newest first |
 | `crawls latest` | Print the newest crawl ID |
 | `crawls resolve <ref>` | Resolve a year or `latest` to a crawl ID |
-| `crawls info <id>` | File counts per archive kind for a crawl |
+| `crawls info [id]` | File counts per archive kind for a crawl |
+
+`crawls info` and `stats` are the same command under two names, so they count the same kinds and answer with the same rows: `crawl`, `kind`, `files`.
+Both take `--kinds` to narrow the list, and both honour `-o`, so `crawls info -o csv` writes CSV the way every other read command does.
+The one difference is how you name the crawl: `crawls info` takes it as a positional argument, `stats` reads `-c`, and either one falls back to the configured crawl when you leave it out.
+A kind whose manifest could not be fetched comes back with `files` of `-1` rather than being dropped, so a row is never silently missing.
 
 ---