diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 00000000..e0561dc3 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,33 @@ +# The two trees below are what scripts/build_dist.py generates from .agents/skills/ and CI holds current, so a finding in either belongs at its source and a review of every copy is one finding three times. +reviews: + # Auto review covers only the default branch unless a base is listed here, each entry a regex, and the default branch stays included whatever is listed. + auto_review: + enabled: true + base_branches: + - "^develop$" + # A fleet pull request routinely passes five pushes before it merges, and the default pauses after five reviewed commits, which reads as a reviewer that stopped. + auto_pause_after_reviewed_commits: 0 + path_filters: + - "!.github/skills/**" + - "!.claude-plugin/fleet-skills/**" + # Canonical prose is linted for style in CI and read whole by the local review pass, so a review comment on it earns its place only by naming something false or unfollowable. + path_instructions: + - path: "**/*.md" + instructions: | + Report a claim about a tool, a path, a command, or another rule only where it is false, stale, or unverifiable from the repository, and an instruction only where following it literally fails. + Do not report wording, tone, or formatting, which CI lints. + # The walkthrough extras below add nothing the fleet's review loop reads. + sequence_diagrams: false + suggested_labels: false + suggested_reviewers: false + in_progress_fortune: false + # CI runs these four linters and fails the pull request on them, so a review comment from the same tool is a second copy of a red check. + tools: + markdownlint: + enabled: false + actionlint: + enabled: false + shellcheck: + enabled: false + ruff: + enabled: false diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index e69b1dea..f4827680 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -24,6 +24,12 @@ Follow the fidelity declared for the file. A byte-locked reference to shared inf this repository does not carry is intentional, not a broken link. Raise substantive defects in canonical content, but locate the fix at its canonical source instead of proposing a local edit. +`.github/skills/`, and in the hub `.claude-plugin/fleet-skills/`, are generated by the hub's +`scripts/build_dist.py` from its `.agents/skills/`, so a defect in either is fixed in the source or +the generator and never in the copy. Post no review comment on a file under those trees. When the +pull request changes the file the fix belongs in, comment on that file instead, and otherwise state +the finding in the review summary. + ## GitHub Copilot Review Runbook For every review: diff --git a/.pr_agent.toml b/.pr_agent.toml new file mode 100644 index 00000000..8e0fda2e --- /dev/null +++ b/.pr_agent.toml @@ -0,0 +1,20 @@ +# The two trees below are what scripts/build_dist.py generates from .agents/skills/ and CI holds current, so a finding in either belongs at its source and a review of every copy is one finding three times. +[ignore] +glob = ['.github/skills/**', '.claude-plugin/fleet-skills/**'] + +[review_agent] +# A crash claimed against a name the build accepts was disproven by running the build, so a claimed failure carries its reproduction or is not a finding. +issues_user_guidelines = "Report a crash, an exception, or a failing path only after reproducing it against the changed code, and state the reproduction in the finding." +# A rule finding is answered against the rule's own sentence, and a rule against text the pull request did not change is a note rather than a thread. +compliance_user_guidelines = "Quote the repository rule's own sentence in the finding, and report a rule against text this pull request did not change in the summary only." +# An informational finding opened a thread the merge ruleset then required resolved, which is a merge block for a note. +comments_routing_preset = "custom" + +[review_agent.comments_routing] +action_required = "both" +remediation_recommended = "both" +informational = "summary" + +[review_agent_ux] +# The severity badge is an image tag on the finding's title line, which hides the title from a text matcher. +use_images_and_animations = false diff --git a/docs/pr-reviewer-evaluation.md b/docs/pr-reviewer-evaluation.md index a93f2269..1c5f8e11 100644 --- a/docs/pr-reviewer-evaluation.md +++ b/docs/pr-reviewer-evaluation.md @@ -16,13 +16,16 @@ This document measures whether additional automated reviewers improve the fleet' ## Status -**State:** Active evaluation\ +**State:** Active evaluation, on public repositories only\ **Incumbent:** GitHub Copilot\ -**Candidates:** CodeRabbit and Qodo\ +**Candidates:** CodeRabbit and Qodo, each on its open-source tier\ +**Installed but unconfigured:** the Claude GitHub App\ **Samples:** [ProjectTemplate pull request #891][pr-891], [pull request #892][pr-892], and [pull request #893][pr-893] No candidate is a required reviewer. A candidate remains advisory until it meets the first-class support criteria below. +As of September 2026 both candidates run on their open-source tiers. CodeRabbit and Qodo review the maintainer's public repositories and never a private one, so a private repository has Copilot as its only pull request reviewer, and Copilot's own review budget, a self-configured premium request cap, runs out under concurrent pull requests. CodeRabbit's open-source tier auto-reviews only a repository with at least ten stars, so on `ProjectTemplate`, which holds fewer, it reviews only on an explicit trigger. The Claude GitHub App is installed on the account and is not configured as a reviewer, since the `local-strict-review` Skill already runs a review pass before every push toward a pull request, so the App's value is unmeasured. + ## Evaluation Method Each finding receives one disposition after verification against the current head, repository rules, and relevant primary documentation. @@ -126,9 +129,9 @@ The current weakness is availability. A terminal error can leave the required re The review body provides an actionable summary and links each finding to an inline thread. This makes manual triage straightforward. -Automatic review skipped a feature-to-`develop` pull request because `develop` is not the repository default. The review loop must explicitly trigger CodeRabbit unless its configuration changes. +Automatic review skips a pull request whose base is not the repository default unless [`.coderabbit.yaml`][coderabbit-auto-review] lists the base under `reviews.auto_review.base_branches`, which the hub's file does for `develop`. The default branch is always included, and each entry is a regex. On the open-source tier, automatic review also needs the repository to hold at least ten stars, which this one does not, so a review here is triggered by commenting `@coderabbitai review`, and until then CodeRabbit's summary comment says so in place of a review while its status check reports success. -Incremental follow-up also requires an explicit command on this pull request. Completion is reported by updating the command reply rather than by creating a new formal review. +Incremental follow-up needed an explicit command on [pull request #892][pr-892]. Completion is reported by updating the command reply rather than by creating a new formal review. The collapsed analysis is verbose and can dominate API output. Machine support should read normalized summaries and thread metadata without loading the analysis transcript. @@ -144,13 +147,29 @@ After a corrective push, Qodo updated the existing review comment and its resolv The first sample shows more policy false positives than CodeRabbit. It also supplied the only command-line length finding, which gives it measurable incremental value. +### Generated Mirrors + +`.github/skills/` and `.claude-plugin/fleet-skills/` are the trees `scripts/build_dist.py` generates from `.agents/skills/`, and CI holds them current, so a finding in either belongs at its source and a review of every copy is one finding three times. Two committed files tell CodeRabbit and Qodo to skip them, and Copilot's exclusion is a repository setting this account does not have, so `.github/copilot-instructions.md` asks instead. + +- **CodeRabbit** reads `reviews.path_filters` from [`.coderabbit.yaml`][coderabbit-config] at the repository root, where a pattern prefixed with `!` excludes. +- **Qodo** reads [`.pr_agent.toml`][qodo-config] from the root of the default branch, so the file binds only once it is promoted to `main`, and its [`[ignore]` glob list][qodo-ignore] names the paths to skip. +- **GitHub Copilot** honors [content exclusion][copilot-exclusion], a repository setting under Copilot rather than a file in the tree, whose paths are `fnmatch` patterns, anchored to the repository root by a leading slash and matched anywhere without one. GitHub documents the setting for organizations on a Business or Enterprise plan, and this repository is under a user account, so it is unavailable here. In its place, `.github/copilot-instructions.md` "Reviewing Carried Fleet Content" asks Copilot to post no comment on either tree, and GitHub's [code review customization tutorial][copilot-customize] documents instruction compliance as non-deterministic, so an instruction may be overlooked where a setting cannot. + +### Review Configuration + +Each reviewer's behavior is shaped by a committed file rather than accepted as given, and each setting below carries the finding or the cost that earned it. + +- **CodeRabbit**, in [`.coderabbit.yaml`][coderabbit-config]: auto review on pull requests into `develop`, which the open-source tier honors only at ten stars or more, no pause after five reviewed commits, since a fleet pull request routinely passes five pushes and the pause reads as a reviewer that stopped. A path instruction for Markdown asks for false, stale, unverifiable, or unfollowable claims only, since CI lints style and the local review pass reads canonical prose whole. Sequence diagrams, suggested labels and reviewers, and the in-progress fortune are off. The markdownlint, actionlint, shellcheck, and ruff tools are off, since CI runs the same four and fails the pull request on them. +- **Qodo**, in [`.pr_agent.toml`][qodo-config]: an issues guideline asks for a reproduction with any claimed crash, after a claimed `IsADirectoryError` on this repository's build was disproven by running it. A compliance guideline asks for the rule's own sentence and routes a rule against unchanged text to the summary. Informational findings go to the [summary][qodo-verbosity] rather than a thread, since a thread blocks the merge until resolved. Images are off so a finding's title is plain text a matcher can see. Qodo's [review standards][qodo-rules] import from `AGENTS.md`, `CLAUDE.md`, `copilot-instructions.md`, and `SKILL.md` files, each scoped to its folder at any depth, when changes merge, and only new rules are added, so an edited or deleted rule is changed in its portal instead. +- **GitHub Copilot**: the carried `.github/copilot-instructions.md`, which bootstraps the `code-review` Skill, is the lever this repository uses. GitHub also documents path-scoped `.github/instructions/*.instructions.md` files, unused here. + ## Plan and Repository Scope -The maintainer intends to leave the paid trial when it expires and use only an available no-cost open-source tier. Candidate use is therefore limited to public repositories unless the maintainer approves a later plan change. +The paid trials are over, and both candidates run on their no-cost open-source tiers. Candidate use is therefore limited to public repositories unless the maintainer approves a later plan change. -[CodeRabbit's current plan documentation][coderabbit-plans] provides an open-source tier for public repositories with rate limits. Confirm its terms again when the trial ends because product plans are external state. +[CodeRabbit's current plan documentation][coderabbit-plans] provides an open-source tier for public repositories with rate limits. Product plans are external state, so confirm the terms again before relying on them. -Qodo remains under evaluation. Confirm its current public-repository availability, limits, and required permissions before relying on it outside this repository. Its [code-review documentation][qodo-review] describes the review product but does not settle the fleet's plan decision. +Qodo remains under evaluation on the same footing. Its [code-review documentation][qodo-review] describes the review product but does not settle the fleet's plan decision. Private repositories remain Copilot-only unless a candidate's approved plan, data terms, and GitHub App permissions receive a separate review. @@ -193,7 +212,7 @@ The existing Copilot adapter remains behaviorally unchanged during extraction. P 1. Record every CodeRabbit and Qodo finding on subsequent public pull requests. 2. Measure time to review, current-head coverage, duplicates, and interaction effort. -3. Recheck candidate plan terms when the CodeRabbit trial expires. +3. Recheck candidate plan terms periodically, since product plans are external state. 4. Decide whether either candidate meets the first-class support criteria. 5. Design `pr_review.py` provider adapters only for candidates that graduate. 6. Decide separately whether a graduated reviewer is advisory or required. @@ -210,5 +229,13 @@ The existing Copilot adapter remains behaviorally unchanged during extraction. P +[coderabbit-auto-review]: https://docs.coderabbit.ai/configuration/auto-review +[coderabbit-config]: https://docs.coderabbit.ai/reference/configuration [coderabbit-plans]: https://docs.coderabbit.ai/management/plans +[copilot-customize]: https://docs.github.com/en/copilot/tutorials/customize-code-review +[copilot-exclusion]: https://docs.github.com/en/copilot/how-tos/configure-content-exclusion/exclude-content-from-copilot +[qodo-config]: https://docs.qodo.ai/install-and-configure/configuration-overview/configuration-file +[qodo-ignore]: https://docs.pr-agent.ai/usage-guide/additional_configurations/ [qodo-review]: https://docs.qodo.ai/code-review +[qodo-rules]: https://docs.qodo.ai/governance/rule-enforcement/building-review-standards +[qodo-verbosity]: https://docs.qodo.ai/code-review/review-verbosity diff --git a/reports/canonical-review.json b/reports/canonical-review.json index b1ce3fca..caa7ba74 100644 --- a/reports/canonical-review.json +++ b/reports/canonical-review.json @@ -547,11 +547,11 @@ }, { "unit": ".github/copilot-instructions.md > Reviewing Carried Fleet Content", - "digest": "sha256:83570e2b15143c6f9b79e4d381b7a578d3ae7970d74ae676405a402f6d13630f", + "digest": "sha256:c6d69949edc20e0e5ad9fa133061b2ee09f98b99507067b11ae3583f8d583b1e", "reviewer": "agent-skill", - "findings": 2, - "hubCommit": "f0ff674ec0d59bb2c9ec3736ef22d9de205059c8", - "stamp": "2026-09-02T03:21:55Z" + "findings": 0, + "hubCommit": "d857170243aaca002a5c76cbf193886f00bdfb17", + "stamp": "2026-09-04T19:14:09Z" }, { "unit": "AGENTS.md > Context and Delegation Discipline",